1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides X86-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86EncodingOptimization.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "X86MCSymbolizer.h" 17 #include "bolt/Core/MCPlus.h" 18 #include "bolt/Core/MCPlusBuilder.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCFixupKindInfo.h" 22 #include "llvm/MC/MCInst.h" 23 #include "llvm/MC/MCInstBuilder.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegister.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/DataExtractor.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/Errc.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/ErrorOr.h" 33 #include <set> 34 35 #define DEBUG_TYPE "mcplus" 36 37 using namespace llvm; 38 using namespace bolt; 39 40 namespace opts { 41 42 extern cl::OptionCategory BoltOptCategory; 43 44 static cl::opt<bool> X86StripRedundantAddressSize( 45 "x86-strip-redundant-address-size", 46 cl::desc("Remove redundant Address-Size override prefix"), cl::init(true), 47 cl::cat(BoltOptCategory)); 48 49 } // namespace opts 50 51 namespace { 52 53 bool isMOVSX64rm32(const MCInst &Inst) { 54 return Inst.getOpcode() == X86::MOVSX64rm32; 55 } 56 57 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; } 58 59 bool isADDri(const MCInst &Inst) { 60 return Inst.getOpcode() == X86::ADD64ri32 || 61 Inst.getOpcode() == X86::ADD64ri8; 62 } 63 64 #define GET_INSTRINFO_OPERAND_TYPES_ENUM 65 #define GET_INSTRINFO_OPERAND_TYPE 66 #define GET_INSTRINFO_MEM_OPERAND_SIZE 67 #include "X86GenInstrInfo.inc" 68 69 class X86MCPlusBuilder : public MCPlusBuilder { 70 public: 71 X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, 72 const MCRegisterInfo *RegInfo) 73 : MCPlusBuilder(Analysis, Info, RegInfo) {} 74 75 std::unique_ptr<MCSymbolizer> 76 createTargetSymbolizer(BinaryFunction &Function, 77 bool CreateNewSymbols) const override { 78 return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols); 79 } 80 81 bool isBranch(const MCInst &Inst) const override { 82 return Analysis->isBranch(Inst) && !isTailCall(Inst); 83 } 84 85 bool isNoop(const MCInst &Inst) const override { 86 return X86::isNOP(Inst.getOpcode()); 87 } 88 89 unsigned getCondCode(const MCInst &Inst) const override { 90 unsigned Opcode = Inst.getOpcode(); 91 if (X86::isJCC(Opcode)) 92 return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm(); 93 return X86::COND_INVALID; 94 } 95 96 unsigned getInvertedCondCode(unsigned CC) const override { 97 switch (CC) { 98 default: return X86::COND_INVALID; 99 case X86::COND_E: return X86::COND_NE; 100 case X86::COND_NE: return X86::COND_E; 101 case X86::COND_L: return X86::COND_GE; 102 case X86::COND_LE: return X86::COND_G; 103 case X86::COND_G: return X86::COND_LE; 104 case X86::COND_GE: return X86::COND_L; 105 case X86::COND_B: return X86::COND_AE; 106 case X86::COND_BE: return X86::COND_A; 107 case X86::COND_A: return X86::COND_BE; 108 case X86::COND_AE: return X86::COND_B; 109 case X86::COND_S: return X86::COND_NS; 110 case X86::COND_NS: return X86::COND_S; 111 case X86::COND_P: return X86::COND_NP; 112 case X86::COND_NP: return X86::COND_P; 113 case X86::COND_O: return X86::COND_NO; 114 case X86::COND_NO: return X86::COND_O; 115 } 116 } 117 118 unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override { 119 enum DecodedCondCode : uint8_t { 120 DCC_EQUAL = 0x1, 121 DCC_GREATER = 0x2, 122 DCC_LESSER = 0x4, 123 DCC_GREATER_OR_LESSER = 0x6, 124 DCC_UNSIGNED = 0x8, 125 DCC_SIGNED = 0x10, 126 DCC_INVALID = 0x20, 127 }; 128 129 auto decodeCondCode = [&](unsigned CC) -> uint8_t { 130 switch (CC) { 131 default: return DCC_INVALID; 132 case X86::COND_E: return DCC_EQUAL; 133 case X86::COND_NE: return DCC_GREATER | DCC_LESSER; 134 case X86::COND_L: return DCC_LESSER | DCC_SIGNED; 135 case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED; 136 case X86::COND_G: return DCC_GREATER | DCC_SIGNED; 137 case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED; 138 case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED; 139 case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED; 140 case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED; 141 case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED; 142 } 143 }; 144 145 uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2); 146 147 if (DCC & DCC_INVALID) 148 return X86::COND_INVALID; 149 150 if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED) 151 return X86::COND_INVALID; 152 153 switch (DCC) { 154 default: return X86::COND_INVALID; 155 case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE; 156 case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE; 157 case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE; 158 case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE; 159 case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE; 160 case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE; 161 case DCC_GREATER | DCC_LESSER: return X86::COND_NE; 162 case DCC_EQUAL | DCC_SIGNED: return X86::COND_E; 163 case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E; 164 case DCC_EQUAL: return X86::COND_E; 165 case DCC_LESSER | DCC_SIGNED: return X86::COND_L; 166 case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B; 167 case DCC_GREATER | DCC_SIGNED: return X86::COND_G; 168 case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A; 169 } 170 } 171 172 bool isValidCondCode(unsigned CC) const override { 173 return (CC != X86::COND_INVALID); 174 } 175 176 bool isBreakpoint(const MCInst &Inst) const override { 177 return Inst.getOpcode() == X86::INT3; 178 } 179 180 bool isPrefix(const MCInst &Inst) const override { 181 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 182 return X86II::isPrefix(Desc.TSFlags); 183 } 184 185 bool isRep(const MCInst &Inst) const override { 186 return Inst.getFlags() == X86::IP_HAS_REPEAT; 187 } 188 189 bool deleteREPPrefix(MCInst &Inst) const override { 190 if (Inst.getFlags() == X86::IP_HAS_REPEAT) { 191 Inst.setFlags(0); 192 return true; 193 } 194 return false; 195 } 196 197 // FIXME: For compatibility with old LLVM only! 198 bool isTerminator(const MCInst &Inst) const override { 199 unsigned Opcode = Inst.getOpcode(); 200 return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || 201 X86::isUD2(Opcode); 202 } 203 204 bool isIndirectCall(const MCInst &Inst) const override { 205 return isCall(Inst) && 206 ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); 207 } 208 209 bool isPop(const MCInst &Inst) const override { 210 return getPopSize(Inst) == 0 ? false : true; 211 } 212 213 bool isTerminateBranch(const MCInst &Inst) const override { 214 return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64; 215 } 216 217 int getPopSize(const MCInst &Inst) const override { 218 switch (Inst.getOpcode()) { 219 case X86::POP16r: 220 case X86::POP16rmm: 221 case X86::POP16rmr: 222 case X86::POPF16: 223 case X86::POPA16: 224 case X86::POPDS16: 225 case X86::POPES16: 226 case X86::POPFS16: 227 case X86::POPGS16: 228 case X86::POPSS16: 229 return 2; 230 case X86::POP32r: 231 case X86::POP32rmm: 232 case X86::POP32rmr: 233 case X86::POPA32: 234 case X86::POPDS32: 235 case X86::POPES32: 236 case X86::POPF32: 237 case X86::POPFS32: 238 case X86::POPGS32: 239 case X86::POPSS32: 240 return 4; 241 case X86::POP64r: 242 case X86::POP64rmm: 243 case X86::POP64rmr: 244 case X86::POPF64: 245 case X86::POPFS64: 246 case X86::POPGS64: 247 return 8; 248 } 249 return 0; 250 } 251 252 bool isPush(const MCInst &Inst) const override { 253 return getPushSize(Inst) == 0 ? false : true; 254 } 255 256 int getPushSize(const MCInst &Inst) const override { 257 switch (Inst.getOpcode()) { 258 case X86::PUSH16i8: 259 case X86::PUSH16r: 260 case X86::PUSH16rmm: 261 case X86::PUSH16rmr: 262 case X86::PUSHA16: 263 case X86::PUSHCS16: 264 case X86::PUSHDS16: 265 case X86::PUSHES16: 266 case X86::PUSHF16: 267 case X86::PUSHFS16: 268 case X86::PUSHGS16: 269 case X86::PUSHSS16: 270 case X86::PUSH16i: 271 return 2; 272 case X86::PUSH32i8: 273 case X86::PUSH32r: 274 case X86::PUSH32rmm: 275 case X86::PUSH32rmr: 276 case X86::PUSHA32: 277 case X86::PUSHCS32: 278 case X86::PUSHDS32: 279 case X86::PUSHES32: 280 case X86::PUSHF32: 281 case X86::PUSHFS32: 282 case X86::PUSHGS32: 283 case X86::PUSHSS32: 284 case X86::PUSH32i: 285 return 4; 286 case X86::PUSH64i32: 287 case X86::PUSH64i8: 288 case X86::PUSH64r: 289 case X86::PUSH64rmm: 290 case X86::PUSH64rmr: 291 case X86::PUSHF64: 292 case X86::PUSHFS64: 293 case X86::PUSHGS64: 294 return 8; 295 } 296 return 0; 297 } 298 299 bool isSUB(const MCInst &Inst) const override { 300 return X86::isSUB(Inst.getOpcode()); 301 } 302 303 bool isLEA64r(const MCInst &Inst) const override { 304 return Inst.getOpcode() == X86::LEA64r; 305 } 306 307 bool isLeave(const MCInst &Inst) const override { 308 return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; 309 } 310 311 bool isMoveMem2Reg(const MCInst &Inst) const override { 312 switch (Inst.getOpcode()) { 313 case X86::MOV16rm: 314 case X86::MOV32rm: 315 case X86::MOV64rm: 316 return true; 317 } 318 return false; 319 } 320 321 bool isUnsupportedBranch(unsigned Opcode) const override { 322 switch (Opcode) { 323 default: 324 return false; 325 case X86::LOOP: 326 case X86::LOOPE: 327 case X86::LOOPNE: 328 case X86::JECXZ: 329 case X86::JRCXZ: 330 return true; 331 } 332 } 333 334 bool isLoad(const MCInst &Inst) const override { 335 if (isPop(Inst)) 336 return true; 337 338 int MemOpNo = getMemoryOperandNo(Inst); 339 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 340 341 if (MemOpNo == -1) 342 return false; 343 344 return MCII.mayLoad(); 345 } 346 347 bool isStore(const MCInst &Inst) const override { 348 if (isPush(Inst)) 349 return true; 350 351 int MemOpNo = getMemoryOperandNo(Inst); 352 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 353 354 if (MemOpNo == -1) 355 return false; 356 357 return MCII.mayStore(); 358 } 359 360 bool isCleanRegXOR(const MCInst &Inst) const override { 361 switch (Inst.getOpcode()) { 362 case X86::XOR16rr: 363 case X86::XOR32rr: 364 case X86::XOR64rr: 365 break; 366 default: 367 return false; 368 } 369 return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg()); 370 } 371 372 bool isPacked(const MCInst &Inst) const override { 373 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 374 return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD; 375 } 376 377 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 378 switch (RelType) { 379 case ELF::R_X86_64_8: 380 case ELF::R_X86_64_16: 381 case ELF::R_X86_64_32: 382 case ELF::R_X86_64_32S: 383 case ELF::R_X86_64_64: 384 case ELF::R_X86_64_PC8: 385 case ELF::R_X86_64_PC32: 386 case ELF::R_X86_64_PC64: 387 case ELF::R_X86_64_GOTPCRELX: 388 case ELF::R_X86_64_REX_GOTPCRELX: 389 return true; 390 case ELF::R_X86_64_PLT32: 391 case ELF::R_X86_64_GOTPCREL: 392 case ELF::R_X86_64_TPOFF32: 393 case ELF::R_X86_64_GOTTPOFF: 394 return false; 395 default: 396 llvm_unreachable("Unexpected x86 relocation type in code"); 397 } 398 } 399 400 unsigned getTrapFillValue() const override { return 0xCC; } 401 402 struct IndJmpMatcherFrag1 : MCInstMatcher { 403 std::unique_ptr<MCInstMatcher> Base; 404 std::unique_ptr<MCInstMatcher> Scale; 405 std::unique_ptr<MCInstMatcher> Index; 406 std::unique_ptr<MCInstMatcher> Offset; 407 408 IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 409 std::unique_ptr<MCInstMatcher> Scale, 410 std::unique_ptr<MCInstMatcher> Index, 411 std::unique_ptr<MCInstMatcher> Offset) 412 : Base(std::move(Base)), Scale(std::move(Scale)), 413 Index(std::move(Index)), Offset(std::move(Offset)) {} 414 415 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 416 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 417 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 418 return false; 419 420 if (CurInst->getOpcode() != X86::JMP64m) 421 return false; 422 423 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 424 if (MemOpNo == -1) 425 return false; 426 427 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 428 return false; 429 if (!Scale->match(MRI, MIB, this->InstrWindow, 430 MemOpNo + X86::AddrScaleAmt)) 431 return false; 432 if (!Index->match(MRI, MIB, this->InstrWindow, 433 MemOpNo + X86::AddrIndexReg)) 434 return false; 435 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 436 return false; 437 return true; 438 } 439 440 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 441 MIB.addAnnotation(*CurInst, Annotation, true); 442 Base->annotate(MIB, Annotation); 443 Scale->annotate(MIB, Annotation); 444 Index->annotate(MIB, Annotation); 445 Offset->annotate(MIB, Annotation); 446 } 447 }; 448 449 std::unique_ptr<MCInstMatcher> 450 matchIndJmp(std::unique_ptr<MCInstMatcher> Base, 451 std::unique_ptr<MCInstMatcher> Scale, 452 std::unique_ptr<MCInstMatcher> Index, 453 std::unique_ptr<MCInstMatcher> Offset) const override { 454 return std::unique_ptr<MCInstMatcher>( 455 new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), 456 std::move(Index), std::move(Offset))); 457 } 458 459 struct IndJmpMatcherFrag2 : MCInstMatcher { 460 std::unique_ptr<MCInstMatcher> Reg; 461 462 IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg) 463 : Reg(std::move(Reg)) {} 464 465 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 466 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 467 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 468 return false; 469 470 if (CurInst->getOpcode() != X86::JMP64r) 471 return false; 472 473 return Reg->match(MRI, MIB, this->InstrWindow, 0); 474 } 475 476 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 477 MIB.addAnnotation(*CurInst, Annotation, true); 478 Reg->annotate(MIB, Annotation); 479 } 480 }; 481 482 std::unique_ptr<MCInstMatcher> 483 matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override { 484 return std::unique_ptr<MCInstMatcher>( 485 new IndJmpMatcherFrag2(std::move(Target))); 486 } 487 488 struct LoadMatcherFrag1 : MCInstMatcher { 489 std::unique_ptr<MCInstMatcher> Base; 490 std::unique_ptr<MCInstMatcher> Scale; 491 std::unique_ptr<MCInstMatcher> Index; 492 std::unique_ptr<MCInstMatcher> Offset; 493 494 LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 495 std::unique_ptr<MCInstMatcher> Scale, 496 std::unique_ptr<MCInstMatcher> Index, 497 std::unique_ptr<MCInstMatcher> Offset) 498 : Base(std::move(Base)), Scale(std::move(Scale)), 499 Index(std::move(Index)), Offset(std::move(Offset)) {} 500 501 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 502 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 503 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 504 return false; 505 506 if (CurInst->getOpcode() != X86::MOV64rm && 507 CurInst->getOpcode() != X86::MOVSX64rm32) 508 return false; 509 510 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 511 if (MemOpNo == -1) 512 return false; 513 514 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 515 return false; 516 if (!Scale->match(MRI, MIB, this->InstrWindow, 517 MemOpNo + X86::AddrScaleAmt)) 518 return false; 519 if (!Index->match(MRI, MIB, this->InstrWindow, 520 MemOpNo + X86::AddrIndexReg)) 521 return false; 522 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 523 return false; 524 return true; 525 } 526 527 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 528 MIB.addAnnotation(*CurInst, Annotation, true); 529 Base->annotate(MIB, Annotation); 530 Scale->annotate(MIB, Annotation); 531 Index->annotate(MIB, Annotation); 532 Offset->annotate(MIB, Annotation); 533 } 534 }; 535 536 std::unique_ptr<MCInstMatcher> 537 matchLoad(std::unique_ptr<MCInstMatcher> Base, 538 std::unique_ptr<MCInstMatcher> Scale, 539 std::unique_ptr<MCInstMatcher> Index, 540 std::unique_ptr<MCInstMatcher> Offset) const override { 541 return std::unique_ptr<MCInstMatcher>( 542 new LoadMatcherFrag1(std::move(Base), std::move(Scale), 543 std::move(Index), std::move(Offset))); 544 } 545 546 struct AddMatcher : MCInstMatcher { 547 std::unique_ptr<MCInstMatcher> A; 548 std::unique_ptr<MCInstMatcher> B; 549 550 AddMatcher(std::unique_ptr<MCInstMatcher> A, 551 std::unique_ptr<MCInstMatcher> B) 552 : A(std::move(A)), B(std::move(B)) {} 553 554 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 555 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 556 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 557 return false; 558 559 if (CurInst->getOpcode() == X86::ADD64rr || 560 CurInst->getOpcode() == X86::ADD64rr_DB || 561 CurInst->getOpcode() == X86::ADD64rr_REV) { 562 if (!A->match(MRI, MIB, this->InstrWindow, 1)) { 563 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 564 return false; 565 return A->match(MRI, MIB, this->InstrWindow, 2); 566 } 567 568 if (B->match(MRI, MIB, this->InstrWindow, 2)) 569 return true; 570 571 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 572 return false; 573 return A->match(MRI, MIB, this->InstrWindow, 2); 574 } 575 576 return false; 577 } 578 579 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 580 MIB.addAnnotation(*CurInst, Annotation, true); 581 A->annotate(MIB, Annotation); 582 B->annotate(MIB, Annotation); 583 } 584 }; 585 586 std::unique_ptr<MCInstMatcher> 587 matchAdd(std::unique_ptr<MCInstMatcher> A, 588 std::unique_ptr<MCInstMatcher> B) const override { 589 return std::unique_ptr<MCInstMatcher>( 590 new AddMatcher(std::move(A), std::move(B))); 591 } 592 593 struct LEAMatcher : MCInstMatcher { 594 std::unique_ptr<MCInstMatcher> Target; 595 596 LEAMatcher(std::unique_ptr<MCInstMatcher> Target) 597 : Target(std::move(Target)) {} 598 599 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 600 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 601 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 602 return false; 603 604 if (CurInst->getOpcode() != X86::LEA64r) 605 return false; 606 607 if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || 608 CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != 609 X86::NoRegister || 610 (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != 611 X86::NoRegister && 612 CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) 613 return false; 614 615 return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); 616 } 617 618 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 619 MIB.addAnnotation(*CurInst, Annotation, true); 620 Target->annotate(MIB, Annotation); 621 } 622 }; 623 624 std::unique_ptr<MCInstMatcher> 625 matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override { 626 return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target))); 627 } 628 629 bool hasPCRelOperand(const MCInst &Inst) const override { 630 for (const MCOperand &Operand : Inst) 631 if (Operand.isReg() && Operand.getReg() == X86::RIP) 632 return true; 633 return false; 634 } 635 636 int getMemoryOperandNo(const MCInst &Inst) const override { 637 unsigned Opcode = Inst.getOpcode(); 638 const MCInstrDesc &Desc = Info->get(Opcode); 639 int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); 640 if (MemOpNo >= 0) 641 MemOpNo += X86II::getOperandBias(Desc); 642 return MemOpNo; 643 } 644 645 bool hasEVEXEncoding(const MCInst &Inst) const override { 646 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 647 return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; 648 } 649 650 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override { 651 const auto *I = Insts.begin(); 652 while (I != Insts.end() && isPrefix(*I)) 653 ++I; 654 if (I == Insts.end()) 655 return false; 656 657 const MCInst &FirstInst = *I; 658 ++I; 659 while (I != Insts.end() && isPrefix(*I)) 660 ++I; 661 if (I == Insts.end()) 662 return false; 663 const MCInst &SecondInst = *I; 664 665 if (!isConditionalBranch(SecondInst)) 666 return false; 667 // Cannot fuse if the first instruction uses RIP-relative memory. 668 if (hasPCRelOperand(FirstInst)) 669 return false; 670 671 const X86::FirstMacroFusionInstKind CmpKind = 672 X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode()); 673 if (CmpKind == X86::FirstMacroFusionInstKind::Invalid) 674 return false; 675 676 X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst)); 677 X86::SecondMacroFusionInstKind BranchKind = 678 X86::classifySecondCondCodeInMacroFusion(CC); 679 if (BranchKind == X86::SecondMacroFusionInstKind::Invalid) 680 return false; 681 return X86::isMacroFused(CmpKind, BranchKind); 682 } 683 684 std::optional<X86MemOperand> 685 evaluateX86MemoryOperand(const MCInst &Inst) const override { 686 int MemOpNo = getMemoryOperandNo(Inst); 687 if (MemOpNo < 0) 688 return std::nullopt; 689 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 690 691 if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst)) 692 return std::nullopt; 693 694 const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); 695 const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); 696 const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); 697 const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); 698 const MCOperand &Segment = 699 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); 700 701 // Make sure it is a well-formed memory operand. 702 if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || 703 (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) 704 return std::nullopt; 705 706 X86MemOperand MO; 707 MO.BaseRegNum = Base.getReg(); 708 MO.ScaleImm = Scale.getImm(); 709 MO.IndexRegNum = Index.getReg(); 710 MO.DispImm = Disp.isImm() ? Disp.getImm() : 0; 711 MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr; 712 MO.SegRegNum = Segment.getReg(); 713 return MO; 714 } 715 716 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 717 uint64_t Address, 718 uint64_t Size) const override { 719 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 720 if (!MO) 721 return false; 722 723 // Make sure it's a well-formed addressing we can statically evaluate. 724 if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) || 725 MO->IndexRegNum != X86::NoRegister || 726 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 727 return false; 728 729 Target = MO->DispImm; 730 if (MO->BaseRegNum == X86::RIP) { 731 assert(Size != 0 && "instruction size required in order to statically " 732 "evaluate RIP-relative address"); 733 Target += Address + Size; 734 } 735 return true; 736 } 737 738 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 739 int MemOpNo = getMemoryOperandNo(Inst); 740 if (MemOpNo < 0) 741 return Inst.end(); 742 return Inst.begin() + (MemOpNo + X86::AddrDisp); 743 } 744 745 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 746 MCOperand *OI = getMemOperandDisp(Inst); 747 if (OI == Inst.end()) 748 return false; 749 *OI = Operand; 750 return true; 751 } 752 753 /// Get the registers used as function parameters. 754 /// This function is specific to the x86_64 abi on Linux. 755 BitVector getRegsUsedAsParams() const override { 756 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 757 Regs |= getAliases(X86::RSI); 758 Regs |= getAliases(X86::RDI); 759 Regs |= getAliases(X86::RDX); 760 Regs |= getAliases(X86::RCX); 761 Regs |= getAliases(X86::R8); 762 Regs |= getAliases(X86::R9); 763 return Regs; 764 } 765 766 void getCalleeSavedRegs(BitVector &Regs) const override { 767 Regs |= getAliases(X86::RBX); 768 Regs |= getAliases(X86::RBP); 769 Regs |= getAliases(X86::R12); 770 Regs |= getAliases(X86::R13); 771 Regs |= getAliases(X86::R14); 772 Regs |= getAliases(X86::R15); 773 } 774 775 void getDefaultDefIn(BitVector &Regs) const override { 776 assert(Regs.size() >= RegInfo->getNumRegs() && 777 "The size of BitVector is less than RegInfo->getNumRegs()."); 778 Regs.set(X86::RAX); 779 Regs.set(X86::RCX); 780 Regs.set(X86::RDX); 781 Regs.set(X86::RSI); 782 Regs.set(X86::RDI); 783 Regs.set(X86::R8); 784 Regs.set(X86::R9); 785 Regs.set(X86::XMM0); 786 Regs.set(X86::XMM1); 787 Regs.set(X86::XMM2); 788 Regs.set(X86::XMM3); 789 Regs.set(X86::XMM4); 790 Regs.set(X86::XMM5); 791 Regs.set(X86::XMM6); 792 Regs.set(X86::XMM7); 793 } 794 795 void getDefaultLiveOut(BitVector &Regs) const override { 796 assert(Regs.size() >= RegInfo->getNumRegs() && 797 "The size of BitVector is less than RegInfo->getNumRegs()."); 798 Regs |= getAliases(X86::RAX); 799 Regs |= getAliases(X86::RDX); 800 Regs |= getAliases(X86::RCX); 801 Regs |= getAliases(X86::XMM0); 802 Regs |= getAliases(X86::XMM1); 803 } 804 805 void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { 806 if (IncludeAlias) { 807 Regs |= getAliases(X86::RAX); 808 Regs |= getAliases(X86::RBX); 809 Regs |= getAliases(X86::RBP); 810 Regs |= getAliases(X86::RSI); 811 Regs |= getAliases(X86::RDI); 812 Regs |= getAliases(X86::RDX); 813 Regs |= getAliases(X86::RCX); 814 Regs |= getAliases(X86::R8); 815 Regs |= getAliases(X86::R9); 816 Regs |= getAliases(X86::R10); 817 Regs |= getAliases(X86::R11); 818 Regs |= getAliases(X86::R12); 819 Regs |= getAliases(X86::R13); 820 Regs |= getAliases(X86::R14); 821 Regs |= getAliases(X86::R15); 822 return; 823 } 824 Regs.set(X86::RAX); 825 Regs.set(X86::RBX); 826 Regs.set(X86::RBP); 827 Regs.set(X86::RSI); 828 Regs.set(X86::RDI); 829 Regs.set(X86::RDX); 830 Regs.set(X86::RCX); 831 Regs.set(X86::R8); 832 Regs.set(X86::R9); 833 Regs.set(X86::R10); 834 Regs.set(X86::R11); 835 Regs.set(X86::R12); 836 Regs.set(X86::R13); 837 Regs.set(X86::R14); 838 Regs.set(X86::R15); 839 } 840 841 void getClassicGPRegs(BitVector &Regs) const override { 842 Regs |= getAliases(X86::RAX); 843 Regs |= getAliases(X86::RBX); 844 Regs |= getAliases(X86::RBP); 845 Regs |= getAliases(X86::RSI); 846 Regs |= getAliases(X86::RDI); 847 Regs |= getAliases(X86::RDX); 848 Regs |= getAliases(X86::RCX); 849 } 850 851 void getRepRegs(BitVector &Regs) const override { 852 Regs |= getAliases(X86::RCX); 853 } 854 855 MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { 856 Reg = getX86SubSuperRegister(Reg, Size * 8); 857 assert((Reg != X86::NoRegister) && "Invalid register"); 858 return Reg; 859 } 860 861 bool isUpper8BitReg(MCPhysReg Reg) const override { 862 switch (Reg) { 863 case X86::AH: 864 case X86::BH: 865 case X86::CH: 866 case X86::DH: 867 return true; 868 default: 869 return false; 870 } 871 } 872 873 bool cannotUseREX(const MCInst &Inst) const override { 874 switch (Inst.getOpcode()) { 875 case X86::MOV8mr_NOREX: 876 case X86::MOV8rm_NOREX: 877 case X86::MOV8rr_NOREX: 878 case X86::MOVSX32rm8_NOREX: 879 case X86::MOVSX32rr8_NOREX: 880 case X86::MOVZX32rm8_NOREX: 881 case X86::MOVZX32rr8_NOREX: 882 case X86::MOV8mr: 883 case X86::MOV8rm: 884 case X86::MOV8rr: 885 case X86::MOVSX32rm8: 886 case X86::MOVSX32rr8: 887 case X86::MOVZX32rm8: 888 case X86::MOVZX32rr8: 889 case X86::TEST8ri: 890 for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) { 891 if (!Operand.isReg()) 892 continue; 893 if (isUpper8BitReg(Operand.getReg())) 894 return true; 895 } 896 [[fallthrough]]; 897 default: 898 return false; 899 } 900 } 901 902 static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) { 903 using namespace llvm::X86; 904 int OpType = getOperandType(Inst.getOpcode(), MemOpNo); 905 return getMemOperandSize(OpType) / 8; 906 } 907 908 /// Classifying a stack access as *not* "SIMPLE" here means we don't know how 909 /// to change this instruction memory access. It will disable any changes to 910 /// the stack layout, so we can't do the most aggressive form of shrink 911 /// wrapping. We must do so in a way that keeps the original stack layout. 912 /// Otherwise you need to adjust the offset of all instructions accessing the 913 /// stack: we can't do that anymore because there is one instruction that is 914 /// not simple. There are other implications as well. We have heuristics to 915 /// detect when a register is callee-saved and thus eligible for shrink 916 /// wrapping. If you are restoring a register using a non-simple stack access, 917 /// then it is classified as NOT callee-saved, and it disables shrink wrapping 918 /// for *that* register (but not for others). 919 /// 920 /// Classifying a stack access as "size 0" or detecting an indexed memory 921 /// access (to address a vector, for example) here means we know there is a 922 /// stack access, but we can't quite understand how wide is the access in 923 /// bytes. This is very serious because we can't understand how memory 924 /// accesses alias with each other for this function. This will essentially 925 /// disable not only shrink wrapping but all frame analysis, it will fail it 926 /// as "we don't understand this function and we give up on it". 927 bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, 928 bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, 929 uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size, 930 bool &IsSimple, bool &IsIndexed) const override { 931 // Detect simple push/pop cases first 932 if (int Sz = getPushSize(Inst)) { 933 IsLoad = false; 934 IsStore = true; 935 IsStoreFromReg = true; 936 StackPtrReg = X86::RSP; 937 StackOffset = -Sz; 938 Size = Sz; 939 IsSimple = true; 940 if (Inst.getOperand(0).isImm()) 941 SrcImm = Inst.getOperand(0).getImm(); 942 else if (Inst.getOperand(0).isReg()) 943 Reg = Inst.getOperand(0).getReg(); 944 else 945 IsSimple = false; 946 947 return true; 948 } 949 if (int Sz = getPopSize(Inst)) { 950 IsLoad = true; 951 IsStore = false; 952 if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) { 953 IsSimple = false; 954 } else { 955 Reg = Inst.getOperand(0).getReg(); 956 IsSimple = true; 957 } 958 StackPtrReg = X86::RSP; 959 StackOffset = 0; 960 Size = Sz; 961 return true; 962 } 963 964 struct InstInfo { 965 // Size in bytes that Inst loads from memory. 966 uint8_t DataSize; 967 bool IsLoad; 968 bool IsStore; 969 bool StoreFromReg; 970 bool Simple; 971 }; 972 973 InstInfo I; 974 int MemOpNo = getMemoryOperandNo(Inst); 975 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 976 // If it is not dealing with a memory operand, we discard it 977 if (MemOpNo == -1 || MCII.isCall()) 978 return false; 979 980 switch (Inst.getOpcode()) { 981 default: { 982 bool IsLoad = MCII.mayLoad(); 983 bool IsStore = MCII.mayStore(); 984 // Is it LEA? (deals with memory but is not loading nor storing) 985 if (!IsLoad && !IsStore) { 986 I = {0, IsLoad, IsStore, false, false}; 987 break; 988 } 989 uint8_t Sz = getMemDataSize(Inst, MemOpNo); 990 I = {Sz, IsLoad, IsStore, false, false}; 991 break; 992 } 993 // Report simple stack accesses 994 case X86::MOV8rm: I = {1, true, false, false, true}; break; 995 case X86::MOV16rm: I = {2, true, false, false, true}; break; 996 case X86::MOV32rm: I = {4, true, false, false, true}; break; 997 case X86::MOV64rm: I = {8, true, false, false, true}; break; 998 case X86::MOV8mr: I = {1, false, true, true, true}; break; 999 case X86::MOV16mr: I = {2, false, true, true, true}; break; 1000 case X86::MOV32mr: I = {4, false, true, true, true}; break; 1001 case X86::MOV64mr: I = {8, false, true, true, true}; break; 1002 case X86::MOV8mi: I = {1, false, true, false, true}; break; 1003 case X86::MOV16mi: I = {2, false, true, false, true}; break; 1004 case X86::MOV32mi: I = {4, false, true, false, true}; break; 1005 } // end switch (Inst.getOpcode()) 1006 1007 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1008 if (!MO) { 1009 LLVM_DEBUG(dbgs() << "Evaluate failed on "); 1010 LLVM_DEBUG(Inst.dump()); 1011 return false; 1012 } 1013 1014 // Make sure it's a stack access 1015 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) 1016 return false; 1017 1018 IsLoad = I.IsLoad; 1019 IsStore = I.IsStore; 1020 IsStoreFromReg = I.StoreFromReg; 1021 Size = I.DataSize; 1022 IsSimple = I.Simple; 1023 StackPtrReg = MO->BaseRegNum; 1024 StackOffset = MO->DispImm; 1025 IsIndexed = 1026 MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister; 1027 1028 if (!I.Simple) 1029 return true; 1030 1031 // Retrieve related register in simple MOV from/to stack operations. 1032 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 1033 if (I.IsLoad) { 1034 MCOperand RegOpnd = Inst.getOperand(0); 1035 assert(RegOpnd.isReg() && "unexpected destination operand"); 1036 Reg = RegOpnd.getReg(); 1037 } else if (I.IsStore) { 1038 MCOperand SrcOpnd = 1039 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1040 if (I.StoreFromReg) { 1041 assert(SrcOpnd.isReg() && "unexpected source operand"); 1042 Reg = SrcOpnd.getReg(); 1043 } else { 1044 assert(SrcOpnd.isImm() && "unexpected source operand"); 1045 SrcImm = SrcOpnd.getImm(); 1046 } 1047 } 1048 1049 return true; 1050 } 1051 1052 void changeToPushOrPop(MCInst &Inst) const override { 1053 assert(!isPush(Inst) && !isPop(Inst)); 1054 1055 struct InstInfo { 1056 // Size in bytes that Inst loads from memory. 1057 uint8_t DataSize; 1058 bool IsLoad; 1059 bool StoreFromReg; 1060 }; 1061 1062 InstInfo I; 1063 switch (Inst.getOpcode()) { 1064 default: { 1065 llvm_unreachable("Unhandled opcode"); 1066 return; 1067 } 1068 case X86::MOV16rm: I = {2, true, false}; break; 1069 case X86::MOV32rm: I = {4, true, false}; break; 1070 case X86::MOV64rm: I = {8, true, false}; break; 1071 case X86::MOV16mr: I = {2, false, true}; break; 1072 case X86::MOV32mr: I = {4, false, true}; break; 1073 case X86::MOV64mr: I = {8, false, true}; break; 1074 case X86::MOV16mi: I = {2, false, false}; break; 1075 case X86::MOV32mi: I = {4, false, false}; break; 1076 } // end switch (Inst.getOpcode()) 1077 1078 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1079 if (!MO) { 1080 llvm_unreachable("Evaluate failed"); 1081 return; 1082 } 1083 // Make sure it's a stack access 1084 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) { 1085 llvm_unreachable("Not a stack access"); 1086 return; 1087 } 1088 1089 unsigned MemOpOffset = getMemoryOperandNo(Inst); 1090 unsigned NewOpcode = 0; 1091 if (I.IsLoad) { 1092 switch (I.DataSize) { 1093 case 2: NewOpcode = X86::POP16r; break; 1094 case 4: NewOpcode = X86::POP32r; break; 1095 case 8: NewOpcode = X86::POP64r; break; 1096 default: 1097 llvm_unreachable("Unexpected size"); 1098 } 1099 unsigned RegOpndNum = Inst.getOperand(0).getReg(); 1100 Inst.clear(); 1101 Inst.setOpcode(NewOpcode); 1102 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1103 } else { 1104 MCOperand SrcOpnd = 1105 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1106 if (I.StoreFromReg) { 1107 switch (I.DataSize) { 1108 case 2: NewOpcode = X86::PUSH16r; break; 1109 case 4: NewOpcode = X86::PUSH32r; break; 1110 case 8: NewOpcode = X86::PUSH64r; break; 1111 default: 1112 llvm_unreachable("Unexpected size"); 1113 } 1114 assert(SrcOpnd.isReg() && "Unexpected source operand"); 1115 unsigned RegOpndNum = SrcOpnd.getReg(); 1116 Inst.clear(); 1117 Inst.setOpcode(NewOpcode); 1118 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1119 } else { 1120 switch (I.DataSize) { 1121 case 2: NewOpcode = X86::PUSH16i8; break; 1122 case 4: NewOpcode = X86::PUSH32i8; break; 1123 case 8: NewOpcode = X86::PUSH64i32; break; 1124 default: 1125 llvm_unreachable("Unexpected size"); 1126 } 1127 assert(SrcOpnd.isImm() && "Unexpected source operand"); 1128 int64_t SrcImm = SrcOpnd.getImm(); 1129 Inst.clear(); 1130 Inst.setOpcode(NewOpcode); 1131 Inst.addOperand(MCOperand::createImm(SrcImm)); 1132 } 1133 } 1134 } 1135 1136 bool isStackAdjustment(const MCInst &Inst) const override { 1137 switch (Inst.getOpcode()) { 1138 default: 1139 return false; 1140 case X86::SUB64ri32: 1141 case X86::SUB64ri8: 1142 case X86::ADD64ri32: 1143 case X86::ADD64ri8: 1144 case X86::LEA64r: 1145 break; 1146 } 1147 1148 return any_of(defOperands(Inst), [](const MCOperand &Op) { 1149 return Op.isReg() && Op.getReg() == X86::RSP; 1150 }); 1151 } 1152 1153 bool 1154 evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output, 1155 std::pair<MCPhysReg, int64_t> Input1, 1156 std::pair<MCPhysReg, int64_t> Input2) const override { 1157 1158 auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> { 1159 if (Reg == Input1.first) 1160 return Input1.second; 1161 if (Reg == Input2.first) 1162 return Input2.second; 1163 return make_error_code(errc::result_out_of_range); 1164 }; 1165 1166 switch (Inst.getOpcode()) { 1167 default: 1168 return false; 1169 1170 case X86::SUB64ri32: 1171 case X86::SUB64ri8: 1172 if (!Inst.getOperand(2).isImm()) 1173 return false; 1174 if (ErrorOr<int64_t> InputVal = 1175 getOperandVal(Inst.getOperand(1).getReg())) 1176 Output = *InputVal - Inst.getOperand(2).getImm(); 1177 else 1178 return false; 1179 break; 1180 case X86::ADD64ri32: 1181 case X86::ADD64ri8: 1182 if (!Inst.getOperand(2).isImm()) 1183 return false; 1184 if (ErrorOr<int64_t> InputVal = 1185 getOperandVal(Inst.getOperand(1).getReg())) 1186 Output = *InputVal + Inst.getOperand(2).getImm(); 1187 else 1188 return false; 1189 break; 1190 case X86::ADD64i32: 1191 if (!Inst.getOperand(0).isImm()) 1192 return false; 1193 if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) 1194 Output = *InputVal + Inst.getOperand(0).getImm(); 1195 else 1196 return false; 1197 break; 1198 1199 case X86::LEA64r: { 1200 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1201 if (!MO) 1202 return false; 1203 1204 if (MO->BaseRegNum == X86::NoRegister || 1205 MO->IndexRegNum != X86::NoRegister || 1206 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 1207 return false; 1208 1209 if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum)) 1210 Output = *InputVal + MO->DispImm; 1211 else 1212 return false; 1213 1214 break; 1215 } 1216 } 1217 return true; 1218 } 1219 1220 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 1221 MCPhysReg &To) const override { 1222 switch (Inst.getOpcode()) { 1223 default: 1224 return false; 1225 case X86::LEAVE: 1226 case X86::LEAVE64: 1227 To = getStackPointer(); 1228 From = getFramePointer(); 1229 return true; 1230 case X86::MOV64rr: 1231 To = Inst.getOperand(0).getReg(); 1232 From = Inst.getOperand(1).getReg(); 1233 return true; 1234 } 1235 } 1236 1237 MCPhysReg getStackPointer() const override { return X86::RSP; } 1238 MCPhysReg getFramePointer() const override { return X86::RBP; } 1239 MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } 1240 1241 bool escapesVariable(const MCInst &Inst, 1242 bool HasFramePointer) const override { 1243 int MemOpNo = getMemoryOperandNo(Inst); 1244 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 1245 const unsigned NumDefs = MCII.getNumDefs(); 1246 static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |= 1247 getAliases(X86::RBP)); 1248 static BitVector SPAliases(getAliases(X86::RSP)); 1249 1250 // FIXME: PUSH can be technically a leak, but let's ignore this for now 1251 // because a lot of harmless prologue code will spill SP to the stack. 1252 // Unless push is clearly pushing an object address to the stack as 1253 // demonstrated by having a MemOp. 1254 bool IsPush = isPush(Inst); 1255 if (IsPush && MemOpNo == -1) 1256 return false; 1257 1258 // We use this to detect LEA (has memop but does not access mem) 1259 bool AccessMem = MCII.mayLoad() || MCII.mayStore(); 1260 bool DoesLeak = false; 1261 for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) { 1262 // Ignore if SP/BP is used to dereference memory -- that's fine 1263 if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && 1264 I <= MemOpNo + 5) 1265 continue; 1266 // Ignore if someone is writing to SP/BP 1267 if (I < static_cast<int>(NumDefs)) 1268 continue; 1269 1270 const MCOperand &Operand = Inst.getOperand(I); 1271 if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { 1272 DoesLeak = true; 1273 break; 1274 } 1275 if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { 1276 DoesLeak = true; 1277 break; 1278 } 1279 } 1280 1281 // If potential leak, check if it is not just writing to itself/sp/bp 1282 if (DoesLeak) { 1283 DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) { 1284 assert(Operand.isReg()); 1285 MCPhysReg Reg = Operand.getReg(); 1286 return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg]; 1287 }); 1288 } 1289 return DoesLeak; 1290 } 1291 1292 bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { 1293 unsigned ImmOpNo = -1U; 1294 int MemOpNo = getMemoryOperandNo(Inst); 1295 if (MemOpNo != -1) 1296 ImmOpNo = MemOpNo + X86::AddrDisp; 1297 else 1298 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1299 ++Index) 1300 if (Inst.getOperand(Index).isImm()) 1301 ImmOpNo = Index; 1302 if (ImmOpNo == -1U) 1303 return false; 1304 1305 MCOperand &Operand = Inst.getOperand(ImmOpNo); 1306 Amt += Operand.getImm(); 1307 Operand.setImm(Amt); 1308 // Check for the need for relaxation 1309 if (int64_t(Amt) == int64_t(int8_t(Amt))) 1310 return true; 1311 1312 // Relax instruction 1313 switch (Inst.getOpcode()) { 1314 case X86::SUB64ri8: 1315 Inst.setOpcode(X86::SUB64ri32); 1316 break; 1317 case X86::ADD64ri8: 1318 Inst.setOpcode(X86::ADD64ri32); 1319 break; 1320 default: 1321 // No need for relaxation 1322 break; 1323 } 1324 return true; 1325 } 1326 1327 /// TODO: this implementation currently works for the most common opcodes that 1328 /// load from memory. It can be extended to work with memory store opcodes as 1329 /// well as more memory load opcodes. 1330 bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, 1331 uint64_t Offset) const override { 1332 enum CheckSignExt : uint8_t { 1333 NOCHECK = 0, 1334 CHECK8, 1335 CHECK32, 1336 }; 1337 1338 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 1339 struct InstInfo { 1340 // Size in bytes that Inst loads from memory. 1341 uint8_t DataSize; 1342 1343 // True when the target operand has to be duplicated because the opcode 1344 // expects a LHS operand. 1345 bool HasLHS; 1346 1347 // List of checks and corresponding opcodes to be used. We try to use the 1348 // smallest possible immediate value when various sizes are available, 1349 // hence we may need to check whether a larger constant fits in a smaller 1350 // immediate. 1351 CheckList Checks; 1352 }; 1353 1354 InstInfo I; 1355 1356 switch (Inst.getOpcode()) { 1357 default: { 1358 switch (getPopSize(Inst)) { 1359 case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1360 case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1361 case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1362 {NOCHECK, X86::MOV64rm}}}; break; 1363 default: return false; 1364 } 1365 break; 1366 } 1367 1368 // MOV 1369 case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 1370 case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1371 case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1372 case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1373 {NOCHECK, X86::MOV64rm}}}; break; 1374 1375 // MOVZX 1376 case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 1377 case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 1378 case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 1379 1380 // CMP 1381 case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 1382 case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, 1383 {NOCHECK, X86::CMP16ri}}}; break; 1384 case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, 1385 {NOCHECK, X86::CMP32ri}}}; break; 1386 case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, 1387 {CHECK32, X86::CMP64ri32}, 1388 {NOCHECK, X86::CMP64rm}}}; break; 1389 1390 // TEST 1391 case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 1392 case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 1393 case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 1394 case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 1395 {NOCHECK, X86::TEST64mr}}}; break; 1396 1397 // ADD 1398 case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 1399 case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, 1400 {NOCHECK, X86::ADD16ri}}}; break; 1401 case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, 1402 {NOCHECK, X86::ADD32ri}}}; break; 1403 case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, 1404 {CHECK32, X86::ADD64ri32}, 1405 {NOCHECK, X86::ADD64rm}}}; break; 1406 1407 // SUB 1408 case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 1409 case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, 1410 {NOCHECK, X86::SUB16ri}}}; break; 1411 case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, 1412 {NOCHECK, X86::SUB32ri}}}; break; 1413 case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, 1414 {CHECK32, X86::SUB64ri32}, 1415 {NOCHECK, X86::SUB64rm}}}; break; 1416 1417 // AND 1418 case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 1419 case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, 1420 {NOCHECK, X86::AND16ri}}}; break; 1421 case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, 1422 {NOCHECK, X86::AND32ri}}}; break; 1423 case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, 1424 {CHECK32, X86::AND64ri32}, 1425 {NOCHECK, X86::AND64rm}}}; break; 1426 1427 // OR 1428 case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 1429 case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, 1430 {NOCHECK, X86::OR16ri}}}; break; 1431 case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, 1432 {NOCHECK, X86::OR32ri}}}; break; 1433 case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, 1434 {CHECK32, X86::OR64ri32}, 1435 {NOCHECK, X86::OR64rm}}}; break; 1436 1437 // XOR 1438 case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 1439 case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, 1440 {NOCHECK, X86::XOR16ri}}}; break; 1441 case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, 1442 {NOCHECK, X86::XOR32ri}}}; break; 1443 case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, 1444 {CHECK32, X86::XOR64ri32}, 1445 {NOCHECK, X86::XOR64rm}}}; break; 1446 } 1447 1448 // Compute the immediate value. 1449 assert(Offset + I.DataSize <= ConstantData.size() && 1450 "invalid offset for given constant data"); 1451 int64_t ImmVal = 1452 DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize); 1453 1454 // Compute the new opcode. 1455 unsigned NewOpcode = 0; 1456 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 1457 NewOpcode = Check.second; 1458 if (Check.first == NOCHECK) 1459 break; 1460 if (Check.first == CHECK8 && isInt<8>(ImmVal)) 1461 break; 1462 if (Check.first == CHECK32 && isInt<32>(ImmVal)) 1463 break; 1464 } 1465 if (NewOpcode == Inst.getOpcode()) 1466 return false; 1467 1468 // Modify the instruction. 1469 MCOperand ImmOp = MCOperand::createImm(ImmVal); 1470 uint32_t TargetOpNum = 0; 1471 // Test instruction does not follow the regular pattern of putting the 1472 // memory reference of a load (5 MCOperands) last in the list of operands. 1473 // Since it is not modifying the register operand, it is not treated as 1474 // a destination operand and it is not the first operand as it is in the 1475 // other instructions we treat here. 1476 if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri || 1477 NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32) 1478 TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; 1479 1480 MCOperand TargetOp = Inst.getOperand(TargetOpNum); 1481 Inst.clear(); 1482 Inst.setOpcode(NewOpcode); 1483 Inst.addOperand(TargetOp); 1484 if (I.HasLHS) 1485 Inst.addOperand(TargetOp); 1486 Inst.addOperand(ImmOp); 1487 1488 return true; 1489 } 1490 1491 /// TODO: this implementation currently works for the most common opcodes that 1492 /// load from memory. It can be extended to work with memory store opcodes as 1493 /// well as more memory load opcodes. 1494 bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { 1495 unsigned NewOpcode; 1496 1497 switch (Inst.getOpcode()) { 1498 default: { 1499 switch (getPopSize(Inst)) { 1500 case 2: NewOpcode = X86::MOV16rr; break; 1501 case 4: NewOpcode = X86::MOV32rr; break; 1502 case 8: NewOpcode = X86::MOV64rr; break; 1503 default: return false; 1504 } 1505 break; 1506 } 1507 1508 // MOV 1509 case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; 1510 case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; 1511 case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; 1512 case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; 1513 } 1514 1515 // Modify the instruction. 1516 MCOperand RegOp = MCOperand::createReg(RegNum); 1517 MCOperand TargetOp = Inst.getOperand(0); 1518 Inst.clear(); 1519 Inst.setOpcode(NewOpcode); 1520 Inst.addOperand(TargetOp); 1521 Inst.addOperand(RegOp); 1522 1523 return true; 1524 } 1525 1526 bool isRedundantMove(const MCInst &Inst) const override { 1527 switch (Inst.getOpcode()) { 1528 default: 1529 return false; 1530 1531 // MOV 1532 case X86::MOV8rr: 1533 case X86::MOV16rr: 1534 case X86::MOV32rr: 1535 case X86::MOV64rr: 1536 break; 1537 } 1538 1539 assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); 1540 return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); 1541 } 1542 1543 bool requiresAlignedAddress(const MCInst &Inst) const override { 1544 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 1545 for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { 1546 const MCOperandInfo &Op = Desc.operands()[I]; 1547 if (Op.OperandType != MCOI::OPERAND_REGISTER) 1548 continue; 1549 if (Op.RegClass == X86::VR128RegClassID) 1550 return true; 1551 } 1552 return false; 1553 } 1554 1555 bool convertJmpToTailCall(MCInst &Inst) override { 1556 if (isTailCall(Inst)) 1557 return false; 1558 1559 int NewOpcode; 1560 switch (Inst.getOpcode()) { 1561 default: 1562 return false; 1563 case X86::JMP_1: 1564 case X86::JMP_2: 1565 case X86::JMP_4: 1566 NewOpcode = X86::JMP_4; 1567 break; 1568 case X86::JMP16m: 1569 case X86::JMP32m: 1570 case X86::JMP64m: 1571 NewOpcode = X86::JMP32m; 1572 break; 1573 case X86::JMP16r: 1574 case X86::JMP32r: 1575 case X86::JMP64r: 1576 NewOpcode = X86::JMP32r; 1577 break; 1578 } 1579 1580 Inst.setOpcode(NewOpcode); 1581 setTailCall(Inst); 1582 return true; 1583 } 1584 1585 bool convertTailCallToJmp(MCInst &Inst) override { 1586 int NewOpcode; 1587 switch (Inst.getOpcode()) { 1588 default: 1589 return false; 1590 case X86::JMP_4: 1591 NewOpcode = X86::JMP_1; 1592 break; 1593 case X86::JMP32m: 1594 NewOpcode = X86::JMP64m; 1595 break; 1596 case X86::JMP32r: 1597 NewOpcode = X86::JMP64r; 1598 break; 1599 } 1600 1601 Inst.setOpcode(NewOpcode); 1602 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1603 clearOffset(Inst); 1604 return true; 1605 } 1606 1607 bool convertTailCallToCall(MCInst &Inst) override { 1608 int NewOpcode; 1609 switch (Inst.getOpcode()) { 1610 default: 1611 return false; 1612 case X86::JMP_4: 1613 NewOpcode = X86::CALL64pcrel32; 1614 break; 1615 case X86::JMP32m: 1616 NewOpcode = X86::CALL64m; 1617 break; 1618 case X86::JMP32r: 1619 NewOpcode = X86::CALL64r; 1620 break; 1621 } 1622 1623 Inst.setOpcode(NewOpcode); 1624 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1625 return true; 1626 } 1627 1628 bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation, 1629 MCContext *Ctx) override { 1630 assert((Inst.getOpcode() == X86::CALL64pcrel32 || 1631 (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) && 1632 "64-bit direct (tail) call instruction expected"); 1633 const auto NewOpcode = 1634 (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m; 1635 Inst.setOpcode(NewOpcode); 1636 1637 // Replace the first operand and preserve auxiliary operands of 1638 // the instruction. 1639 Inst.erase(Inst.begin()); 1640 Inst.insert(Inst.begin(), 1641 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 1642 Inst.insert(Inst.begin(), 1643 MCOperand::createExpr( // Displacement 1644 MCSymbolRefExpr::create(TargetLocation, 1645 MCSymbolRefExpr::VK_None, *Ctx))); 1646 Inst.insert(Inst.begin(), 1647 MCOperand::createReg(X86::NoRegister)); // IndexReg 1648 Inst.insert(Inst.begin(), 1649 MCOperand::createImm(1)); // ScaleAmt 1650 Inst.insert(Inst.begin(), 1651 MCOperand::createReg(X86::RIP)); // BaseReg 1652 1653 return true; 1654 } 1655 1656 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1657 bool IsTailCall = isTailCall(Inst); 1658 if (IsTailCall) 1659 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1660 if (Inst.getOpcode() == X86::CALL64m || 1661 (Inst.getOpcode() == X86::JMP32m && IsTailCall)) { 1662 Inst.setOpcode(X86::MOV64rm); 1663 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1664 return; 1665 } 1666 if (Inst.getOpcode() == X86::CALL64r || 1667 (Inst.getOpcode() == X86::JMP32r && IsTailCall)) { 1668 Inst.setOpcode(X86::MOV64rr); 1669 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1670 return; 1671 } 1672 LLVM_DEBUG(Inst.dump()); 1673 llvm_unreachable("not implemented"); 1674 } 1675 1676 bool shortenInstruction(MCInst &Inst, 1677 const MCSubtargetInfo &STI) const override { 1678 unsigned OldOpcode = Inst.getOpcode(); 1679 unsigned NewOpcode = OldOpcode; 1680 1681 int MemOpNo = getMemoryOperandNo(Inst); 1682 1683 // Check and remove redundant Address-Size override prefix. 1684 if (opts::X86StripRedundantAddressSize) { 1685 uint64_t TSFlags = Info->get(OldOpcode).TSFlags; 1686 unsigned Flags = Inst.getFlags(); 1687 1688 if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) && 1689 Flags & X86::IP_HAS_AD_SIZE) 1690 Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE); 1691 } 1692 1693 // Check and remove EIZ/RIZ. These cases represent ambiguous cases where 1694 // SIB byte is present, but no index is used and modrm alone should have 1695 // been enough. Converting to NoRegister effectively removes the SIB byte. 1696 if (MemOpNo >= 0) { 1697 MCOperand &IndexOp = 1698 Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg); 1699 if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ) 1700 IndexOp = MCOperand::createReg(X86::NoRegister); 1701 } 1702 1703 if (isBranch(Inst)) { 1704 NewOpcode = getShortBranchOpcode(OldOpcode); 1705 } else if (OldOpcode == X86::MOV64ri) { 1706 if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1707 const int64_t Imm = 1708 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1709 if (int64_t(Imm) == int64_t(int32_t(Imm))) 1710 NewOpcode = X86::MOV64ri32; 1711 } 1712 } else { 1713 // If it's arithmetic instruction check if signed operand fits in 1 byte. 1714 const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(OldOpcode); 1715 if (ShortOpcode != OldOpcode && 1716 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1717 int64_t Imm = 1718 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1719 if (int64_t(Imm) == int64_t(int8_t(Imm))) 1720 NewOpcode = ShortOpcode; 1721 } 1722 } 1723 1724 if (NewOpcode == OldOpcode) 1725 return false; 1726 1727 Inst.setOpcode(NewOpcode); 1728 return true; 1729 } 1730 1731 bool 1732 convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp, 1733 bool AllowBasePtrStackMemOp) const override { 1734 // - Register-register moves are OK 1735 // - Stores are filtered out by opcode (no store CMOV) 1736 // - Non-stack loads are prohibited (generally unsafe) 1737 // - Stack loads are OK if AllowStackMemOp is true 1738 // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true 1739 if (isLoad(Inst)) { 1740 // If stack memory operands are not allowed, no loads are allowed 1741 if (!AllowStackMemOp) 1742 return false; 1743 1744 // If stack memory operands are allowed, check if it's a load from stack 1745 bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed; 1746 MCPhysReg Reg; 1747 int32_t SrcImm; 1748 uint16_t StackPtrReg; 1749 int64_t StackOffset; 1750 uint8_t Size; 1751 bool IsStackAccess = 1752 isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, 1753 StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); 1754 // Prohibit non-stack-based loads 1755 if (!IsStackAccess) 1756 return false; 1757 // If stack memory operands are allowed, check if it's RBP-based 1758 if (!AllowBasePtrStackMemOp && 1759 RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg)) 1760 return false; 1761 } 1762 1763 unsigned NewOpcode = 0; 1764 switch (Inst.getOpcode()) { 1765 case X86::MOV16rr: 1766 NewOpcode = X86::CMOV16rr; 1767 break; 1768 case X86::MOV16rm: 1769 NewOpcode = X86::CMOV16rm; 1770 break; 1771 case X86::MOV32rr: 1772 NewOpcode = X86::CMOV32rr; 1773 break; 1774 case X86::MOV32rm: 1775 NewOpcode = X86::CMOV32rm; 1776 break; 1777 case X86::MOV64rr: 1778 NewOpcode = X86::CMOV64rr; 1779 break; 1780 case X86::MOV64rm: 1781 NewOpcode = X86::CMOV64rm; 1782 break; 1783 default: 1784 return false; 1785 } 1786 Inst.setOpcode(NewOpcode); 1787 // Insert CC at the end of prime operands, before annotations 1788 Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst), 1789 MCOperand::createImm(CC)); 1790 // CMOV is a 3-operand MCInst, so duplicate the destination as src1 1791 Inst.insert(Inst.begin(), Inst.getOperand(0)); 1792 return true; 1793 } 1794 1795 bool lowerTailCall(MCInst &Inst) override { 1796 if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) { 1797 Inst.setOpcode(X86::JMP_1); 1798 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1799 return true; 1800 } 1801 return false; 1802 } 1803 1804 const MCSymbol *getTargetSymbol(const MCInst &Inst, 1805 unsigned OpNum = 0) const override { 1806 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 1807 return nullptr; 1808 1809 const MCOperand &Op = Inst.getOperand(OpNum); 1810 if (!Op.isExpr()) 1811 return nullptr; 1812 1813 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 1814 if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) 1815 return nullptr; 1816 1817 return &SymExpr->getSymbol(); 1818 } 1819 1820 // This is the same as the base class, but since we are overriding one of 1821 // getTargetSymbol's signatures above, we need to override all of them. 1822 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 1823 return &cast<const MCSymbolRefExpr>(Expr)->getSymbol(); 1824 } 1825 1826 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1827 const MCSymbol *&TBB, const MCSymbol *&FBB, 1828 MCInst *&CondBranch, 1829 MCInst *&UncondBranch) const override { 1830 auto I = End; 1831 1832 // Bottom-up analysis 1833 while (I != Begin) { 1834 --I; 1835 1836 // Ignore nops and CFIs 1837 if (isPseudo(*I)) 1838 continue; 1839 1840 // Stop when we find the first non-terminator 1841 if (!isTerminator(*I)) 1842 break; 1843 1844 if (!isBranch(*I)) 1845 break; 1846 1847 // Handle unconditional branches. 1848 if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 || 1849 I->getOpcode() == X86::JMP_4) && 1850 !isTailCall(*I)) { 1851 // If any code was seen after this unconditional branch, we've seen 1852 // unreachable code. Ignore them. 1853 CondBranch = nullptr; 1854 UncondBranch = &*I; 1855 const MCSymbol *Sym = getTargetSymbol(*I); 1856 assert(Sym != nullptr && 1857 "Couldn't extract BB symbol from jump operand"); 1858 TBB = Sym; 1859 continue; 1860 } 1861 1862 // Handle conditional branches and ignore indirect branches 1863 if (!isUnsupportedBranch(I->getOpcode()) && 1864 getCondCode(*I) == X86::COND_INVALID) { 1865 // Indirect branch 1866 return false; 1867 } 1868 1869 if (CondBranch == nullptr) { 1870 const MCSymbol *TargetBB = getTargetSymbol(*I); 1871 if (TargetBB == nullptr) { 1872 // Unrecognized branch target 1873 return false; 1874 } 1875 FBB = TBB; 1876 TBB = TargetBB; 1877 CondBranch = &*I; 1878 continue; 1879 } 1880 1881 llvm_unreachable("multiple conditional branches in one BB"); 1882 } 1883 return true; 1884 } 1885 1886 template <typename Itr> 1887 std::pair<IndirectBranchType, MCInst *> 1888 analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const { 1889 // Analyze PIC-style jump table code template: 1890 // 1891 // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr 1892 // mov ({%r1|%r2}, %index, 4), {%r2|%r1} 1893 // add %r2, %r1 1894 // jmp *%r1 1895 // 1896 // (with any irrelevant instructions in-between) 1897 // 1898 // When we call this helper we've already determined %r1 and %r2, and 1899 // reverse instruction iterator \p II is pointing to the ADD instruction. 1900 // 1901 // PIC jump table looks like following: 1902 // 1903 // JT: ---------- 1904 // E1:| L1 - JT | 1905 // |----------| 1906 // E2:| L2 - JT | 1907 // |----------| 1908 // | | 1909 // ...... 1910 // En:| Ln - JT | 1911 // ---------- 1912 // 1913 // Where L1, L2, ..., Ln represent labels in the function. 1914 // 1915 // The actual relocations in the table will be of the form: 1916 // 1917 // Ln - JT 1918 // = (Ln - En) + (En - JT) 1919 // = R_X86_64_PC32(Ln) + En - JT 1920 // = R_X86_64_PC32(Ln + offsetof(En)) 1921 // 1922 LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n"); 1923 MCInst *MemLocInstr = nullptr; 1924 const MCInst *MovInstr = nullptr; 1925 while (++II != IE) { 1926 MCInst &Instr = *II; 1927 const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode()); 1928 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && 1929 !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { 1930 // Ignore instructions that don't affect R1, R2 registers. 1931 continue; 1932 } 1933 if (!MovInstr) { 1934 // Expect to see MOV instruction. 1935 if (!isMOVSX64rm32(Instr)) { 1936 LLVM_DEBUG(dbgs() << "MOV instruction expected.\n"); 1937 break; 1938 } 1939 1940 // Check if it's setting %r1 or %r2. In canonical form it sets %r2. 1941 // If it sets %r1 - rename the registers so we have to only check 1942 // a single form. 1943 unsigned MovDestReg = Instr.getOperand(0).getReg(); 1944 if (MovDestReg != R2) 1945 std::swap(R1, R2); 1946 if (MovDestReg != R2) { 1947 LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); 1948 break; 1949 } 1950 1951 // Verify operands for MOV. 1952 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1953 if (!MO) 1954 break; 1955 if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 || 1956 MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 || 1957 MO->SegRegNum != X86::NoRegister) 1958 break; 1959 MovInstr = &Instr; 1960 } else { 1961 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) 1962 continue; 1963 if (!isLEA64r(Instr)) { 1964 LLVM_DEBUG(dbgs() << "LEA instruction expected\n"); 1965 break; 1966 } 1967 if (Instr.getOperand(0).getReg() != R1) { 1968 LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); 1969 break; 1970 } 1971 1972 // Verify operands for LEA. 1973 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1974 if (!MO) 1975 break; 1976 if (MO->BaseRegNum != RegInfo->getProgramCounter() || 1977 MO->IndexRegNum != X86::NoRegister || 1978 MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr) 1979 break; 1980 MemLocInstr = &Instr; 1981 break; 1982 } 1983 } 1984 1985 if (!MemLocInstr) 1986 return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); 1987 1988 LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n"); 1989 return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, 1990 MemLocInstr); 1991 } 1992 1993 IndirectBranchType analyzeIndirectBranch( 1994 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End, 1995 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 1996 unsigned &IndexRegNumOut, int64_t &DispValueOut, 1997 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override { 1998 // Try to find a (base) memory location from where the address for 1999 // the indirect branch is loaded. For X86-64 the memory will be specified 2000 // in the following format: 2001 // 2002 // {%rip}/{%basereg} + Imm + IndexReg * Scale 2003 // 2004 // We are interested in the cases where Scale == sizeof(uintptr_t) and 2005 // the contents of the memory are presumably an array of pointers to code. 2006 // 2007 // Normal jump table: 2008 // 2009 // jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr 2010 // 2011 // or 2012 // 2013 // mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr 2014 // ... 2015 // jmp %r1 2016 // 2017 // We handle PIC-style jump tables separately. 2018 // 2019 MemLocInstrOut = nullptr; 2020 BaseRegNumOut = X86::NoRegister; 2021 IndexRegNumOut = X86::NoRegister; 2022 DispValueOut = 0; 2023 DispExprOut = nullptr; 2024 2025 std::reverse_iterator<InstructionIterator> II(End); 2026 std::reverse_iterator<InstructionIterator> IE(Begin); 2027 2028 IndirectBranchType Type = IndirectBranchType::UNKNOWN; 2029 2030 // An instruction referencing memory used by jump instruction (directly or 2031 // via register). This location could be an array of function pointers 2032 // in case of indirect tail call, or a jump table. 2033 MCInst *MemLocInstr = nullptr; 2034 2035 if (MCPlus::getNumPrimeOperands(Instruction) == 1) { 2036 // If the indirect jump is on register - try to detect if the 2037 // register value is loaded from a memory location. 2038 assert(Instruction.getOperand(0).isReg() && "register operand expected"); 2039 const unsigned R1 = Instruction.getOperand(0).getReg(); 2040 // Check if one of the previous instructions defines the jump-on register. 2041 for (auto PrevII = II; PrevII != IE; ++PrevII) { 2042 MCInst &PrevInstr = *PrevII; 2043 const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); 2044 2045 if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) 2046 continue; 2047 2048 if (isMoveMem2Reg(PrevInstr)) { 2049 MemLocInstr = &PrevInstr; 2050 break; 2051 } 2052 if (isADD64rr(PrevInstr)) { 2053 unsigned R2 = PrevInstr.getOperand(2).getReg(); 2054 if (R1 == R2) 2055 return IndirectBranchType::UNKNOWN; 2056 std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); 2057 break; 2058 } 2059 return IndirectBranchType::UNKNOWN; 2060 } 2061 if (!MemLocInstr) { 2062 // No definition seen for the register in this function so far. Could be 2063 // an input parameter - which means it is an external code reference. 2064 // It also could be that the definition happens to be in the code that 2065 // we haven't processed yet. Since we have to be conservative, return 2066 // as UNKNOWN case. 2067 return IndirectBranchType::UNKNOWN; 2068 } 2069 } else { 2070 MemLocInstr = &Instruction; 2071 } 2072 2073 const MCRegister RIPRegister = RegInfo->getProgramCounter(); 2074 2075 // Analyze the memory location. 2076 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr); 2077 if (!MO) 2078 return IndirectBranchType::UNKNOWN; 2079 2080 BaseRegNumOut = MO->BaseRegNum; 2081 IndexRegNumOut = MO->IndexRegNum; 2082 DispValueOut = MO->DispImm; 2083 DispExprOut = MO->DispExpr; 2084 2085 if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) || 2086 MO->SegRegNum != X86::NoRegister) 2087 return IndirectBranchType::UNKNOWN; 2088 2089 if (MemLocInstr == &Instruction && 2090 (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) { 2091 MemLocInstrOut = MemLocInstr; 2092 return IndirectBranchType::POSSIBLE_FIXED_BRANCH; 2093 } 2094 2095 if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2096 (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)) 2097 return IndirectBranchType::UNKNOWN; 2098 2099 if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2100 MO->ScaleImm != PtrSize) 2101 return IndirectBranchType::UNKNOWN; 2102 2103 MemLocInstrOut = MemLocInstr; 2104 2105 return Type; 2106 } 2107 2108 /// Analyze a callsite to see if it could be a virtual method call. This only 2109 /// checks to see if the overall pattern is satisfied, it does not guarantee 2110 /// that the callsite is a true virtual method call. 2111 /// The format of virtual method calls that are recognized is one of the 2112 /// following: 2113 /// 2114 /// Form 1: (found in debug code) 2115 /// add METHOD_OFFSET, %VtableReg 2116 /// mov (%VtableReg), %MethodReg 2117 /// ... 2118 /// call or jmp *%MethodReg 2119 /// 2120 /// Form 2: 2121 /// mov METHOD_OFFSET(%VtableReg), %MethodReg 2122 /// ... 2123 /// call or jmp *%MethodReg 2124 /// 2125 /// Form 3: 2126 /// ... 2127 /// call or jmp *METHOD_OFFSET(%VtableReg) 2128 /// 2129 bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, 2130 InstructionIterator ForwardEnd, 2131 std::vector<MCInst *> &MethodFetchInsns, 2132 unsigned &VtableRegNum, unsigned &MethodRegNum, 2133 uint64_t &MethodOffset) const override { 2134 VtableRegNum = X86::NoRegister; 2135 MethodRegNum = X86::NoRegister; 2136 MethodOffset = 0; 2137 2138 std::reverse_iterator<InstructionIterator> Itr(ForwardEnd); 2139 std::reverse_iterator<InstructionIterator> End(ForwardBegin); 2140 2141 MCInst &CallInst = *Itr++; 2142 assert(isIndirectBranch(CallInst) || isCall(CallInst)); 2143 2144 // The call can just be jmp offset(reg) 2145 if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) { 2146 if (!MO->DispExpr && MO->BaseRegNum != X86::RIP && 2147 MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) { 2148 MethodRegNum = MO->BaseRegNum; 2149 if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister && 2150 MO->SegRegNum == X86::NoRegister) { 2151 VtableRegNum = MethodRegNum; 2152 MethodOffset = MO->DispImm; 2153 MethodFetchInsns.push_back(&CallInst); 2154 return true; 2155 } 2156 } 2157 return false; 2158 } 2159 if (CallInst.getOperand(0).isReg()) 2160 MethodRegNum = CallInst.getOperand(0).getReg(); 2161 else 2162 return false; 2163 2164 if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { 2165 VtableRegNum = X86::NoRegister; 2166 MethodRegNum = X86::NoRegister; 2167 return false; 2168 } 2169 2170 // find load from vtable, this may or may not include the method offset 2171 while (Itr != End) { 2172 MCInst &CurInst = *Itr++; 2173 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2174 if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { 2175 if (!isLoad(CurInst)) 2176 return false; 2177 if (std::optional<X86MemOperand> MO = 2178 evaluateX86MemoryOperand(CurInst)) { 2179 if (!MO->DispExpr && MO->ScaleImm == 1 && 2180 MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP && 2181 MO->BaseRegNum != X86::NoRegister && 2182 MO->IndexRegNum == X86::NoRegister && 2183 MO->SegRegNum == X86::NoRegister && MO->BaseRegNum != X86::RIP) { 2184 VtableRegNum = MO->BaseRegNum; 2185 MethodOffset = MO->DispImm; 2186 MethodFetchInsns.push_back(&CurInst); 2187 if (MethodOffset != 0) 2188 return true; 2189 break; 2190 } 2191 } 2192 return false; 2193 } 2194 } 2195 2196 if (!VtableRegNum) 2197 return false; 2198 2199 // look for any adds affecting the method register. 2200 while (Itr != End) { 2201 MCInst &CurInst = *Itr++; 2202 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2203 if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { 2204 if (isADDri(CurInst)) { 2205 assert(!MethodOffset); 2206 MethodOffset = CurInst.getOperand(2).getImm(); 2207 MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); 2208 break; 2209 } 2210 } 2211 } 2212 2213 return true; 2214 } 2215 2216 bool createStackPointerIncrement(MCInst &Inst, int Size, 2217 bool NoFlagsClobber) const override { 2218 if (NoFlagsClobber) { 2219 Inst.setOpcode(X86::LEA64r); 2220 Inst.clear(); 2221 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2222 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2223 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2224 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2225 Inst.addOperand(MCOperand::createImm(-Size)); // Displacement 2226 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2227 return true; 2228 } 2229 Inst.setOpcode(X86::SUB64ri8); 2230 Inst.clear(); 2231 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2232 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2233 Inst.addOperand(MCOperand::createImm(Size)); 2234 return true; 2235 } 2236 2237 bool createStackPointerDecrement(MCInst &Inst, int Size, 2238 bool NoFlagsClobber) const override { 2239 if (NoFlagsClobber) { 2240 Inst.setOpcode(X86::LEA64r); 2241 Inst.clear(); 2242 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2243 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2244 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2245 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2246 Inst.addOperand(MCOperand::createImm(Size)); // Displacement 2247 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2248 return true; 2249 } 2250 Inst.setOpcode(X86::ADD64ri8); 2251 Inst.clear(); 2252 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2253 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2254 Inst.addOperand(MCOperand::createImm(Size)); 2255 return true; 2256 } 2257 2258 bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, 2259 const MCPhysReg &SrcReg, int Size) const override { 2260 unsigned NewOpcode; 2261 switch (Size) { 2262 default: 2263 return false; 2264 case 2: NewOpcode = X86::MOV16mr; break; 2265 case 4: NewOpcode = X86::MOV32mr; break; 2266 case 8: NewOpcode = X86::MOV64mr; break; 2267 } 2268 Inst.setOpcode(NewOpcode); 2269 Inst.clear(); 2270 Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg 2271 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2272 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2273 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2274 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2275 Inst.addOperand(MCOperand::createReg(SrcReg)); 2276 return true; 2277 } 2278 2279 bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, 2280 int Offset, const MCPhysReg &DstReg, 2281 int Size) const override { 2282 return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, 2283 Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister, 2284 DstReg, Size); 2285 } 2286 2287 bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale, 2288 const MCPhysReg &IndexReg, int64_t Offset, 2289 const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg, 2290 const MCPhysReg &DstReg, int Size) const override { 2291 unsigned NewOpcode; 2292 switch (Size) { 2293 default: 2294 return false; 2295 case 2: NewOpcode = X86::MOV16rm; break; 2296 case 4: NewOpcode = X86::MOV32rm; break; 2297 case 8: NewOpcode = X86::MOV64rm; break; 2298 } 2299 Inst.setOpcode(NewOpcode); 2300 Inst.clear(); 2301 Inst.addOperand(MCOperand::createReg(DstReg)); 2302 Inst.addOperand(MCOperand::createReg(BaseReg)); 2303 Inst.addOperand(MCOperand::createImm(Scale)); 2304 Inst.addOperand(MCOperand::createReg(IndexReg)); 2305 if (OffsetExpr) 2306 Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement 2307 else 2308 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2309 Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg 2310 return true; 2311 } 2312 2313 void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest, 2314 uint32_t Imm) const override { 2315 Inst.setOpcode(X86::MOV64ri32); 2316 Inst.clear(); 2317 Inst.addOperand(MCOperand::createReg(Dest)); 2318 Inst.addOperand(MCOperand::createImm(Imm)); 2319 } 2320 2321 bool createIncMemory(MCInst &Inst, const MCSymbol *Target, 2322 MCContext *Ctx) const override { 2323 2324 Inst.setOpcode(X86::LOCK_INC64m); 2325 Inst.clear(); 2326 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 2327 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2328 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2329 2330 Inst.addOperand(MCOperand::createExpr( 2331 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, 2332 *Ctx))); // Displacement 2333 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2334 return true; 2335 } 2336 2337 bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts, 2338 const MCOperand &BaseReg, const MCOperand &Scale, 2339 const MCOperand &IndexReg, const MCOperand &Offset, 2340 const MCOperand &TmpReg) const override { 2341 // The code fragment we emit here is: 2342 // 2343 // mov32 (%base, %index, scale), %tmpreg 2344 // ijmp *(%tmpreg) 2345 // 2346 MCInst IJmp; 2347 IJmp.setOpcode(X86::JMP64r); 2348 IJmp.addOperand(TmpReg); 2349 2350 MCInst Load; 2351 Load.setOpcode(X86::MOV32rm); 2352 Load.addOperand(TmpReg); 2353 Load.addOperand(BaseReg); 2354 Load.addOperand(Scale); 2355 Load.addOperand(IndexReg); 2356 Load.addOperand(Offset); 2357 Load.addOperand(MCOperand::createReg(X86::NoRegister)); 2358 2359 Insts.push_back(Load); 2360 Insts.push_back(IJmp); 2361 return true; 2362 } 2363 2364 bool createNoop(MCInst &Inst) const override { 2365 Inst.setOpcode(X86::NOOP); 2366 return true; 2367 } 2368 2369 bool createReturn(MCInst &Inst) const override { 2370 Inst.setOpcode(X86::RET64); 2371 return true; 2372 } 2373 2374 InstructionListType createInlineMemcpy(bool ReturnEnd) const override { 2375 InstructionListType Code; 2376 if (ReturnEnd) 2377 Code.emplace_back(MCInstBuilder(X86::LEA64r) 2378 .addReg(X86::RAX) 2379 .addReg(X86::RDI) 2380 .addImm(1) 2381 .addReg(X86::RDX) 2382 .addImm(0) 2383 .addReg(X86::NoRegister)); 2384 else 2385 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2386 .addReg(X86::RAX) 2387 .addReg(X86::RDI)); 2388 2389 Code.emplace_back(MCInstBuilder(X86::MOV32rr) 2390 .addReg(X86::ECX) 2391 .addReg(X86::EDX)); 2392 Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64)); 2393 2394 return Code; 2395 } 2396 2397 InstructionListType createOneByteMemcpy() const override { 2398 InstructionListType Code; 2399 Code.emplace_back(MCInstBuilder(X86::MOV8rm) 2400 .addReg(X86::CL) 2401 .addReg(X86::RSI) 2402 .addImm(0) 2403 .addReg(X86::NoRegister) 2404 .addImm(0) 2405 .addReg(X86::NoRegister)); 2406 Code.emplace_back(MCInstBuilder(X86::MOV8mr) 2407 .addReg(X86::RDI) 2408 .addImm(0) 2409 .addReg(X86::NoRegister) 2410 .addImm(0) 2411 .addReg(X86::NoRegister) 2412 .addReg(X86::CL)); 2413 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2414 .addReg(X86::RAX) 2415 .addReg(X86::RDI)); 2416 return Code; 2417 } 2418 2419 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 2420 const MCSymbol *Target, 2421 MCContext *Ctx) const override { 2422 InstructionListType Code; 2423 Code.emplace_back(MCInstBuilder(X86::CMP64ri8) 2424 .addReg(RegNo) 2425 .addImm(Imm)); 2426 Code.emplace_back(MCInstBuilder(X86::JCC_1) 2427 .addExpr(MCSymbolRefExpr::create( 2428 Target, MCSymbolRefExpr::VK_None, *Ctx)) 2429 .addImm(X86::COND_E)); 2430 return Code; 2431 } 2432 2433 std::optional<Relocation> 2434 createRelocation(const MCFixup &Fixup, 2435 const MCAsmBackend &MAB) const override { 2436 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2437 2438 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2439 const uint64_t RelOffset = Fixup.getOffset(); 2440 2441 uint64_t RelType; 2442 if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2443 switch (FKI.TargetSize) { 2444 default: 2445 return std::nullopt; 2446 case 8: RelType = ELF::R_X86_64_PC8; break; 2447 case 16: RelType = ELF::R_X86_64_PC16; break; 2448 case 32: RelType = ELF::R_X86_64_PC32; break; 2449 case 64: RelType = ELF::R_X86_64_PC64; break; 2450 } 2451 } else { 2452 switch (FKI.TargetSize) { 2453 default: 2454 return std::nullopt; 2455 case 8: RelType = ELF::R_X86_64_8; break; 2456 case 16: RelType = ELF::R_X86_64_16; break; 2457 case 32: RelType = ELF::R_X86_64_32; break; 2458 case 64: RelType = ELF::R_X86_64_64; break; 2459 } 2460 } 2461 2462 // Extract a symbol and an addend out of the fixup value expression. 2463 // 2464 // Only the following limited expression types are supported: 2465 // Symbol + Addend 2466 // Symbol + Constant + Addend 2467 // Const + Addend 2468 // Symbol 2469 uint64_t Addend = 0; 2470 MCSymbol *Symbol = nullptr; 2471 const MCExpr *ValueExpr = Fixup.getValue(); 2472 if (ValueExpr->getKind() == MCExpr::Binary) { 2473 const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr); 2474 assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add && 2475 "unexpected binary expression"); 2476 const MCExpr *LHS = BinaryExpr->getLHS(); 2477 if (LHS->getKind() == MCExpr::Constant) { 2478 Addend = cast<MCConstantExpr>(LHS)->getValue(); 2479 } else if (LHS->getKind() == MCExpr::Binary) { 2480 const auto *LHSBinaryExpr = cast<MCBinaryExpr>(LHS); 2481 assert(LHSBinaryExpr->getOpcode() == MCBinaryExpr::Add && 2482 "unexpected binary expression"); 2483 const MCExpr *LLHS = LHSBinaryExpr->getLHS(); 2484 assert(LLHS->getKind() == MCExpr::SymbolRef && "unexpected LLHS"); 2485 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LLHS)); 2486 const MCExpr *RLHS = LHSBinaryExpr->getRHS(); 2487 assert(RLHS->getKind() == MCExpr::Constant && "unexpected RLHS"); 2488 Addend = cast<MCConstantExpr>(RLHS)->getValue(); 2489 } else { 2490 assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS"); 2491 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS)); 2492 } 2493 const MCExpr *RHS = BinaryExpr->getRHS(); 2494 assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS"); 2495 Addend += cast<MCConstantExpr>(RHS)->getValue(); 2496 } else { 2497 assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value"); 2498 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr)); 2499 } 2500 2501 return Relocation({RelOffset, Symbol, RelType, Addend, 0}); 2502 } 2503 2504 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 2505 int64_t Addend, MCContext *Ctx, int64_t &Value, 2506 uint64_t RelType) const override { 2507 unsigned ImmOpNo = -1U; 2508 2509 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 2510 ++Index) { 2511 if (Inst.getOperand(Index).isImm()) { 2512 ImmOpNo = Index; 2513 // TODO: this is a bit hacky. It finds the correct operand by 2514 // searching for a specific immediate value. If no value is 2515 // provided it defaults to the last immediate operand found. 2516 // This could lead to unexpected results if the instruction 2517 // has more than one immediate with the same value. 2518 if (Inst.getOperand(ImmOpNo).getImm() == Value) 2519 break; 2520 } 2521 } 2522 2523 if (ImmOpNo == -1U) 2524 return false; 2525 2526 Value = Inst.getOperand(ImmOpNo).getImm(); 2527 2528 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 2529 2530 return true; 2531 } 2532 2533 bool replaceRegWithImm(MCInst &Inst, unsigned Register, 2534 int64_t Imm) const override { 2535 2536 enum CheckSignExt : uint8_t { 2537 NOCHECK = 0, 2538 CHECK8, 2539 CHECK32, 2540 }; 2541 2542 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 2543 struct InstInfo { 2544 // Size in bytes that Inst loads from memory. 2545 uint8_t DataSize; 2546 2547 // True when the target operand has to be duplicated because the opcode 2548 // expects a LHS operand. 2549 bool HasLHS; 2550 2551 // List of checks and corresponding opcodes to be used. We try to use the 2552 // smallest possible immediate value when various sizes are available, 2553 // hence we may need to check whether a larger constant fits in a smaller 2554 // immediate. 2555 CheckList Checks; 2556 }; 2557 2558 InstInfo I; 2559 2560 switch (Inst.getOpcode()) { 2561 default: { 2562 switch (getPushSize(Inst)) { 2563 2564 case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break; 2565 case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break; 2566 case 8: I = {8, false, {{CHECK8, X86::PUSH64i8}, 2567 {CHECK32, X86::PUSH64i32}, 2568 {NOCHECK, Inst.getOpcode()}}}; break; 2569 default: return false; 2570 } 2571 break; 2572 } 2573 2574 // MOV 2575 case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 2576 case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 2577 case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 2578 case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32}, 2579 {NOCHECK, X86::MOV64ri}}}; break; 2580 2581 case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break; 2582 case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break; 2583 case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break; 2584 case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32}, 2585 {NOCHECK, X86::MOV64mr}}}; break; 2586 2587 // MOVZX 2588 case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 2589 case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 2590 case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 2591 2592 // CMP 2593 case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 2594 case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8}, 2595 {NOCHECK, X86::CMP16ri}}}; break; 2596 case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8}, 2597 {NOCHECK, X86::CMP32ri}}}; break; 2598 case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8}, 2599 {CHECK32, X86::CMP64ri32}, 2600 {NOCHECK, X86::CMP64rr}}}; break; 2601 2602 // TEST 2603 case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 2604 case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 2605 case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 2606 case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 2607 {NOCHECK, X86::TEST64rr}}}; break; 2608 2609 // ADD 2610 case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 2611 case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8}, 2612 {NOCHECK, X86::ADD16ri}}}; break; 2613 case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8}, 2614 {NOCHECK, X86::ADD32ri}}}; break; 2615 case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8}, 2616 {CHECK32, X86::ADD64ri32}, 2617 {NOCHECK, X86::ADD64rr}}}; break; 2618 2619 // SUB 2620 case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 2621 case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8}, 2622 {NOCHECK, X86::SUB16ri}}}; break; 2623 case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8}, 2624 {NOCHECK, X86::SUB32ri}}}; break; 2625 case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8}, 2626 {CHECK32, X86::SUB64ri32}, 2627 {NOCHECK, X86::SUB64rr}}}; break; 2628 2629 // AND 2630 case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 2631 case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8}, 2632 {NOCHECK, X86::AND16ri}}}; break; 2633 case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8}, 2634 {NOCHECK, X86::AND32ri}}}; break; 2635 case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8}, 2636 {CHECK32, X86::AND64ri32}, 2637 {NOCHECK, X86::AND64rr}}}; break; 2638 2639 // OR 2640 case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 2641 case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8}, 2642 {NOCHECK, X86::OR16ri}}}; break; 2643 case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8}, 2644 {NOCHECK, X86::OR32ri}}}; break; 2645 case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8}, 2646 {CHECK32, X86::OR64ri32}, 2647 {NOCHECK, X86::OR64rr}}}; break; 2648 2649 // XOR 2650 case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 2651 case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8}, 2652 {NOCHECK, X86::XOR16ri}}}; break; 2653 case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8}, 2654 {NOCHECK, X86::XOR32ri}}}; break; 2655 case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8}, 2656 {CHECK32, X86::XOR64ri32}, 2657 {NOCHECK, X86::XOR64rr}}}; break; 2658 } 2659 2660 // Compute the new opcode. 2661 unsigned NewOpcode = 0; 2662 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 2663 NewOpcode = Check.second; 2664 if (Check.first == NOCHECK) 2665 break; 2666 if (Check.first == CHECK8 && isInt<8>(Imm)) 2667 break; 2668 if (Check.first == CHECK32 && isInt<32>(Imm)) 2669 break; 2670 } 2671 if (NewOpcode == Inst.getOpcode()) 2672 return false; 2673 2674 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2675 2676 unsigned NumFound = 0; 2677 for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0), 2678 E = InstDesc.getNumOperands(); 2679 Index != E; ++Index) 2680 if (Inst.getOperand(Index).isReg() && 2681 Inst.getOperand(Index).getReg() == Register) 2682 NumFound++; 2683 2684 if (NumFound != 1) 2685 return false; 2686 2687 MCOperand TargetOp = Inst.getOperand(0); 2688 Inst.clear(); 2689 Inst.setOpcode(NewOpcode); 2690 Inst.addOperand(TargetOp); 2691 if (I.HasLHS) 2692 Inst.addOperand(TargetOp); 2693 Inst.addOperand(MCOperand::createImm(Imm)); 2694 2695 return true; 2696 } 2697 2698 bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace, 2699 unsigned ReplaceWith) const override { 2700 2701 // Get the HasLHS value so that iteration can be done 2702 bool HasLHS; 2703 if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) || 2704 X86::isSUB(Inst.getOpcode())) { 2705 HasLHS = true; 2706 } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) || 2707 X86::isTEST(Inst.getOpcode())) { 2708 HasLHS = false; 2709 } else { 2710 switch (Inst.getOpcode()) { 2711 case X86::MOV8rr: 2712 case X86::MOV8rm: 2713 case X86::MOV8mr: 2714 case X86::MOV8ri: 2715 case X86::MOV16rr: 2716 case X86::MOV16rm: 2717 case X86::MOV16mr: 2718 case X86::MOV16ri: 2719 case X86::MOV32rr: 2720 case X86::MOV32rm: 2721 case X86::MOV32mr: 2722 case X86::MOV32ri: 2723 case X86::MOV64rr: 2724 case X86::MOV64rm: 2725 case X86::MOV64mr: 2726 case X86::MOV64ri: 2727 case X86::MOVZX16rr8: 2728 case X86::MOVZX32rr8: 2729 case X86::MOVZX32rr16: 2730 case X86::MOVSX32rm8: 2731 case X86::MOVSX32rr8: 2732 case X86::MOVSX64rm32: 2733 case X86::LEA64r: 2734 HasLHS = false; 2735 break; 2736 default: 2737 return false; 2738 } 2739 } 2740 2741 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2742 2743 bool FoundOne = false; 2744 2745 // Iterate only through src operands that arent also dest operands 2746 for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0), 2747 E = InstDesc.getNumOperands(); 2748 Index != E; ++Index) { 2749 BitVector RegAliases = getAliases(ToReplace, true); 2750 if (!Inst.getOperand(Index).isReg() || 2751 !RegAliases.test(Inst.getOperand(Index).getReg())) 2752 continue; 2753 // Resize register if needed 2754 unsigned SizedReplaceWith = getAliasSized( 2755 ReplaceWith, getRegSize(Inst.getOperand(Index).getReg())); 2756 MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith); 2757 Inst.getOperand(Index) = NewOperand; 2758 FoundOne = true; 2759 } 2760 2761 // Return true if at least one operand was replaced 2762 return FoundOne; 2763 } 2764 2765 bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 2766 MCContext *Ctx) const override { 2767 Inst.setOpcode(X86::JMP_1); 2768 Inst.addOperand(MCOperand::createExpr( 2769 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); 2770 return true; 2771 } 2772 2773 bool createCall(MCInst &Inst, const MCSymbol *Target, 2774 MCContext *Ctx) override { 2775 Inst.setOpcode(X86::CALL64pcrel32); 2776 Inst.addOperand(MCOperand::createExpr( 2777 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2778 return true; 2779 } 2780 2781 bool createTailCall(MCInst &Inst, const MCSymbol *Target, 2782 MCContext *Ctx) override { 2783 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 2784 } 2785 2786 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 2787 MCContext *Ctx) override { 2788 Seq.clear(); 2789 Seq.emplace_back(); 2790 createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true); 2791 } 2792 2793 bool createTrap(MCInst &Inst) const override { 2794 Inst.clear(); 2795 Inst.setOpcode(X86::TRAP); 2796 return true; 2797 } 2798 2799 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 2800 MCContext *Ctx) const override { 2801 unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); 2802 assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); 2803 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); 2804 Inst.getOperand(0) = MCOperand::createExpr( 2805 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2806 return true; 2807 } 2808 2809 bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, 2810 unsigned CC) const override { 2811 if (CC == X86::COND_INVALID) 2812 return false; 2813 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC); 2814 Inst.getOperand(0) = MCOperand::createExpr( 2815 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2816 return true; 2817 } 2818 2819 unsigned getCanonicalBranchCondCode(unsigned CC) const override { 2820 switch (CC) { 2821 default: return X86::COND_INVALID; 2822 2823 case X86::COND_E: return X86::COND_E; 2824 case X86::COND_NE: return X86::COND_E; 2825 2826 case X86::COND_L: return X86::COND_L; 2827 case X86::COND_GE: return X86::COND_L; 2828 2829 case X86::COND_LE: return X86::COND_G; 2830 case X86::COND_G: return X86::COND_G; 2831 2832 case X86::COND_B: return X86::COND_B; 2833 case X86::COND_AE: return X86::COND_B; 2834 2835 case X86::COND_BE: return X86::COND_A; 2836 case X86::COND_A: return X86::COND_A; 2837 2838 case X86::COND_S: return X86::COND_S; 2839 case X86::COND_NS: return X86::COND_S; 2840 2841 case X86::COND_P: return X86::COND_P; 2842 case X86::COND_NP: return X86::COND_P; 2843 2844 case X86::COND_O: return X86::COND_O; 2845 case X86::COND_NO: return X86::COND_O; 2846 } 2847 } 2848 2849 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 2850 MCContext *Ctx) const override { 2851 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 2852 "Invalid instruction"); 2853 Inst.getOperand(0) = MCOperand::createExpr( 2854 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2855 return true; 2856 } 2857 2858 MCPhysReg getX86R11() const override { return X86::R11; } 2859 2860 unsigned getShortBranchOpcode(unsigned Opcode) const override { 2861 switch (Opcode) { 2862 default: 2863 return Opcode; 2864 case X86::JMP_2: 2865 return X86::JMP_1; 2866 case X86::JMP_4: 2867 return X86::JMP_1; 2868 case X86::JCC_2: 2869 return X86::JCC_1; 2870 case X86::JCC_4: 2871 return X86::JCC_1; 2872 } 2873 } 2874 2875 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 2876 // FIXME: this should depend on the calling convention. 2877 switch (ArgNo) { 2878 case 0: return X86::RDI; 2879 case 1: return X86::RSI; 2880 case 2: return X86::RDX; 2881 case 3: return X86::RCX; 2882 case 4: return X86::R8; 2883 case 5: return X86::R9; 2884 default: return getNoRegister(); 2885 } 2886 } 2887 2888 void createPause(MCInst &Inst) const override { 2889 Inst.clear(); 2890 Inst.setOpcode(X86::PAUSE); 2891 } 2892 2893 void createLfence(MCInst &Inst) const override { 2894 Inst.clear(); 2895 Inst.setOpcode(X86::LFENCE); 2896 } 2897 2898 bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 2899 bool IsTailCall) override { 2900 Inst.clear(); 2901 Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 2902 Inst.addOperand(MCOperand::createExpr( 2903 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2904 if (IsTailCall) 2905 setTailCall(Inst); 2906 return true; 2907 } 2908 2909 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 2910 MCContext *Ctx, bool IsTailCall) override { 2911 Seq.clear(); 2912 MCInst Inst; 2913 Inst.setOpcode(X86::JMP_1); 2914 Inst.addOperand(MCOperand::createExpr( 2915 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2916 if (IsTailCall) 2917 setTailCall(Inst); 2918 Seq.emplace_back(Inst); 2919 } 2920 2921 bool isConditionalMove(const MCInst &Inst) const override { 2922 unsigned OpCode = Inst.getOpcode(); 2923 return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr || 2924 OpCode == X86::CMOV64rr); 2925 } 2926 2927 bool isBranchOnMem(const MCInst &Inst) const override { 2928 unsigned OpCode = Inst.getOpcode(); 2929 if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) || 2930 OpCode == X86::JMP64m) 2931 return true; 2932 2933 return false; 2934 } 2935 2936 bool isBranchOnReg(const MCInst &Inst) const override { 2937 unsigned OpCode = Inst.getOpcode(); 2938 if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) || 2939 OpCode == X86::JMP64r) 2940 return true; 2941 2942 return false; 2943 } 2944 2945 void createPushRegister(MCInst &Inst, MCPhysReg Reg, 2946 unsigned Size) const override { 2947 Inst.clear(); 2948 unsigned NewOpcode = 0; 2949 if (Reg == X86::EFLAGS) { 2950 switch (Size) { 2951 case 2: NewOpcode = X86::PUSHF16; break; 2952 case 4: NewOpcode = X86::PUSHF32; break; 2953 case 8: NewOpcode = X86::PUSHF64; break; 2954 default: 2955 llvm_unreachable("Unexpected size"); 2956 } 2957 Inst.setOpcode(NewOpcode); 2958 return; 2959 } 2960 switch (Size) { 2961 case 2: NewOpcode = X86::PUSH16r; break; 2962 case 4: NewOpcode = X86::PUSH32r; break; 2963 case 8: NewOpcode = X86::PUSH64r; break; 2964 default: 2965 llvm_unreachable("Unexpected size"); 2966 } 2967 Inst.setOpcode(NewOpcode); 2968 Inst.addOperand(MCOperand::createReg(Reg)); 2969 } 2970 2971 void createPopRegister(MCInst &Inst, MCPhysReg Reg, 2972 unsigned Size) const override { 2973 Inst.clear(); 2974 unsigned NewOpcode = 0; 2975 if (Reg == X86::EFLAGS) { 2976 switch (Size) { 2977 case 2: NewOpcode = X86::POPF16; break; 2978 case 4: NewOpcode = X86::POPF32; break; 2979 case 8: NewOpcode = X86::POPF64; break; 2980 default: 2981 llvm_unreachable("Unexpected size"); 2982 } 2983 Inst.setOpcode(NewOpcode); 2984 return; 2985 } 2986 switch (Size) { 2987 case 2: NewOpcode = X86::POP16r; break; 2988 case 4: NewOpcode = X86::POP32r; break; 2989 case 8: NewOpcode = X86::POP64r; break; 2990 default: 2991 llvm_unreachable("Unexpected size"); 2992 } 2993 Inst.setOpcode(NewOpcode); 2994 Inst.addOperand(MCOperand::createReg(Reg)); 2995 } 2996 2997 void createPushFlags(MCInst &Inst, unsigned Size) const override { 2998 return createPushRegister(Inst, X86::EFLAGS, Size); 2999 } 3000 3001 void createPopFlags(MCInst &Inst, unsigned Size) const override { 3002 return createPopRegister(Inst, X86::EFLAGS, Size); 3003 } 3004 3005 void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value, 3006 unsigned Size) const { 3007 unsigned int Opcode; 3008 switch (Size) { 3009 case 1: Opcode = X86::ADD8ri; break; 3010 case 2: Opcode = X86::ADD16ri; break; 3011 case 4: Opcode = X86::ADD32ri; break; 3012 default: 3013 llvm_unreachable("Unexpected size"); 3014 } 3015 Inst.setOpcode(Opcode); 3016 Inst.clear(); 3017 Inst.addOperand(MCOperand::createReg(Reg)); 3018 Inst.addOperand(MCOperand::createReg(Reg)); 3019 Inst.addOperand(MCOperand::createImm(Value)); 3020 } 3021 3022 void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg, 3023 unsigned Size) const { 3024 unsigned int Opcode; 3025 switch (Size) { 3026 case 1: Opcode = X86::MOV8ri; break; 3027 case 2: Opcode = X86::MOV16ri; break; 3028 case 4: Opcode = X86::MOV32ri; break; 3029 // Writing to a 32-bit register always zeros the upper 32 bits of the 3030 // full-width register 3031 case 8: 3032 Opcode = X86::MOV32ri; 3033 Reg = getAliasSized(Reg, 4); 3034 break; 3035 default: 3036 llvm_unreachable("Unexpected size"); 3037 } 3038 Inst.setOpcode(Opcode); 3039 Inst.clear(); 3040 Inst.addOperand(MCOperand::createReg(Reg)); 3041 Inst.addOperand(MCOperand::createImm(0)); 3042 } 3043 3044 void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const { 3045 Inst.setOpcode(X86::SETCCr); 3046 Inst.clear(); 3047 Inst.addOperand(MCOperand::createReg(Reg)); 3048 Inst.addOperand(MCOperand::createImm(X86::COND_O)); 3049 } 3050 3051 void createX86Lahf(MCInst &Inst) const { 3052 Inst.setOpcode(X86::LAHF); 3053 Inst.clear(); 3054 } 3055 3056 void createX86Sahf(MCInst &Inst) const { 3057 Inst.setOpcode(X86::SAHF); 3058 Inst.clear(); 3059 } 3060 3061 InstructionListType createInstrIncMemory(const MCSymbol *Target, 3062 MCContext *Ctx, 3063 bool IsLeaf) const override { 3064 InstructionListType Instrs(IsLeaf ? 13 : 11); 3065 unsigned int I = 0; 3066 3067 // Don't clobber application red zone (ABI dependent) 3068 if (IsLeaf) 3069 createStackPointerIncrement(Instrs[I++], 128, 3070 /*NoFlagsClobber=*/true); 3071 3072 // Performance improvements based on the optimization discussed at 3073 // https://reviews.llvm.org/D6629 3074 // LAHF/SAHF are used instead of PUSHF/POPF 3075 // PUSHF 3076 createPushRegister(Instrs[I++], X86::RAX, 8); 3077 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3078 createX86Lahf(Instrs[I++]); 3079 createPushRegister(Instrs[I++], X86::RAX, 8); 3080 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3081 createX86SaveOVFlagToRegister(Instrs[I++], X86::AL); 3082 // LOCK INC 3083 createIncMemory(Instrs[I++], Target, Ctx); 3084 // POPF 3085 createAddRegImm(Instrs[I++], X86::AL, 127, 1); 3086 createPopRegister(Instrs[I++], X86::RAX, 8); 3087 createX86Sahf(Instrs[I++]); 3088 createPopRegister(Instrs[I++], X86::RAX, 8); 3089 3090 if (IsLeaf) 3091 createStackPointerDecrement(Instrs[I], 128, 3092 /*NoFlagsClobber=*/true); 3093 return Instrs; 3094 } 3095 3096 void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg, 3097 int64_t Disp) const { 3098 Inst.setOpcode(X86::XCHG64rm); 3099 Inst.addOperand(MCOperand::createReg(Source)); 3100 Inst.addOperand(MCOperand::createReg(Source)); 3101 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3102 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3103 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3104 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3105 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3106 } 3107 3108 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 3109 int64_t Disp) const { 3110 Inst.setOpcode(X86::JMP64m); 3111 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3112 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3113 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3114 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3115 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3116 } 3117 3118 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 3119 MCSymbol *HandlerFuncAddr, 3120 int CallSiteID, 3121 MCContext *Ctx) override { 3122 // Check if the target address expression used in the original indirect call 3123 // uses the stack pointer, which we are going to clobber. 3124 static BitVector SPAliases(getAliases(X86::RSP)); 3125 bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) { 3126 return Op.isReg() && SPAliases[Op.getReg()]; 3127 }); 3128 3129 InstructionListType Insts; 3130 MCPhysReg TempReg = getIntArgRegister(0); 3131 // Code sequence used to enter indirect call instrumentation helper: 3132 // push %rdi 3133 // add $8, %rsp ;; $rsp may be used in target, so fix it to prev val 3134 // movq target, %rdi ;; via convertIndirectCallTargetToLoad 3135 // sub $8, %rsp ;; restore correct stack value 3136 // push %rdi 3137 // movq $CallSiteID, %rdi 3138 // push %rdi 3139 // callq/jmp HandlerFuncAddr 3140 Insts.emplace_back(); 3141 createPushRegister(Insts.back(), TempReg, 8); 3142 if (UsesSP) { // Only adjust SP if we really need to 3143 Insts.emplace_back(); 3144 createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3145 } 3146 Insts.emplace_back(CallInst); 3147 // Insts.back() and CallInst now share the same annotation instruction. 3148 // Strip it from Insts.back(), only preserving tail call annotation. 3149 stripAnnotations(Insts.back(), /*KeepTC=*/true); 3150 convertIndirectCallToLoad(Insts.back(), TempReg); 3151 if (UsesSP) { 3152 Insts.emplace_back(); 3153 createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3154 } 3155 Insts.emplace_back(); 3156 createPushRegister(Insts.back(), TempReg, 8); 3157 Insts.emplace_back(); 3158 createLoadImmediate(Insts.back(), TempReg, CallSiteID); 3159 Insts.emplace_back(); 3160 createPushRegister(Insts.back(), TempReg, 8); 3161 3162 MCInst &NewCallInst = Insts.emplace_back(); 3163 createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); 3164 3165 // Carry over metadata including tail call marker if present. 3166 stripAnnotations(NewCallInst); 3167 moveAnnotations(std::move(CallInst), NewCallInst); 3168 3169 return Insts; 3170 } 3171 3172 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 3173 const MCPhysReg TempReg = getIntArgRegister(0); 3174 // We just need to undo the sequence created for every ind call in 3175 // instrumentIndirectTarget(), which can be accomplished minimally with: 3176 // popfq 3177 // pop %rdi 3178 // add $16, %rsp 3179 // xchg (%rsp), %rdi 3180 // jmp *-8(%rsp) 3181 InstructionListType Insts(5); 3182 createPopFlags(Insts[0], 8); 3183 createPopRegister(Insts[1], TempReg, 8); 3184 createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); 3185 createSwap(Insts[3], TempReg, X86::RSP, 0); 3186 createIndirectBranch(Insts[4], X86::RSP, -8); 3187 return Insts; 3188 } 3189 3190 InstructionListType 3191 createInstrumentedIndTailCallHandlerExitBB() const override { 3192 const MCPhysReg TempReg = getIntArgRegister(0); 3193 // Same thing as above, but for tail calls 3194 // popfq 3195 // add $16, %rsp 3196 // pop %rdi 3197 // jmp *-16(%rsp) 3198 InstructionListType Insts(4); 3199 createPopFlags(Insts[0], 8); 3200 createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); 3201 createPopRegister(Insts[2], TempReg, 8); 3202 createIndirectBranch(Insts[3], X86::RSP, -16); 3203 return Insts; 3204 } 3205 3206 InstructionListType 3207 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 3208 const MCSymbol *IndCallHandler, 3209 MCContext *Ctx) override { 3210 const MCPhysReg TempReg = getIntArgRegister(0); 3211 // Code sequence used to check whether InstrTampoline was initialized 3212 // and call it if so, returns via IndCallHandler. 3213 // pushfq 3214 // mov InstrTrampoline,%rdi 3215 // cmp $0x0,%rdi 3216 // je IndCallHandler 3217 // callq *%rdi 3218 // jmpq IndCallHandler 3219 InstructionListType Insts; 3220 Insts.emplace_back(); 3221 createPushFlags(Insts.back(), 8); 3222 Insts.emplace_back(); 3223 createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); 3224 InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); 3225 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 3226 Insts.emplace_back(); 3227 Insts.back().setOpcode(X86::CALL64r); 3228 Insts.back().addOperand(MCOperand::createReg(TempReg)); 3229 Insts.emplace_back(); 3230 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 3231 return Insts; 3232 } 3233 3234 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 3235 InstructionListType Insts(2); 3236 MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); 3237 createMove(Insts[0], NumLocs, X86::EAX, Ctx); 3238 createReturn(Insts[1]); 3239 return Insts; 3240 } 3241 3242 InstructionListType 3243 createInstrLocationsGetter(MCContext *Ctx) const override { 3244 InstructionListType Insts(2); 3245 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); 3246 createLea(Insts[0], Locs, X86::EAX, Ctx); 3247 createReturn(Insts[1]); 3248 return Insts; 3249 } 3250 3251 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 3252 InstructionListType Insts(2); 3253 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); 3254 createLea(Insts[0], Locs, X86::EAX, Ctx); 3255 createReturn(Insts[1]); 3256 return Insts; 3257 } 3258 3259 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 3260 InstructionListType Insts(2); 3261 MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); 3262 createMove(Insts[0], NumFuncs, X86::EAX, Ctx); 3263 createReturn(Insts[1]); 3264 return Insts; 3265 } 3266 3267 InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, 3268 MCContext *Ctx) const override { 3269 InstructionListType Insts(1); 3270 createUncondBranch(Insts[0], TgtSym, Ctx); 3271 return Insts; 3272 } 3273 3274 InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { 3275 InstructionListType Insts(1); 3276 createReturn(Insts[0]); 3277 return Insts; 3278 } 3279 3280 BlocksVectorTy indirectCallPromotion( 3281 const MCInst &CallInst, 3282 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3283 const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms, 3284 const std::vector<MCInst *> &MethodFetchInsns, 3285 const bool MinimizeCodeSize, MCContext *Ctx) override { 3286 const bool IsTailCall = isTailCall(CallInst); 3287 const bool IsJumpTable = getJumpTable(CallInst) != 0; 3288 BlocksVectorTy Results; 3289 3290 // Label for the current code block. 3291 MCSymbol *NextTarget = nullptr; 3292 3293 // The join block which contains all the instructions following CallInst. 3294 // MergeBlock remains null if CallInst is a tail call. 3295 MCSymbol *MergeBlock = nullptr; 3296 3297 unsigned FuncAddrReg = X86::R10; 3298 3299 const bool LoadElim = !VtableSyms.empty(); 3300 assert((!LoadElim || VtableSyms.size() == Targets.size()) && 3301 "There must be a vtable entry for every method " 3302 "in the targets vector."); 3303 3304 if (MinimizeCodeSize && !LoadElim) { 3305 std::set<unsigned> UsedRegs; 3306 3307 for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) { 3308 const MCOperand &Op = CallInst.getOperand(I); 3309 if (Op.isReg()) 3310 UsedRegs.insert(Op.getReg()); 3311 } 3312 3313 if (UsedRegs.count(X86::R10) == 0) 3314 FuncAddrReg = X86::R10; 3315 else if (UsedRegs.count(X86::R11) == 0) 3316 FuncAddrReg = X86::R11; 3317 else 3318 return Results; 3319 } 3320 3321 const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { 3322 assert(MergeBlock); 3323 NewCall.push_back(CallInst); 3324 MCInst &Merge = NewCall.back(); 3325 Merge.clear(); 3326 createUncondBranch(Merge, MergeBlock, Ctx); 3327 }; 3328 3329 for (unsigned int i = 0; i < Targets.size(); ++i) { 3330 Results.emplace_back(NextTarget, InstructionListType()); 3331 InstructionListType *NewCall = &Results.back().second; 3332 3333 if (MinimizeCodeSize && !LoadElim) { 3334 // Load the call target into FuncAddrReg. 3335 NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc 3336 MCInst &Target = NewCall->back(); 3337 Target.clear(); 3338 Target.setOpcode(X86::MOV64ri32); 3339 Target.addOperand(MCOperand::createReg(FuncAddrReg)); 3340 if (Targets[i].first) { 3341 // Is this OK? 3342 Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3343 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3344 } else { 3345 const uint64_t Addr = Targets[i].second; 3346 // Immediate address is out of sign extended 32 bit range. 3347 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3348 return BlocksVectorTy(); 3349 3350 Target.addOperand(MCOperand::createImm(Addr)); 3351 } 3352 3353 // Compare current call target to a specific address. 3354 NewCall->push_back(CallInst); 3355 MCInst &Compare = NewCall->back(); 3356 Compare.clear(); 3357 if (isBranchOnReg(CallInst)) 3358 Compare.setOpcode(X86::CMP64rr); 3359 else if (CallInst.getOpcode() == X86::CALL64pcrel32) 3360 Compare.setOpcode(X86::CMP64ri32); 3361 else 3362 Compare.setOpcode(X86::CMP64rm); 3363 3364 Compare.addOperand(MCOperand::createReg(FuncAddrReg)); 3365 3366 // TODO: Would be preferable to only load this value once. 3367 for (unsigned i = 0; 3368 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3369 if (!CallInst.getOperand(i).isInst()) 3370 Compare.addOperand(CallInst.getOperand(i)); 3371 } else { 3372 // Compare current call target to a specific address. 3373 NewCall->push_back(CallInst); 3374 MCInst &Compare = NewCall->back(); 3375 Compare.clear(); 3376 if (isBranchOnReg(CallInst)) 3377 Compare.setOpcode(X86::CMP64ri32); 3378 else 3379 Compare.setOpcode(X86::CMP64mi32); 3380 3381 // Original call address. 3382 for (unsigned i = 0; 3383 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3384 if (!CallInst.getOperand(i).isInst()) 3385 Compare.addOperand(CallInst.getOperand(i)); 3386 3387 // Target address. 3388 if (Targets[i].first || LoadElim) { 3389 const MCSymbol *Sym = 3390 LoadElim ? VtableSyms[i].first : Targets[i].first; 3391 const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0; 3392 const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx); 3393 if (Addend) 3394 Expr = MCBinaryExpr::createAdd( 3395 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx); 3396 Compare.addOperand(MCOperand::createExpr(Expr)); 3397 } else { 3398 const uint64_t Addr = Targets[i].second; 3399 // Immediate address is out of sign extended 32 bit range. 3400 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3401 return BlocksVectorTy(); 3402 3403 Compare.addOperand(MCOperand::createImm(Addr)); 3404 } 3405 } 3406 3407 // jump to next target compare. 3408 NextTarget = 3409 Ctx->createNamedTempSymbol(); // generate label for the next block 3410 NewCall->push_back(CallInst); 3411 3412 if (IsJumpTable) { 3413 MCInst &Je = NewCall->back(); 3414 3415 // Jump to next compare if target addresses don't match. 3416 Je.clear(); 3417 Je.setOpcode(X86::JCC_1); 3418 if (Targets[i].first) 3419 Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3420 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3421 else 3422 Je.addOperand(MCOperand::createImm(Targets[i].second)); 3423 3424 Je.addOperand(MCOperand::createImm(X86::COND_E)); 3425 assert(!isInvoke(CallInst)); 3426 } else { 3427 MCInst &Jne = NewCall->back(); 3428 3429 // Jump to next compare if target addresses don't match. 3430 Jne.clear(); 3431 Jne.setOpcode(X86::JCC_1); 3432 Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3433 NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); 3434 Jne.addOperand(MCOperand::createImm(X86::COND_NE)); 3435 3436 // Call specific target directly. 3437 Results.emplace_back(Ctx->createNamedTempSymbol(), 3438 InstructionListType()); 3439 NewCall = &Results.back().second; 3440 NewCall->push_back(CallInst); 3441 MCInst &CallOrJmp = NewCall->back(); 3442 3443 CallOrJmp.clear(); 3444 3445 if (MinimizeCodeSize && !LoadElim) { 3446 CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r); 3447 CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); 3448 } else { 3449 CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 3450 3451 if (Targets[i].first) 3452 CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3453 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3454 else 3455 CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); 3456 } 3457 if (IsTailCall) 3458 setTailCall(CallOrJmp); 3459 3460 if (CallOrJmp.getOpcode() == X86::CALL64r || 3461 CallOrJmp.getOpcode() == X86::CALL64pcrel32) { 3462 if (std::optional<uint32_t> Offset = getOffset(CallInst)) 3463 // Annotated as duplicated call 3464 setOffset(CallOrJmp, *Offset); 3465 } 3466 3467 if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { 3468 // Copy over any EH or GNU args size information from the original 3469 // call. 3470 std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst); 3471 if (EHInfo) 3472 addEHInfo(CallOrJmp, *EHInfo); 3473 int64_t GnuArgsSize = getGnuArgsSize(CallInst); 3474 if (GnuArgsSize >= 0) 3475 addGnuArgsSize(CallOrJmp, GnuArgsSize); 3476 } 3477 3478 if (!IsTailCall) { 3479 // The fallthrough block for the most common target should be 3480 // the merge block. 3481 if (i == 0) { 3482 // Fallthrough to merge block. 3483 MergeBlock = Ctx->createNamedTempSymbol(); 3484 } else { 3485 // Insert jump to the merge block if we are not doing a fallthrough. 3486 jumpToMergeBlock(*NewCall); 3487 } 3488 } 3489 } 3490 } 3491 3492 // Cold call block. 3493 Results.emplace_back(NextTarget, InstructionListType()); 3494 InstructionListType &NewCall = Results.back().second; 3495 for (const MCInst *Inst : MethodFetchInsns) 3496 if (Inst != &CallInst) 3497 NewCall.push_back(*Inst); 3498 NewCall.push_back(CallInst); 3499 3500 // Jump to merge block from cold call block 3501 if (!IsTailCall && !IsJumpTable) { 3502 jumpToMergeBlock(NewCall); 3503 3504 // Record merge block 3505 Results.emplace_back(MergeBlock, InstructionListType()); 3506 } 3507 3508 return Results; 3509 } 3510 3511 BlocksVectorTy jumpTablePromotion( 3512 const MCInst &IJmpInst, 3513 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3514 const std::vector<MCInst *> &TargetFetchInsns, 3515 MCContext *Ctx) const override { 3516 assert(getJumpTable(IJmpInst) != 0); 3517 uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg"); 3518 if (IndexReg == 0) 3519 return BlocksVectorTy(); 3520 3521 BlocksVectorTy Results; 3522 3523 // Label for the current code block. 3524 MCSymbol *NextTarget = nullptr; 3525 3526 for (unsigned int i = 0; i < Targets.size(); ++i) { 3527 Results.emplace_back(NextTarget, InstructionListType()); 3528 InstructionListType *CurBB = &Results.back().second; 3529 3530 // Compare current index to a specific index. 3531 CurBB->emplace_back(MCInst()); 3532 MCInst &CompareInst = CurBB->back(); 3533 CompareInst.setLoc(IJmpInst.getLoc()); 3534 CompareInst.setOpcode(X86::CMP64ri32); 3535 CompareInst.addOperand(MCOperand::createReg(IndexReg)); 3536 3537 const uint64_t CaseIdx = Targets[i].second; 3538 // Immediate address is out of sign extended 32 bit range. 3539 if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) 3540 return BlocksVectorTy(); 3541 3542 CompareInst.addOperand(MCOperand::createImm(CaseIdx)); 3543 shortenInstruction(CompareInst, *Ctx->getSubtargetInfo()); 3544 3545 // jump to next target compare. 3546 NextTarget = 3547 Ctx->createNamedTempSymbol(); // generate label for the next block 3548 CurBB->push_back(MCInst()); 3549 3550 MCInst &JEInst = CurBB->back(); 3551 JEInst.setLoc(IJmpInst.getLoc()); 3552 3553 // Jump to target if indices match 3554 JEInst.setOpcode(X86::JCC_1); 3555 JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3556 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3557 JEInst.addOperand(MCOperand::createImm(X86::COND_E)); 3558 } 3559 3560 // Cold call block. 3561 Results.emplace_back(NextTarget, InstructionListType()); 3562 InstructionListType &CurBB = Results.back().second; 3563 for (const MCInst *Inst : TargetFetchInsns) 3564 if (Inst != &IJmpInst) 3565 CurBB.push_back(*Inst); 3566 3567 CurBB.push_back(IJmpInst); 3568 3569 return Results; 3570 } 3571 3572 private: 3573 bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3574 MCContext *Ctx) const { 3575 Inst.setOpcode(X86::MOV64rm); 3576 Inst.addOperand(MCOperand::createReg(Reg)); 3577 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3578 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3579 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3580 Inst.addOperand(MCOperand::createExpr( 3581 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3582 *Ctx))); // Displacement 3583 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3584 3585 return true; 3586 } 3587 3588 bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3589 MCContext *Ctx) const { 3590 Inst.setOpcode(X86::LEA64r); 3591 Inst.addOperand(MCOperand::createReg(Reg)); 3592 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3593 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3594 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3595 Inst.addOperand(MCOperand::createExpr( 3596 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3597 *Ctx))); // Displacement 3598 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3599 return true; 3600 } 3601 }; 3602 3603 } // namespace 3604 3605 namespace llvm { 3606 namespace bolt { 3607 3608 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, 3609 const MCInstrInfo *Info, 3610 const MCRegisterInfo *RegInfo) { 3611 return new X86MCPlusBuilder(Analysis, Info, RegInfo); 3612 } 3613 3614 } // namespace bolt 3615 } // namespace llvm 3616