1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides X86-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86EncodingOptimization.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "X86MCSymbolizer.h" 17 #include "bolt/Core/MCPlus.h" 18 #include "bolt/Core/MCPlusBuilder.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCFixupKindInfo.h" 22 #include "llvm/MC/MCInst.h" 23 #include "llvm/MC/MCInstBuilder.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegister.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/DataExtractor.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/Errc.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/ErrorOr.h" 33 #include <set> 34 35 #define DEBUG_TYPE "mcplus" 36 37 using namespace llvm; 38 using namespace bolt; 39 40 namespace opts { 41 42 extern cl::OptionCategory BoltOptCategory; 43 44 static cl::opt<bool> X86StripRedundantAddressSize( 45 "x86-strip-redundant-address-size", 46 cl::desc("Remove redundant Address-Size override prefix"), cl::init(true), 47 cl::cat(BoltOptCategory)); 48 49 } // namespace opts 50 51 namespace { 52 53 bool isMOVSX64rm32(const MCInst &Inst) { 54 return Inst.getOpcode() == X86::MOVSX64rm32; 55 } 56 57 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; } 58 59 bool isADDri(const MCInst &Inst) { 60 return Inst.getOpcode() == X86::ADD64ri32 || 61 Inst.getOpcode() == X86::ADD64ri8; 62 } 63 64 // Create instruction to increment contents of target by 1 65 static InstructionListType createIncMemory(const MCSymbol *Target, 66 MCContext *Ctx) { 67 InstructionListType Insts; 68 Insts.emplace_back(); 69 Insts.back().setOpcode(X86::LOCK_INC64m); 70 Insts.back().clear(); 71 Insts.back().addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 72 Insts.back().addOperand(MCOperand::createImm(1)); // ScaleAmt 73 Insts.back().addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 74 75 Insts.back().addOperand(MCOperand::createExpr( 76 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, 77 *Ctx))); // Displacement 78 Insts.back().addOperand( 79 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 80 return Insts; 81 } 82 83 #define GET_INSTRINFO_OPERAND_TYPES_ENUM 84 #define GET_INSTRINFO_OPERAND_TYPE 85 #define GET_INSTRINFO_MEM_OPERAND_SIZE 86 #include "X86GenInstrInfo.inc" 87 88 class X86MCPlusBuilder : public MCPlusBuilder { 89 public: 90 using MCPlusBuilder::MCPlusBuilder; 91 92 std::unique_ptr<MCSymbolizer> 93 createTargetSymbolizer(BinaryFunction &Function, 94 bool CreateNewSymbols) const override { 95 return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols); 96 } 97 98 bool isBranch(const MCInst &Inst) const override { 99 return Analysis->isBranch(Inst) && !isTailCall(Inst); 100 } 101 102 bool isNoop(const MCInst &Inst) const override { 103 return X86::isNOP(Inst.getOpcode()); 104 } 105 106 unsigned getCondCode(const MCInst &Inst) const override { 107 unsigned Opcode = Inst.getOpcode(); 108 if (X86::isJCC(Opcode)) 109 return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm(); 110 return X86::COND_INVALID; 111 } 112 113 unsigned getInvertedCondCode(unsigned CC) const override { 114 switch (CC) { 115 default: return X86::COND_INVALID; 116 case X86::COND_E: return X86::COND_NE; 117 case X86::COND_NE: return X86::COND_E; 118 case X86::COND_L: return X86::COND_GE; 119 case X86::COND_LE: return X86::COND_G; 120 case X86::COND_G: return X86::COND_LE; 121 case X86::COND_GE: return X86::COND_L; 122 case X86::COND_B: return X86::COND_AE; 123 case X86::COND_BE: return X86::COND_A; 124 case X86::COND_A: return X86::COND_BE; 125 case X86::COND_AE: return X86::COND_B; 126 case X86::COND_S: return X86::COND_NS; 127 case X86::COND_NS: return X86::COND_S; 128 case X86::COND_P: return X86::COND_NP; 129 case X86::COND_NP: return X86::COND_P; 130 case X86::COND_O: return X86::COND_NO; 131 case X86::COND_NO: return X86::COND_O; 132 } 133 } 134 135 unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override { 136 enum DecodedCondCode : uint8_t { 137 DCC_EQUAL = 0x1, 138 DCC_GREATER = 0x2, 139 DCC_LESSER = 0x4, 140 DCC_GREATER_OR_LESSER = 0x6, 141 DCC_UNSIGNED = 0x8, 142 DCC_SIGNED = 0x10, 143 DCC_INVALID = 0x20, 144 }; 145 146 auto decodeCondCode = [&](unsigned CC) -> uint8_t { 147 switch (CC) { 148 default: return DCC_INVALID; 149 case X86::COND_E: return DCC_EQUAL; 150 case X86::COND_NE: return DCC_GREATER | DCC_LESSER; 151 case X86::COND_L: return DCC_LESSER | DCC_SIGNED; 152 case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED; 153 case X86::COND_G: return DCC_GREATER | DCC_SIGNED; 154 case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED; 155 case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED; 156 case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED; 157 case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED; 158 case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED; 159 } 160 }; 161 162 uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2); 163 164 if (DCC & DCC_INVALID) 165 return X86::COND_INVALID; 166 167 if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED) 168 return X86::COND_INVALID; 169 170 switch (DCC) { 171 default: return X86::COND_INVALID; 172 case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE; 173 case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE; 174 case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE; 175 case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE; 176 case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE; 177 case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE; 178 case DCC_GREATER | DCC_LESSER: return X86::COND_NE; 179 case DCC_EQUAL | DCC_SIGNED: return X86::COND_E; 180 case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E; 181 case DCC_EQUAL: return X86::COND_E; 182 case DCC_LESSER | DCC_SIGNED: return X86::COND_L; 183 case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B; 184 case DCC_GREATER | DCC_SIGNED: return X86::COND_G; 185 case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A; 186 } 187 } 188 189 bool isValidCondCode(unsigned CC) const override { 190 return (CC != X86::COND_INVALID); 191 } 192 193 bool isBreakpoint(const MCInst &Inst) const override { 194 return Inst.getOpcode() == X86::INT3; 195 } 196 197 bool isPrefix(const MCInst &Inst) const override { 198 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 199 return X86II::isPrefix(Desc.TSFlags); 200 } 201 202 bool isRep(const MCInst &Inst) const override { 203 return Inst.getFlags() == X86::IP_HAS_REPEAT; 204 } 205 206 bool deleteREPPrefix(MCInst &Inst) const override { 207 if (Inst.getFlags() == X86::IP_HAS_REPEAT) { 208 Inst.setFlags(0); 209 return true; 210 } 211 return false; 212 } 213 214 // FIXME: For compatibility with old LLVM only! 215 bool isTerminator(const MCInst &Inst) const override { 216 unsigned Opcode = Inst.getOpcode(); 217 return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || 218 X86::isUD2(Opcode); 219 } 220 221 bool isIndirectCall(const MCInst &Inst) const override { 222 return isCall(Inst) && 223 ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); 224 } 225 226 bool isPop(const MCInst &Inst) const override { 227 return getPopSize(Inst) == 0 ? false : true; 228 } 229 230 bool isTerminateBranch(const MCInst &Inst) const override { 231 return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64; 232 } 233 234 int getPopSize(const MCInst &Inst) const override { 235 switch (Inst.getOpcode()) { 236 case X86::POP16r: 237 case X86::POP16rmm: 238 case X86::POP16rmr: 239 case X86::POPF16: 240 case X86::POPA16: 241 case X86::POPDS16: 242 case X86::POPES16: 243 case X86::POPFS16: 244 case X86::POPGS16: 245 case X86::POPSS16: 246 return 2; 247 case X86::POP32r: 248 case X86::POP32rmm: 249 case X86::POP32rmr: 250 case X86::POPA32: 251 case X86::POPDS32: 252 case X86::POPES32: 253 case X86::POPF32: 254 case X86::POPFS32: 255 case X86::POPGS32: 256 case X86::POPSS32: 257 return 4; 258 case X86::POP64r: 259 case X86::POP64rmm: 260 case X86::POP64rmr: 261 case X86::POPF64: 262 case X86::POPFS64: 263 case X86::POPGS64: 264 return 8; 265 } 266 return 0; 267 } 268 269 bool isPush(const MCInst &Inst) const override { 270 return getPushSize(Inst) == 0 ? false : true; 271 } 272 273 int getPushSize(const MCInst &Inst) const override { 274 switch (Inst.getOpcode()) { 275 case X86::PUSH16i8: 276 case X86::PUSH16r: 277 case X86::PUSH16rmm: 278 case X86::PUSH16rmr: 279 case X86::PUSHA16: 280 case X86::PUSHCS16: 281 case X86::PUSHDS16: 282 case X86::PUSHES16: 283 case X86::PUSHF16: 284 case X86::PUSHFS16: 285 case X86::PUSHGS16: 286 case X86::PUSHSS16: 287 case X86::PUSH16i: 288 return 2; 289 case X86::PUSH32i8: 290 case X86::PUSH32r: 291 case X86::PUSH32rmm: 292 case X86::PUSH32rmr: 293 case X86::PUSHA32: 294 case X86::PUSHCS32: 295 case X86::PUSHDS32: 296 case X86::PUSHES32: 297 case X86::PUSHF32: 298 case X86::PUSHFS32: 299 case X86::PUSHGS32: 300 case X86::PUSHSS32: 301 case X86::PUSH32i: 302 return 4; 303 case X86::PUSH64i32: 304 case X86::PUSH64i8: 305 case X86::PUSH64r: 306 case X86::PUSH64rmm: 307 case X86::PUSH64rmr: 308 case X86::PUSHF64: 309 case X86::PUSHFS64: 310 case X86::PUSHGS64: 311 return 8; 312 } 313 return 0; 314 } 315 316 bool isSUB(const MCInst &Inst) const override { 317 return X86::isSUB(Inst.getOpcode()); 318 } 319 320 bool isLEA64r(const MCInst &Inst) const override { 321 return Inst.getOpcode() == X86::LEA64r; 322 } 323 324 bool isLeave(const MCInst &Inst) const override { 325 return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; 326 } 327 328 bool isMoveMem2Reg(const MCInst &Inst) const override { 329 switch (Inst.getOpcode()) { 330 case X86::MOV16rm: 331 case X86::MOV32rm: 332 case X86::MOV64rm: 333 return true; 334 } 335 return false; 336 } 337 338 bool isUnsupportedBranch(const MCInst &Inst) const override { 339 switch (Inst.getOpcode()) { 340 default: 341 return false; 342 case X86::LOOP: 343 case X86::LOOPE: 344 case X86::LOOPNE: 345 case X86::JECXZ: 346 case X86::JRCXZ: 347 return true; 348 } 349 } 350 351 bool mayLoad(const MCInst &Inst) const override { 352 if (isPop(Inst)) 353 return true; 354 355 int MemOpNo = getMemoryOperandNo(Inst); 356 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 357 358 if (MemOpNo == -1) 359 return false; 360 361 return MCII.mayLoad(); 362 } 363 364 bool mayStore(const MCInst &Inst) const override { 365 if (isPush(Inst)) 366 return true; 367 368 int MemOpNo = getMemoryOperandNo(Inst); 369 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 370 371 if (MemOpNo == -1) 372 return false; 373 374 return MCII.mayStore(); 375 } 376 377 bool isCleanRegXOR(const MCInst &Inst) const override { 378 switch (Inst.getOpcode()) { 379 case X86::XOR16rr: 380 case X86::XOR32rr: 381 case X86::XOR64rr: 382 break; 383 default: 384 return false; 385 } 386 return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg()); 387 } 388 389 bool isPacked(const MCInst &Inst) const override { 390 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 391 return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD; 392 } 393 394 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 395 switch (RelType) { 396 case ELF::R_X86_64_8: 397 case ELF::R_X86_64_16: 398 case ELF::R_X86_64_32: 399 case ELF::R_X86_64_32S: 400 case ELF::R_X86_64_64: 401 case ELF::R_X86_64_PC8: 402 case ELF::R_X86_64_PC32: 403 case ELF::R_X86_64_PC64: 404 case ELF::R_X86_64_GOTPC64: 405 case ELF::R_X86_64_GOTPCRELX: 406 case ELF::R_X86_64_REX_GOTPCRELX: 407 return true; 408 case ELF::R_X86_64_PLT32: 409 case ELF::R_X86_64_GOTPCREL: 410 case ELF::R_X86_64_TPOFF32: 411 case ELF::R_X86_64_GOTTPOFF: 412 return false; 413 default: 414 llvm_unreachable("Unexpected x86 relocation type in code"); 415 } 416 } 417 418 StringRef getTrapFillValue() const override { return StringRef("\314", 1); } 419 420 struct IndJmpMatcherFrag1 : MCInstMatcher { 421 std::unique_ptr<MCInstMatcher> Base; 422 std::unique_ptr<MCInstMatcher> Scale; 423 std::unique_ptr<MCInstMatcher> Index; 424 std::unique_ptr<MCInstMatcher> Offset; 425 426 IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 427 std::unique_ptr<MCInstMatcher> Scale, 428 std::unique_ptr<MCInstMatcher> Index, 429 std::unique_ptr<MCInstMatcher> Offset) 430 : Base(std::move(Base)), Scale(std::move(Scale)), 431 Index(std::move(Index)), Offset(std::move(Offset)) {} 432 433 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 434 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 435 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 436 return false; 437 438 if (CurInst->getOpcode() != X86::JMP64m) 439 return false; 440 441 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 442 if (MemOpNo == -1) 443 return false; 444 445 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 446 return false; 447 if (!Scale->match(MRI, MIB, this->InstrWindow, 448 MemOpNo + X86::AddrScaleAmt)) 449 return false; 450 if (!Index->match(MRI, MIB, this->InstrWindow, 451 MemOpNo + X86::AddrIndexReg)) 452 return false; 453 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 454 return false; 455 return true; 456 } 457 458 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 459 MIB.addAnnotation(*CurInst, Annotation, true); 460 Base->annotate(MIB, Annotation); 461 Scale->annotate(MIB, Annotation); 462 Index->annotate(MIB, Annotation); 463 Offset->annotate(MIB, Annotation); 464 } 465 }; 466 467 std::unique_ptr<MCInstMatcher> 468 matchIndJmp(std::unique_ptr<MCInstMatcher> Base, 469 std::unique_ptr<MCInstMatcher> Scale, 470 std::unique_ptr<MCInstMatcher> Index, 471 std::unique_ptr<MCInstMatcher> Offset) const override { 472 return std::unique_ptr<MCInstMatcher>( 473 new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), 474 std::move(Index), std::move(Offset))); 475 } 476 477 struct IndJmpMatcherFrag2 : MCInstMatcher { 478 std::unique_ptr<MCInstMatcher> Reg; 479 480 IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg) 481 : Reg(std::move(Reg)) {} 482 483 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 484 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 485 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 486 return false; 487 488 if (CurInst->getOpcode() != X86::JMP64r) 489 return false; 490 491 return Reg->match(MRI, MIB, this->InstrWindow, 0); 492 } 493 494 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 495 MIB.addAnnotation(*CurInst, Annotation, true); 496 Reg->annotate(MIB, Annotation); 497 } 498 }; 499 500 std::unique_ptr<MCInstMatcher> 501 matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override { 502 return std::unique_ptr<MCInstMatcher>( 503 new IndJmpMatcherFrag2(std::move(Target))); 504 } 505 506 struct LoadMatcherFrag1 : MCInstMatcher { 507 std::unique_ptr<MCInstMatcher> Base; 508 std::unique_ptr<MCInstMatcher> Scale; 509 std::unique_ptr<MCInstMatcher> Index; 510 std::unique_ptr<MCInstMatcher> Offset; 511 512 LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 513 std::unique_ptr<MCInstMatcher> Scale, 514 std::unique_ptr<MCInstMatcher> Index, 515 std::unique_ptr<MCInstMatcher> Offset) 516 : Base(std::move(Base)), Scale(std::move(Scale)), 517 Index(std::move(Index)), Offset(std::move(Offset)) {} 518 519 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 520 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 521 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 522 return false; 523 524 if (CurInst->getOpcode() != X86::MOV64rm && 525 CurInst->getOpcode() != X86::MOVSX64rm32) 526 return false; 527 528 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 529 if (MemOpNo == -1) 530 return false; 531 532 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 533 return false; 534 if (!Scale->match(MRI, MIB, this->InstrWindow, 535 MemOpNo + X86::AddrScaleAmt)) 536 return false; 537 if (!Index->match(MRI, MIB, this->InstrWindow, 538 MemOpNo + X86::AddrIndexReg)) 539 return false; 540 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 541 return false; 542 return true; 543 } 544 545 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 546 MIB.addAnnotation(*CurInst, Annotation, true); 547 Base->annotate(MIB, Annotation); 548 Scale->annotate(MIB, Annotation); 549 Index->annotate(MIB, Annotation); 550 Offset->annotate(MIB, Annotation); 551 } 552 }; 553 554 std::unique_ptr<MCInstMatcher> 555 matchLoad(std::unique_ptr<MCInstMatcher> Base, 556 std::unique_ptr<MCInstMatcher> Scale, 557 std::unique_ptr<MCInstMatcher> Index, 558 std::unique_ptr<MCInstMatcher> Offset) const override { 559 return std::unique_ptr<MCInstMatcher>( 560 new LoadMatcherFrag1(std::move(Base), std::move(Scale), 561 std::move(Index), std::move(Offset))); 562 } 563 564 struct AddMatcher : MCInstMatcher { 565 std::unique_ptr<MCInstMatcher> A; 566 std::unique_ptr<MCInstMatcher> B; 567 568 AddMatcher(std::unique_ptr<MCInstMatcher> A, 569 std::unique_ptr<MCInstMatcher> B) 570 : A(std::move(A)), B(std::move(B)) {} 571 572 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 573 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 574 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 575 return false; 576 577 if (CurInst->getOpcode() == X86::ADD64rr || 578 CurInst->getOpcode() == X86::ADD64rr_DB || 579 CurInst->getOpcode() == X86::ADD64rr_REV) { 580 if (!A->match(MRI, MIB, this->InstrWindow, 1)) { 581 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 582 return false; 583 return A->match(MRI, MIB, this->InstrWindow, 2); 584 } 585 586 if (B->match(MRI, MIB, this->InstrWindow, 2)) 587 return true; 588 589 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 590 return false; 591 return A->match(MRI, MIB, this->InstrWindow, 2); 592 } 593 594 return false; 595 } 596 597 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 598 MIB.addAnnotation(*CurInst, Annotation, true); 599 A->annotate(MIB, Annotation); 600 B->annotate(MIB, Annotation); 601 } 602 }; 603 604 std::unique_ptr<MCInstMatcher> 605 matchAdd(std::unique_ptr<MCInstMatcher> A, 606 std::unique_ptr<MCInstMatcher> B) const override { 607 return std::unique_ptr<MCInstMatcher>( 608 new AddMatcher(std::move(A), std::move(B))); 609 } 610 611 struct LEAMatcher : MCInstMatcher { 612 std::unique_ptr<MCInstMatcher> Target; 613 614 LEAMatcher(std::unique_ptr<MCInstMatcher> Target) 615 : Target(std::move(Target)) {} 616 617 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 618 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 619 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 620 return false; 621 622 if (CurInst->getOpcode() != X86::LEA64r) 623 return false; 624 625 if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || 626 CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != 627 X86::NoRegister || 628 (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != 629 X86::NoRegister && 630 CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) 631 return false; 632 633 return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); 634 } 635 636 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 637 MIB.addAnnotation(*CurInst, Annotation, true); 638 Target->annotate(MIB, Annotation); 639 } 640 }; 641 642 std::unique_ptr<MCInstMatcher> 643 matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override { 644 return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target))); 645 } 646 647 bool hasPCRelOperand(const MCInst &Inst) const override { 648 for (const MCOperand &Operand : Inst) 649 if (Operand.isReg() && Operand.getReg() == X86::RIP) 650 return true; 651 return false; 652 } 653 654 int getMemoryOperandNo(const MCInst &Inst) const override { 655 unsigned Opcode = Inst.getOpcode(); 656 const MCInstrDesc &Desc = Info->get(Opcode); 657 int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); 658 if (MemOpNo >= 0) 659 MemOpNo += X86II::getOperandBias(Desc); 660 return MemOpNo; 661 } 662 663 bool hasEVEXEncoding(const MCInst &Inst) const override { 664 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 665 return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; 666 } 667 668 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override { 669 const auto *I = Insts.begin(); 670 while (I != Insts.end() && isPrefix(*I)) 671 ++I; 672 if (I == Insts.end()) 673 return false; 674 675 const MCInst &FirstInst = *I; 676 ++I; 677 while (I != Insts.end() && isPrefix(*I)) 678 ++I; 679 if (I == Insts.end()) 680 return false; 681 const MCInst &SecondInst = *I; 682 683 if (!isConditionalBranch(SecondInst)) 684 return false; 685 // Cannot fuse if the first instruction uses RIP-relative memory. 686 if (hasPCRelOperand(FirstInst)) 687 return false; 688 689 const X86::FirstMacroFusionInstKind CmpKind = 690 X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode()); 691 if (CmpKind == X86::FirstMacroFusionInstKind::Invalid) 692 return false; 693 694 X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst)); 695 X86::SecondMacroFusionInstKind BranchKind = 696 X86::classifySecondCondCodeInMacroFusion(CC); 697 if (BranchKind == X86::SecondMacroFusionInstKind::Invalid) 698 return false; 699 return X86::isMacroFused(CmpKind, BranchKind); 700 } 701 702 std::optional<X86MemOperand> 703 evaluateX86MemoryOperand(const MCInst &Inst) const override { 704 int MemOpNo = getMemoryOperandNo(Inst); 705 if (MemOpNo < 0) 706 return std::nullopt; 707 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 708 709 if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst)) 710 return std::nullopt; 711 712 const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); 713 const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); 714 const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); 715 const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); 716 const MCOperand &Segment = 717 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); 718 719 // Make sure it is a well-formed memory operand. 720 if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || 721 (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) 722 return std::nullopt; 723 724 X86MemOperand MO; 725 MO.BaseRegNum = Base.getReg(); 726 MO.ScaleImm = Scale.getImm(); 727 MO.IndexRegNum = Index.getReg(); 728 MO.DispImm = Disp.isImm() ? Disp.getImm() : 0; 729 MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr; 730 MO.SegRegNum = Segment.getReg(); 731 return MO; 732 } 733 734 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 735 uint64_t Address, 736 uint64_t Size) const override { 737 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 738 if (!MO) 739 return false; 740 741 // Make sure it's a well-formed addressing we can statically evaluate. 742 if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) || 743 MO->IndexRegNum != X86::NoRegister || 744 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 745 return false; 746 747 Target = MO->DispImm; 748 if (MO->BaseRegNum == X86::RIP) { 749 assert(Size != 0 && "instruction size required in order to statically " 750 "evaluate RIP-relative address"); 751 Target += Address + Size; 752 } 753 return true; 754 } 755 756 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 757 int MemOpNo = getMemoryOperandNo(Inst); 758 if (MemOpNo < 0) 759 return Inst.end(); 760 return Inst.begin() + (MemOpNo + X86::AddrDisp); 761 } 762 763 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 764 MCOperand *OI = getMemOperandDisp(Inst); 765 if (OI == Inst.end()) 766 return false; 767 *OI = Operand; 768 return true; 769 } 770 771 /// Get the registers used as function parameters. 772 /// This function is specific to the x86_64 abi on Linux. 773 BitVector getRegsUsedAsParams() const override { 774 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 775 Regs |= getAliases(X86::RSI); 776 Regs |= getAliases(X86::RDI); 777 Regs |= getAliases(X86::RDX); 778 Regs |= getAliases(X86::RCX); 779 Regs |= getAliases(X86::R8); 780 Regs |= getAliases(X86::R9); 781 return Regs; 782 } 783 784 void getCalleeSavedRegs(BitVector &Regs) const override { 785 Regs |= getAliases(X86::RBX); 786 Regs |= getAliases(X86::RBP); 787 Regs |= getAliases(X86::R12); 788 Regs |= getAliases(X86::R13); 789 Regs |= getAliases(X86::R14); 790 Regs |= getAliases(X86::R15); 791 } 792 793 void getDefaultDefIn(BitVector &Regs) const override { 794 assert(Regs.size() >= RegInfo->getNumRegs() && 795 "The size of BitVector is less than RegInfo->getNumRegs()."); 796 Regs.set(X86::RAX); 797 Regs.set(X86::RCX); 798 Regs.set(X86::RDX); 799 Regs.set(X86::RSI); 800 Regs.set(X86::RDI); 801 Regs.set(X86::R8); 802 Regs.set(X86::R9); 803 Regs.set(X86::XMM0); 804 Regs.set(X86::XMM1); 805 Regs.set(X86::XMM2); 806 Regs.set(X86::XMM3); 807 Regs.set(X86::XMM4); 808 Regs.set(X86::XMM5); 809 Regs.set(X86::XMM6); 810 Regs.set(X86::XMM7); 811 } 812 813 void getDefaultLiveOut(BitVector &Regs) const override { 814 assert(Regs.size() >= RegInfo->getNumRegs() && 815 "The size of BitVector is less than RegInfo->getNumRegs()."); 816 Regs |= getAliases(X86::RAX); 817 Regs |= getAliases(X86::RDX); 818 Regs |= getAliases(X86::RCX); 819 Regs |= getAliases(X86::XMM0); 820 Regs |= getAliases(X86::XMM1); 821 } 822 823 void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { 824 if (IncludeAlias) { 825 Regs |= getAliases(X86::RAX); 826 Regs |= getAliases(X86::RBX); 827 Regs |= getAliases(X86::RBP); 828 Regs |= getAliases(X86::RSI); 829 Regs |= getAliases(X86::RDI); 830 Regs |= getAliases(X86::RDX); 831 Regs |= getAliases(X86::RCX); 832 Regs |= getAliases(X86::R8); 833 Regs |= getAliases(X86::R9); 834 Regs |= getAliases(X86::R10); 835 Regs |= getAliases(X86::R11); 836 Regs |= getAliases(X86::R12); 837 Regs |= getAliases(X86::R13); 838 Regs |= getAliases(X86::R14); 839 Regs |= getAliases(X86::R15); 840 return; 841 } 842 Regs.set(X86::RAX); 843 Regs.set(X86::RBX); 844 Regs.set(X86::RBP); 845 Regs.set(X86::RSI); 846 Regs.set(X86::RDI); 847 Regs.set(X86::RDX); 848 Regs.set(X86::RCX); 849 Regs.set(X86::R8); 850 Regs.set(X86::R9); 851 Regs.set(X86::R10); 852 Regs.set(X86::R11); 853 Regs.set(X86::R12); 854 Regs.set(X86::R13); 855 Regs.set(X86::R14); 856 Regs.set(X86::R15); 857 } 858 859 void getClassicGPRegs(BitVector &Regs) const override { 860 Regs |= getAliases(X86::RAX); 861 Regs |= getAliases(X86::RBX); 862 Regs |= getAliases(X86::RBP); 863 Regs |= getAliases(X86::RSI); 864 Regs |= getAliases(X86::RDI); 865 Regs |= getAliases(X86::RDX); 866 Regs |= getAliases(X86::RCX); 867 } 868 869 void getRepRegs(BitVector &Regs) const override { 870 Regs |= getAliases(X86::RCX); 871 } 872 873 MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { 874 Reg = getX86SubSuperRegister(Reg, Size * 8); 875 assert((Reg != X86::NoRegister) && "Invalid register"); 876 return Reg; 877 } 878 879 bool isUpper8BitReg(MCPhysReg Reg) const override { 880 switch (Reg) { 881 case X86::AH: 882 case X86::BH: 883 case X86::CH: 884 case X86::DH: 885 return true; 886 default: 887 return false; 888 } 889 } 890 891 bool cannotUseREX(const MCInst &Inst) const override { 892 switch (Inst.getOpcode()) { 893 case X86::MOV8mr_NOREX: 894 case X86::MOV8rm_NOREX: 895 case X86::MOV8rr_NOREX: 896 case X86::MOVSX32rm8_NOREX: 897 case X86::MOVSX32rr8_NOREX: 898 case X86::MOVZX32rm8_NOREX: 899 case X86::MOVZX32rr8_NOREX: 900 case X86::MOV8mr: 901 case X86::MOV8rm: 902 case X86::MOV8rr: 903 case X86::MOVSX32rm8: 904 case X86::MOVSX32rr8: 905 case X86::MOVZX32rm8: 906 case X86::MOVZX32rr8: 907 case X86::TEST8ri: 908 for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) { 909 if (!Operand.isReg()) 910 continue; 911 if (isUpper8BitReg(Operand.getReg())) 912 return true; 913 } 914 [[fallthrough]]; 915 default: 916 return false; 917 } 918 } 919 920 static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) { 921 using namespace llvm::X86; 922 int OpType = getOperandType(Inst.getOpcode(), MemOpNo); 923 return getMemOperandSize(OpType) / 8; 924 } 925 926 /// Classifying a stack access as *not* "SIMPLE" here means we don't know how 927 /// to change this instruction memory access. It will disable any changes to 928 /// the stack layout, so we can't do the most aggressive form of shrink 929 /// wrapping. We must do so in a way that keeps the original stack layout. 930 /// Otherwise you need to adjust the offset of all instructions accessing the 931 /// stack: we can't do that anymore because there is one instruction that is 932 /// not simple. There are other implications as well. We have heuristics to 933 /// detect when a register is callee-saved and thus eligible for shrink 934 /// wrapping. If you are restoring a register using a non-simple stack access, 935 /// then it is classified as NOT callee-saved, and it disables shrink wrapping 936 /// for *that* register (but not for others). 937 /// 938 /// Classifying a stack access as "size 0" or detecting an indexed memory 939 /// access (to address a vector, for example) here means we know there is a 940 /// stack access, but we can't quite understand how wide is the access in 941 /// bytes. This is very serious because we can't understand how memory 942 /// accesses alias with each other for this function. This will essentially 943 /// disable not only shrink wrapping but all frame analysis, it will fail it 944 /// as "we don't understand this function and we give up on it". 945 bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, 946 bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, 947 uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size, 948 bool &IsSimple, bool &IsIndexed) const override { 949 // Detect simple push/pop cases first 950 if (int Sz = getPushSize(Inst)) { 951 IsLoad = false; 952 IsStore = true; 953 IsStoreFromReg = true; 954 StackPtrReg = X86::RSP; 955 StackOffset = -Sz; 956 Size = Sz; 957 IsSimple = true; 958 if (Inst.getOperand(0).isImm()) 959 SrcImm = Inst.getOperand(0).getImm(); 960 else if (Inst.getOperand(0).isReg()) 961 Reg = Inst.getOperand(0).getReg(); 962 else 963 IsSimple = false; 964 965 return true; 966 } 967 if (int Sz = getPopSize(Inst)) { 968 IsLoad = true; 969 IsStore = false; 970 if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) { 971 IsSimple = false; 972 } else { 973 Reg = Inst.getOperand(0).getReg(); 974 IsSimple = true; 975 } 976 StackPtrReg = X86::RSP; 977 StackOffset = 0; 978 Size = Sz; 979 return true; 980 } 981 982 struct InstInfo { 983 // Size in bytes that Inst loads from memory. 984 uint8_t DataSize; 985 bool IsLoad; 986 bool IsStore; 987 bool StoreFromReg; 988 bool Simple; 989 }; 990 991 InstInfo I; 992 int MemOpNo = getMemoryOperandNo(Inst); 993 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 994 // If it is not dealing with a memory operand, we discard it 995 if (MemOpNo == -1 || MCII.isCall()) 996 return false; 997 998 switch (Inst.getOpcode()) { 999 default: { 1000 bool IsLoad = MCII.mayLoad(); 1001 bool IsStore = MCII.mayStore(); 1002 // Is it LEA? (deals with memory but is not loading nor storing) 1003 if (!IsLoad && !IsStore) { 1004 I = {0, IsLoad, IsStore, false, false}; 1005 break; 1006 } 1007 uint8_t Sz = getMemDataSize(Inst, MemOpNo); 1008 I = {Sz, IsLoad, IsStore, false, false}; 1009 break; 1010 } 1011 // Report simple stack accesses 1012 case X86::MOV8rm: I = {1, true, false, false, true}; break; 1013 case X86::MOV16rm: I = {2, true, false, false, true}; break; 1014 case X86::MOV32rm: I = {4, true, false, false, true}; break; 1015 case X86::MOV64rm: I = {8, true, false, false, true}; break; 1016 case X86::MOV8mr: I = {1, false, true, true, true}; break; 1017 case X86::MOV16mr: I = {2, false, true, true, true}; break; 1018 case X86::MOV32mr: I = {4, false, true, true, true}; break; 1019 case X86::MOV64mr: I = {8, false, true, true, true}; break; 1020 case X86::MOV8mi: I = {1, false, true, false, true}; break; 1021 case X86::MOV16mi: I = {2, false, true, false, true}; break; 1022 case X86::MOV32mi: I = {4, false, true, false, true}; break; 1023 } // end switch (Inst.getOpcode()) 1024 1025 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1026 if (!MO) { 1027 LLVM_DEBUG(dbgs() << "Evaluate failed on "); 1028 LLVM_DEBUG(Inst.dump()); 1029 return false; 1030 } 1031 1032 // Make sure it's a stack access 1033 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) 1034 return false; 1035 1036 IsLoad = I.IsLoad; 1037 IsStore = I.IsStore; 1038 IsStoreFromReg = I.StoreFromReg; 1039 Size = I.DataSize; 1040 IsSimple = I.Simple; 1041 StackPtrReg = MO->BaseRegNum; 1042 StackOffset = MO->DispImm; 1043 IsIndexed = 1044 MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister; 1045 1046 if (!I.Simple) 1047 return true; 1048 1049 // Retrieve related register in simple MOV from/to stack operations. 1050 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 1051 if (I.IsLoad) { 1052 MCOperand RegOpnd = Inst.getOperand(0); 1053 assert(RegOpnd.isReg() && "unexpected destination operand"); 1054 Reg = RegOpnd.getReg(); 1055 } else if (I.IsStore) { 1056 MCOperand SrcOpnd = 1057 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1058 if (I.StoreFromReg) { 1059 assert(SrcOpnd.isReg() && "unexpected source operand"); 1060 Reg = SrcOpnd.getReg(); 1061 } else { 1062 assert(SrcOpnd.isImm() && "unexpected source operand"); 1063 SrcImm = SrcOpnd.getImm(); 1064 } 1065 } 1066 1067 return true; 1068 } 1069 1070 void changeToPushOrPop(MCInst &Inst) const override { 1071 assert(!isPush(Inst) && !isPop(Inst)); 1072 1073 struct InstInfo { 1074 // Size in bytes that Inst loads from memory. 1075 uint8_t DataSize; 1076 bool IsLoad; 1077 bool StoreFromReg; 1078 }; 1079 1080 InstInfo I; 1081 switch (Inst.getOpcode()) { 1082 default: { 1083 llvm_unreachable("Unhandled opcode"); 1084 return; 1085 } 1086 case X86::MOV16rm: I = {2, true, false}; break; 1087 case X86::MOV32rm: I = {4, true, false}; break; 1088 case X86::MOV64rm: I = {8, true, false}; break; 1089 case X86::MOV16mr: I = {2, false, true}; break; 1090 case X86::MOV32mr: I = {4, false, true}; break; 1091 case X86::MOV64mr: I = {8, false, true}; break; 1092 case X86::MOV16mi: I = {2, false, false}; break; 1093 case X86::MOV32mi: I = {4, false, false}; break; 1094 } // end switch (Inst.getOpcode()) 1095 1096 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1097 if (!MO) { 1098 llvm_unreachable("Evaluate failed"); 1099 return; 1100 } 1101 // Make sure it's a stack access 1102 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) { 1103 llvm_unreachable("Not a stack access"); 1104 return; 1105 } 1106 1107 unsigned MemOpOffset = getMemoryOperandNo(Inst); 1108 unsigned NewOpcode = 0; 1109 if (I.IsLoad) { 1110 switch (I.DataSize) { 1111 case 2: NewOpcode = X86::POP16r; break; 1112 case 4: NewOpcode = X86::POP32r; break; 1113 case 8: NewOpcode = X86::POP64r; break; 1114 default: 1115 llvm_unreachable("Unexpected size"); 1116 } 1117 unsigned RegOpndNum = Inst.getOperand(0).getReg(); 1118 Inst.clear(); 1119 Inst.setOpcode(NewOpcode); 1120 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1121 } else { 1122 MCOperand SrcOpnd = 1123 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1124 if (I.StoreFromReg) { 1125 switch (I.DataSize) { 1126 case 2: NewOpcode = X86::PUSH16r; break; 1127 case 4: NewOpcode = X86::PUSH32r; break; 1128 case 8: NewOpcode = X86::PUSH64r; break; 1129 default: 1130 llvm_unreachable("Unexpected size"); 1131 } 1132 assert(SrcOpnd.isReg() && "Unexpected source operand"); 1133 unsigned RegOpndNum = SrcOpnd.getReg(); 1134 Inst.clear(); 1135 Inst.setOpcode(NewOpcode); 1136 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1137 } else { 1138 switch (I.DataSize) { 1139 case 2: NewOpcode = X86::PUSH16i8; break; 1140 case 4: NewOpcode = X86::PUSH32i8; break; 1141 case 8: NewOpcode = X86::PUSH64i32; break; 1142 default: 1143 llvm_unreachable("Unexpected size"); 1144 } 1145 assert(SrcOpnd.isImm() && "Unexpected source operand"); 1146 int64_t SrcImm = SrcOpnd.getImm(); 1147 Inst.clear(); 1148 Inst.setOpcode(NewOpcode); 1149 Inst.addOperand(MCOperand::createImm(SrcImm)); 1150 } 1151 } 1152 } 1153 1154 bool isStackAdjustment(const MCInst &Inst) const override { 1155 switch (Inst.getOpcode()) { 1156 default: 1157 return false; 1158 case X86::SUB64ri32: 1159 case X86::SUB64ri8: 1160 case X86::ADD64ri32: 1161 case X86::ADD64ri8: 1162 case X86::LEA64r: 1163 break; 1164 } 1165 1166 return any_of(defOperands(Inst), [](const MCOperand &Op) { 1167 return Op.isReg() && Op.getReg() == X86::RSP; 1168 }); 1169 } 1170 1171 bool 1172 evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output, 1173 std::pair<MCPhysReg, int64_t> Input1, 1174 std::pair<MCPhysReg, int64_t> Input2) const override { 1175 1176 auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> { 1177 if (Reg == Input1.first) 1178 return Input1.second; 1179 if (Reg == Input2.first) 1180 return Input2.second; 1181 return make_error_code(errc::result_out_of_range); 1182 }; 1183 1184 switch (Inst.getOpcode()) { 1185 default: 1186 return false; 1187 1188 case X86::SUB64ri32: 1189 case X86::SUB64ri8: 1190 if (!Inst.getOperand(2).isImm()) 1191 return false; 1192 if (ErrorOr<int64_t> InputVal = 1193 getOperandVal(Inst.getOperand(1).getReg())) 1194 Output = *InputVal - Inst.getOperand(2).getImm(); 1195 else 1196 return false; 1197 break; 1198 case X86::ADD64ri32: 1199 case X86::ADD64ri8: 1200 if (!Inst.getOperand(2).isImm()) 1201 return false; 1202 if (ErrorOr<int64_t> InputVal = 1203 getOperandVal(Inst.getOperand(1).getReg())) 1204 Output = *InputVal + Inst.getOperand(2).getImm(); 1205 else 1206 return false; 1207 break; 1208 case X86::ADD64i32: 1209 if (!Inst.getOperand(0).isImm()) 1210 return false; 1211 if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) 1212 Output = *InputVal + Inst.getOperand(0).getImm(); 1213 else 1214 return false; 1215 break; 1216 1217 case X86::LEA64r: { 1218 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1219 if (!MO) 1220 return false; 1221 1222 if (MO->BaseRegNum == X86::NoRegister || 1223 MO->IndexRegNum != X86::NoRegister || 1224 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 1225 return false; 1226 1227 if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum)) 1228 Output = *InputVal + MO->DispImm; 1229 else 1230 return false; 1231 1232 break; 1233 } 1234 } 1235 return true; 1236 } 1237 1238 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 1239 MCPhysReg &To) const override { 1240 switch (Inst.getOpcode()) { 1241 default: 1242 return false; 1243 case X86::LEAVE: 1244 case X86::LEAVE64: 1245 To = getStackPointer(); 1246 From = getFramePointer(); 1247 return true; 1248 case X86::MOV64rr: 1249 To = Inst.getOperand(0).getReg(); 1250 From = Inst.getOperand(1).getReg(); 1251 return true; 1252 } 1253 } 1254 1255 MCPhysReg getStackPointer() const override { return X86::RSP; } 1256 MCPhysReg getFramePointer() const override { return X86::RBP; } 1257 MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } 1258 1259 bool escapesVariable(const MCInst &Inst, 1260 bool HasFramePointer) const override { 1261 int MemOpNo = getMemoryOperandNo(Inst); 1262 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 1263 const unsigned NumDefs = MCII.getNumDefs(); 1264 static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |= 1265 getAliases(X86::RBP)); 1266 static BitVector SPAliases(getAliases(X86::RSP)); 1267 1268 // FIXME: PUSH can be technically a leak, but let's ignore this for now 1269 // because a lot of harmless prologue code will spill SP to the stack. 1270 // Unless push is clearly pushing an object address to the stack as 1271 // demonstrated by having a MemOp. 1272 bool IsPush = isPush(Inst); 1273 if (IsPush && MemOpNo == -1) 1274 return false; 1275 1276 // We use this to detect LEA (has memop but does not access mem) 1277 bool AccessMem = MCII.mayLoad() || MCII.mayStore(); 1278 bool DoesLeak = false; 1279 for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) { 1280 // Ignore if SP/BP is used to dereference memory -- that's fine 1281 if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && 1282 I <= MemOpNo + 5) 1283 continue; 1284 // Ignore if someone is writing to SP/BP 1285 if (I < static_cast<int>(NumDefs)) 1286 continue; 1287 1288 const MCOperand &Operand = Inst.getOperand(I); 1289 if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { 1290 DoesLeak = true; 1291 break; 1292 } 1293 if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { 1294 DoesLeak = true; 1295 break; 1296 } 1297 } 1298 1299 // If potential leak, check if it is not just writing to itself/sp/bp 1300 if (DoesLeak) { 1301 DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) { 1302 assert(Operand.isReg()); 1303 MCPhysReg Reg = Operand.getReg(); 1304 return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg]; 1305 }); 1306 } 1307 return DoesLeak; 1308 } 1309 1310 bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { 1311 unsigned ImmOpNo = -1U; 1312 int MemOpNo = getMemoryOperandNo(Inst); 1313 if (MemOpNo != -1) 1314 ImmOpNo = MemOpNo + X86::AddrDisp; 1315 else 1316 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1317 ++Index) 1318 if (Inst.getOperand(Index).isImm()) 1319 ImmOpNo = Index; 1320 if (ImmOpNo == -1U) 1321 return false; 1322 1323 MCOperand &Operand = Inst.getOperand(ImmOpNo); 1324 Amt += Operand.getImm(); 1325 Operand.setImm(Amt); 1326 // Check for the need for relaxation 1327 if (int64_t(Amt) == int64_t(int8_t(Amt))) 1328 return true; 1329 1330 // Relax instruction 1331 switch (Inst.getOpcode()) { 1332 case X86::SUB64ri8: 1333 Inst.setOpcode(X86::SUB64ri32); 1334 break; 1335 case X86::ADD64ri8: 1336 Inst.setOpcode(X86::ADD64ri32); 1337 break; 1338 default: 1339 // No need for relaxation 1340 break; 1341 } 1342 return true; 1343 } 1344 1345 /// TODO: this implementation currently works for the most common opcodes that 1346 /// load from memory. It can be extended to work with memory store opcodes as 1347 /// well as more memory load opcodes. 1348 bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, 1349 uint64_t Offset) const override { 1350 enum CheckSignExt : uint8_t { 1351 NOCHECK = 0, 1352 CHECK8, 1353 CHECK32, 1354 }; 1355 1356 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 1357 struct InstInfo { 1358 // Size in bytes that Inst loads from memory. 1359 uint8_t DataSize; 1360 1361 // True when the target operand has to be duplicated because the opcode 1362 // expects a LHS operand. 1363 bool HasLHS; 1364 1365 // List of checks and corresponding opcodes to be used. We try to use the 1366 // smallest possible immediate value when various sizes are available, 1367 // hence we may need to check whether a larger constant fits in a smaller 1368 // immediate. 1369 CheckList Checks; 1370 }; 1371 1372 InstInfo I; 1373 1374 switch (Inst.getOpcode()) { 1375 default: { 1376 switch (getPopSize(Inst)) { 1377 case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1378 case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1379 case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1380 {NOCHECK, X86::MOV64rm}}}; break; 1381 default: return false; 1382 } 1383 break; 1384 } 1385 1386 // MOV 1387 case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 1388 case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1389 case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1390 case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1391 {NOCHECK, X86::MOV64rm}}}; break; 1392 1393 // MOVZX 1394 case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 1395 case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 1396 case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 1397 1398 // CMP 1399 case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 1400 case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, 1401 {NOCHECK, X86::CMP16ri}}}; break; 1402 case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, 1403 {NOCHECK, X86::CMP32ri}}}; break; 1404 case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, 1405 {CHECK32, X86::CMP64ri32}, 1406 {NOCHECK, X86::CMP64rm}}}; break; 1407 1408 // TEST 1409 case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 1410 case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 1411 case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 1412 case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 1413 {NOCHECK, X86::TEST64mr}}}; break; 1414 1415 // ADD 1416 case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 1417 case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, 1418 {NOCHECK, X86::ADD16ri}}}; break; 1419 case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, 1420 {NOCHECK, X86::ADD32ri}}}; break; 1421 case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, 1422 {CHECK32, X86::ADD64ri32}, 1423 {NOCHECK, X86::ADD64rm}}}; break; 1424 1425 // SUB 1426 case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 1427 case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, 1428 {NOCHECK, X86::SUB16ri}}}; break; 1429 case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, 1430 {NOCHECK, X86::SUB32ri}}}; break; 1431 case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, 1432 {CHECK32, X86::SUB64ri32}, 1433 {NOCHECK, X86::SUB64rm}}}; break; 1434 1435 // AND 1436 case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 1437 case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, 1438 {NOCHECK, X86::AND16ri}}}; break; 1439 case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, 1440 {NOCHECK, X86::AND32ri}}}; break; 1441 case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, 1442 {CHECK32, X86::AND64ri32}, 1443 {NOCHECK, X86::AND64rm}}}; break; 1444 1445 // OR 1446 case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 1447 case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, 1448 {NOCHECK, X86::OR16ri}}}; break; 1449 case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, 1450 {NOCHECK, X86::OR32ri}}}; break; 1451 case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, 1452 {CHECK32, X86::OR64ri32}, 1453 {NOCHECK, X86::OR64rm}}}; break; 1454 1455 // XOR 1456 case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 1457 case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, 1458 {NOCHECK, X86::XOR16ri}}}; break; 1459 case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, 1460 {NOCHECK, X86::XOR32ri}}}; break; 1461 case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, 1462 {CHECK32, X86::XOR64ri32}, 1463 {NOCHECK, X86::XOR64rm}}}; break; 1464 } 1465 1466 // Compute the immediate value. 1467 assert(Offset + I.DataSize <= ConstantData.size() && 1468 "invalid offset for given constant data"); 1469 int64_t ImmVal = 1470 DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize); 1471 1472 // Compute the new opcode. 1473 unsigned NewOpcode = 0; 1474 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 1475 NewOpcode = Check.second; 1476 if (Check.first == NOCHECK) 1477 break; 1478 if (Check.first == CHECK8 && isInt<8>(ImmVal)) 1479 break; 1480 if (Check.first == CHECK32 && isInt<32>(ImmVal)) 1481 break; 1482 } 1483 if (NewOpcode == Inst.getOpcode()) 1484 return false; 1485 1486 // Modify the instruction. 1487 MCOperand ImmOp = MCOperand::createImm(ImmVal); 1488 uint32_t TargetOpNum = 0; 1489 // Test instruction does not follow the regular pattern of putting the 1490 // memory reference of a load (5 MCOperands) last in the list of operands. 1491 // Since it is not modifying the register operand, it is not treated as 1492 // a destination operand and it is not the first operand as it is in the 1493 // other instructions we treat here. 1494 if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri || 1495 NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32) 1496 TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; 1497 1498 MCOperand TargetOp = Inst.getOperand(TargetOpNum); 1499 Inst.clear(); 1500 Inst.setOpcode(NewOpcode); 1501 Inst.addOperand(TargetOp); 1502 if (I.HasLHS) 1503 Inst.addOperand(TargetOp); 1504 Inst.addOperand(ImmOp); 1505 1506 return true; 1507 } 1508 1509 /// TODO: this implementation currently works for the most common opcodes that 1510 /// load from memory. It can be extended to work with memory store opcodes as 1511 /// well as more memory load opcodes. 1512 bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { 1513 unsigned NewOpcode; 1514 1515 switch (Inst.getOpcode()) { 1516 default: { 1517 switch (getPopSize(Inst)) { 1518 case 2: NewOpcode = X86::MOV16rr; break; 1519 case 4: NewOpcode = X86::MOV32rr; break; 1520 case 8: NewOpcode = X86::MOV64rr; break; 1521 default: return false; 1522 } 1523 break; 1524 } 1525 1526 // MOV 1527 case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; 1528 case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; 1529 case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; 1530 case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; 1531 } 1532 1533 // Modify the instruction. 1534 MCOperand RegOp = MCOperand::createReg(RegNum); 1535 MCOperand TargetOp = Inst.getOperand(0); 1536 Inst.clear(); 1537 Inst.setOpcode(NewOpcode); 1538 Inst.addOperand(TargetOp); 1539 Inst.addOperand(RegOp); 1540 1541 return true; 1542 } 1543 1544 bool isRedundantMove(const MCInst &Inst) const override { 1545 switch (Inst.getOpcode()) { 1546 default: 1547 return false; 1548 1549 // MOV 1550 case X86::MOV8rr: 1551 case X86::MOV16rr: 1552 case X86::MOV32rr: 1553 case X86::MOV64rr: 1554 break; 1555 } 1556 1557 assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); 1558 return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); 1559 } 1560 1561 bool requiresAlignedAddress(const MCInst &Inst) const override { 1562 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 1563 for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { 1564 const MCOperandInfo &Op = Desc.operands()[I]; 1565 if (Op.OperandType != MCOI::OPERAND_REGISTER) 1566 continue; 1567 if (Op.RegClass == X86::VR128RegClassID) 1568 return true; 1569 } 1570 return false; 1571 } 1572 1573 bool convertJmpToTailCall(MCInst &Inst) override { 1574 if (isTailCall(Inst)) 1575 return false; 1576 1577 int NewOpcode; 1578 switch (Inst.getOpcode()) { 1579 default: 1580 return false; 1581 case X86::JMP_1: 1582 case X86::JMP_2: 1583 case X86::JMP_4: 1584 NewOpcode = X86::JMP_4; 1585 break; 1586 case X86::JMP16m: 1587 case X86::JMP32m: 1588 case X86::JMP64m: 1589 NewOpcode = X86::JMP32m; 1590 break; 1591 case X86::JMP16r: 1592 case X86::JMP32r: 1593 case X86::JMP64r: 1594 NewOpcode = X86::JMP32r; 1595 break; 1596 } 1597 1598 Inst.setOpcode(NewOpcode); 1599 setTailCall(Inst); 1600 return true; 1601 } 1602 1603 bool convertTailCallToJmp(MCInst &Inst) override { 1604 int NewOpcode; 1605 switch (Inst.getOpcode()) { 1606 default: 1607 return false; 1608 case X86::JMP_4: 1609 NewOpcode = X86::JMP_1; 1610 break; 1611 case X86::JMP32m: 1612 NewOpcode = X86::JMP64m; 1613 break; 1614 case X86::JMP32r: 1615 NewOpcode = X86::JMP64r; 1616 break; 1617 } 1618 1619 Inst.setOpcode(NewOpcode); 1620 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1621 clearOffset(Inst); 1622 return true; 1623 } 1624 1625 bool convertTailCallToCall(MCInst &Inst) override { 1626 int NewOpcode; 1627 switch (Inst.getOpcode()) { 1628 default: 1629 return false; 1630 case X86::JMP_4: 1631 NewOpcode = X86::CALL64pcrel32; 1632 break; 1633 case X86::JMP32m: 1634 NewOpcode = X86::CALL64m; 1635 break; 1636 case X86::JMP32r: 1637 NewOpcode = X86::CALL64r; 1638 break; 1639 } 1640 1641 Inst.setOpcode(NewOpcode); 1642 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1643 return true; 1644 } 1645 1646 bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation, 1647 MCContext *Ctx) override { 1648 assert((Inst.getOpcode() == X86::CALL64pcrel32 || 1649 (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) && 1650 "64-bit direct (tail) call instruction expected"); 1651 const auto NewOpcode = 1652 (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m; 1653 Inst.setOpcode(NewOpcode); 1654 1655 // Replace the first operand and preserve auxiliary operands of 1656 // the instruction. 1657 Inst.erase(Inst.begin()); 1658 Inst.insert(Inst.begin(), 1659 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 1660 Inst.insert(Inst.begin(), 1661 MCOperand::createExpr( // Displacement 1662 MCSymbolRefExpr::create(TargetLocation, 1663 MCSymbolRefExpr::VK_None, *Ctx))); 1664 Inst.insert(Inst.begin(), 1665 MCOperand::createReg(X86::NoRegister)); // IndexReg 1666 Inst.insert(Inst.begin(), 1667 MCOperand::createImm(1)); // ScaleAmt 1668 Inst.insert(Inst.begin(), 1669 MCOperand::createReg(X86::RIP)); // BaseReg 1670 1671 return true; 1672 } 1673 1674 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1675 bool IsTailCall = isTailCall(Inst); 1676 if (IsTailCall) 1677 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1678 if (Inst.getOpcode() == X86::CALL64m || 1679 (Inst.getOpcode() == X86::JMP32m && IsTailCall)) { 1680 Inst.setOpcode(X86::MOV64rm); 1681 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1682 return; 1683 } 1684 if (Inst.getOpcode() == X86::CALL64r || 1685 (Inst.getOpcode() == X86::JMP32r && IsTailCall)) { 1686 Inst.setOpcode(X86::MOV64rr); 1687 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1688 return; 1689 } 1690 LLVM_DEBUG(Inst.dump()); 1691 llvm_unreachable("not implemented"); 1692 } 1693 1694 bool shortenInstruction(MCInst &Inst, 1695 const MCSubtargetInfo &STI) const override { 1696 unsigned OldOpcode = Inst.getOpcode(); 1697 unsigned NewOpcode = OldOpcode; 1698 1699 int MemOpNo = getMemoryOperandNo(Inst); 1700 1701 // Check and remove redundant Address-Size override prefix. 1702 if (opts::X86StripRedundantAddressSize) { 1703 uint64_t TSFlags = Info->get(OldOpcode).TSFlags; 1704 unsigned Flags = Inst.getFlags(); 1705 1706 if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) && 1707 Flags & X86::IP_HAS_AD_SIZE) 1708 Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE); 1709 } 1710 1711 // Check and remove EIZ/RIZ. These cases represent ambiguous cases where 1712 // SIB byte is present, but no index is used and modrm alone should have 1713 // been enough. Converting to NoRegister effectively removes the SIB byte. 1714 if (MemOpNo >= 0) { 1715 MCOperand &IndexOp = 1716 Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg); 1717 if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ) 1718 IndexOp = MCOperand::createReg(X86::NoRegister); 1719 } 1720 1721 if (isBranch(Inst)) { 1722 NewOpcode = getShortBranchOpcode(OldOpcode); 1723 } else if (OldOpcode == X86::MOV64ri) { 1724 if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1725 const int64_t Imm = 1726 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1727 if (int64_t(Imm) == int64_t(int32_t(Imm))) 1728 NewOpcode = X86::MOV64ri32; 1729 } 1730 } else { 1731 // If it's arithmetic instruction check if signed operand fits in 1 byte. 1732 const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(OldOpcode); 1733 if (ShortOpcode != OldOpcode && 1734 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1735 int64_t Imm = 1736 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1737 if (int64_t(Imm) == int64_t(int8_t(Imm))) 1738 NewOpcode = ShortOpcode; 1739 } 1740 } 1741 1742 if (NewOpcode == OldOpcode) 1743 return false; 1744 1745 Inst.setOpcode(NewOpcode); 1746 return true; 1747 } 1748 1749 bool 1750 convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp, 1751 bool AllowBasePtrStackMemOp) const override { 1752 // - Register-register moves are OK 1753 // - Stores are filtered out by opcode (no store CMOV) 1754 // - Non-stack loads are prohibited (generally unsafe) 1755 // - Stack loads are OK if AllowStackMemOp is true 1756 // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true 1757 if (mayLoad(Inst)) { 1758 // If stack memory operands are not allowed, no loads are allowed 1759 if (!AllowStackMemOp) 1760 return false; 1761 1762 // If stack memory operands are allowed, check if it's a load from stack 1763 bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed; 1764 MCPhysReg Reg; 1765 int32_t SrcImm; 1766 uint16_t StackPtrReg; 1767 int64_t StackOffset; 1768 uint8_t Size; 1769 bool IsStackAccess = 1770 isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, 1771 StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); 1772 // Prohibit non-stack-based loads 1773 if (!IsStackAccess) 1774 return false; 1775 // If stack memory operands are allowed, check if it's RBP-based 1776 if (!AllowBasePtrStackMemOp && 1777 RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg)) 1778 return false; 1779 } 1780 1781 unsigned NewOpcode = 0; 1782 switch (Inst.getOpcode()) { 1783 case X86::MOV16rr: 1784 NewOpcode = X86::CMOV16rr; 1785 break; 1786 case X86::MOV16rm: 1787 NewOpcode = X86::CMOV16rm; 1788 break; 1789 case X86::MOV32rr: 1790 NewOpcode = X86::CMOV32rr; 1791 break; 1792 case X86::MOV32rm: 1793 NewOpcode = X86::CMOV32rm; 1794 break; 1795 case X86::MOV64rr: 1796 NewOpcode = X86::CMOV64rr; 1797 break; 1798 case X86::MOV64rm: 1799 NewOpcode = X86::CMOV64rm; 1800 break; 1801 default: 1802 return false; 1803 } 1804 Inst.setOpcode(NewOpcode); 1805 // Insert CC at the end of prime operands, before annotations 1806 Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst), 1807 MCOperand::createImm(CC)); 1808 // CMOV is a 3-operand MCInst, so duplicate the destination as src1 1809 Inst.insert(Inst.begin(), Inst.getOperand(0)); 1810 return true; 1811 } 1812 1813 bool lowerTailCall(MCInst &Inst) override { 1814 if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) { 1815 Inst.setOpcode(X86::JMP_1); 1816 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1817 return true; 1818 } 1819 return false; 1820 } 1821 1822 const MCSymbol *getTargetSymbol(const MCInst &Inst, 1823 unsigned OpNum = 0) const override { 1824 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 1825 return nullptr; 1826 1827 const MCOperand &Op = Inst.getOperand(OpNum); 1828 if (!Op.isExpr()) 1829 return nullptr; 1830 1831 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 1832 if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) 1833 return nullptr; 1834 1835 return &SymExpr->getSymbol(); 1836 } 1837 1838 // This is the same as the base class, but since we are overriding one of 1839 // getTargetSymbol's signatures above, we need to override all of them. 1840 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 1841 return &cast<const MCSymbolRefExpr>(Expr)->getSymbol(); 1842 } 1843 1844 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1845 const MCSymbol *&TBB, const MCSymbol *&FBB, 1846 MCInst *&CondBranch, 1847 MCInst *&UncondBranch) const override { 1848 auto I = End; 1849 1850 // Bottom-up analysis 1851 while (I != Begin) { 1852 --I; 1853 1854 // Ignore nops and CFIs 1855 if (isPseudo(*I)) 1856 continue; 1857 1858 // Stop when we find the first non-terminator 1859 if (!isTerminator(*I)) 1860 break; 1861 1862 if (!isBranch(*I)) 1863 break; 1864 1865 // Handle unconditional branches. 1866 if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 || 1867 I->getOpcode() == X86::JMP_4) && 1868 !isTailCall(*I)) { 1869 // If any code was seen after this unconditional branch, we've seen 1870 // unreachable code. Ignore them. 1871 CondBranch = nullptr; 1872 UncondBranch = &*I; 1873 const MCSymbol *Sym = getTargetSymbol(*I); 1874 assert(Sym != nullptr && 1875 "Couldn't extract BB symbol from jump operand"); 1876 TBB = Sym; 1877 continue; 1878 } 1879 1880 // Handle conditional branches and ignore indirect branches 1881 if (!isUnsupportedBranch(*I) && getCondCode(*I) == X86::COND_INVALID) { 1882 // Indirect branch 1883 return false; 1884 } 1885 1886 if (CondBranch == nullptr) { 1887 const MCSymbol *TargetBB = getTargetSymbol(*I); 1888 if (TargetBB == nullptr) { 1889 // Unrecognized branch target 1890 return false; 1891 } 1892 FBB = TBB; 1893 TBB = TargetBB; 1894 CondBranch = &*I; 1895 continue; 1896 } 1897 1898 llvm_unreachable("multiple conditional branches in one BB"); 1899 } 1900 return true; 1901 } 1902 1903 template <typename Itr> 1904 std::pair<IndirectBranchType, MCInst *> 1905 analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const { 1906 // Analyze PIC-style jump table code template: 1907 // 1908 // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr 1909 // mov ({%r1|%r2}, %index, 4), {%r2|%r1} 1910 // add %r2, %r1 1911 // jmp *%r1 1912 // 1913 // (with any irrelevant instructions in-between) 1914 // 1915 // When we call this helper we've already determined %r1 and %r2, and 1916 // reverse instruction iterator \p II is pointing to the ADD instruction. 1917 // 1918 // PIC jump table looks like following: 1919 // 1920 // JT: ---------- 1921 // E1:| L1 - JT | 1922 // |----------| 1923 // E2:| L2 - JT | 1924 // |----------| 1925 // | | 1926 // ...... 1927 // En:| Ln - JT | 1928 // ---------- 1929 // 1930 // Where L1, L2, ..., Ln represent labels in the function. 1931 // 1932 // The actual relocations in the table will be of the form: 1933 // 1934 // Ln - JT 1935 // = (Ln - En) + (En - JT) 1936 // = R_X86_64_PC32(Ln) + En - JT 1937 // = R_X86_64_PC32(Ln + offsetof(En)) 1938 // 1939 LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n"); 1940 MCInst *MemLocInstr = nullptr; 1941 const MCInst *MovInstr = nullptr; 1942 while (++II != IE) { 1943 MCInst &Instr = *II; 1944 const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode()); 1945 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && 1946 !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { 1947 // Ignore instructions that don't affect R1, R2 registers. 1948 continue; 1949 } 1950 if (!MovInstr) { 1951 // Expect to see MOV instruction. 1952 if (!isMOVSX64rm32(Instr)) { 1953 LLVM_DEBUG(dbgs() << "MOV instruction expected.\n"); 1954 break; 1955 } 1956 1957 // Check if it's setting %r1 or %r2. In canonical form it sets %r2. 1958 // If it sets %r1 - rename the registers so we have to only check 1959 // a single form. 1960 unsigned MovDestReg = Instr.getOperand(0).getReg(); 1961 if (MovDestReg != R2) 1962 std::swap(R1, R2); 1963 if (MovDestReg != R2) { 1964 LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); 1965 break; 1966 } 1967 1968 // Verify operands for MOV. 1969 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1970 if (!MO) 1971 break; 1972 if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 || 1973 MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 || 1974 MO->SegRegNum != X86::NoRegister) 1975 break; 1976 MovInstr = &Instr; 1977 } else { 1978 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) 1979 continue; 1980 if (!isLEA64r(Instr)) { 1981 LLVM_DEBUG(dbgs() << "LEA instruction expected\n"); 1982 break; 1983 } 1984 if (Instr.getOperand(0).getReg() != R1) { 1985 LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); 1986 break; 1987 } 1988 1989 // Verify operands for LEA. 1990 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1991 if (!MO) 1992 break; 1993 if (MO->BaseRegNum != RegInfo->getProgramCounter() || 1994 MO->IndexRegNum != X86::NoRegister || 1995 MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr) 1996 break; 1997 MemLocInstr = &Instr; 1998 break; 1999 } 2000 } 2001 2002 if (!MemLocInstr) 2003 return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); 2004 2005 LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n"); 2006 return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, 2007 MemLocInstr); 2008 } 2009 2010 IndirectBranchType analyzeIndirectBranch( 2011 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End, 2012 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 2013 unsigned &IndexRegNumOut, int64_t &DispValueOut, 2014 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override { 2015 // Try to find a (base) memory location from where the address for 2016 // the indirect branch is loaded. For X86-64 the memory will be specified 2017 // in the following format: 2018 // 2019 // {%rip}/{%basereg} + Imm + IndexReg * Scale 2020 // 2021 // We are interested in the cases where Scale == sizeof(uintptr_t) and 2022 // the contents of the memory are presumably an array of pointers to code. 2023 // 2024 // Normal jump table: 2025 // 2026 // jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr 2027 // 2028 // or 2029 // 2030 // mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr 2031 // ... 2032 // jmp %r1 2033 // 2034 // We handle PIC-style jump tables separately. 2035 // 2036 MemLocInstrOut = nullptr; 2037 BaseRegNumOut = X86::NoRegister; 2038 IndexRegNumOut = X86::NoRegister; 2039 DispValueOut = 0; 2040 DispExprOut = nullptr; 2041 2042 std::reverse_iterator<InstructionIterator> II(End); 2043 std::reverse_iterator<InstructionIterator> IE(Begin); 2044 2045 IndirectBranchType Type = IndirectBranchType::UNKNOWN; 2046 2047 // An instruction referencing memory used by jump instruction (directly or 2048 // via register). This location could be an array of function pointers 2049 // in case of indirect tail call, or a jump table. 2050 MCInst *MemLocInstr = nullptr; 2051 2052 if (MCPlus::getNumPrimeOperands(Instruction) == 1) { 2053 // If the indirect jump is on register - try to detect if the 2054 // register value is loaded from a memory location. 2055 assert(Instruction.getOperand(0).isReg() && "register operand expected"); 2056 const unsigned R1 = Instruction.getOperand(0).getReg(); 2057 // Check if one of the previous instructions defines the jump-on register. 2058 for (auto PrevII = II; PrevII != IE; ++PrevII) { 2059 MCInst &PrevInstr = *PrevII; 2060 const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); 2061 2062 if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) 2063 continue; 2064 2065 if (isMoveMem2Reg(PrevInstr)) { 2066 MemLocInstr = &PrevInstr; 2067 break; 2068 } 2069 if (isADD64rr(PrevInstr)) { 2070 unsigned R2 = PrevInstr.getOperand(2).getReg(); 2071 if (R1 == R2) 2072 return IndirectBranchType::UNKNOWN; 2073 std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); 2074 break; 2075 } 2076 return IndirectBranchType::UNKNOWN; 2077 } 2078 if (!MemLocInstr) { 2079 // No definition seen for the register in this function so far. Could be 2080 // an input parameter - which means it is an external code reference. 2081 // It also could be that the definition happens to be in the code that 2082 // we haven't processed yet. Since we have to be conservative, return 2083 // as UNKNOWN case. 2084 return IndirectBranchType::UNKNOWN; 2085 } 2086 } else { 2087 MemLocInstr = &Instruction; 2088 } 2089 2090 const MCRegister RIPRegister = RegInfo->getProgramCounter(); 2091 2092 // Analyze the memory location. 2093 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr); 2094 if (!MO) 2095 return IndirectBranchType::UNKNOWN; 2096 2097 BaseRegNumOut = MO->BaseRegNum; 2098 IndexRegNumOut = MO->IndexRegNum; 2099 DispValueOut = MO->DispImm; 2100 DispExprOut = MO->DispExpr; 2101 2102 if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) || 2103 MO->SegRegNum != X86::NoRegister) 2104 return IndirectBranchType::UNKNOWN; 2105 2106 if (MemLocInstr == &Instruction && 2107 (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) { 2108 MemLocInstrOut = MemLocInstr; 2109 return IndirectBranchType::POSSIBLE_FIXED_BRANCH; 2110 } 2111 2112 if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2113 (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)) 2114 return IndirectBranchType::UNKNOWN; 2115 2116 if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2117 MO->ScaleImm != PtrSize) 2118 return IndirectBranchType::UNKNOWN; 2119 2120 MemLocInstrOut = MemLocInstr; 2121 2122 return Type; 2123 } 2124 2125 /// Analyze a callsite to see if it could be a virtual method call. This only 2126 /// checks to see if the overall pattern is satisfied, it does not guarantee 2127 /// that the callsite is a true virtual method call. 2128 /// The format of virtual method calls that are recognized is one of the 2129 /// following: 2130 /// 2131 /// Form 1: (found in debug code) 2132 /// add METHOD_OFFSET, %VtableReg 2133 /// mov (%VtableReg), %MethodReg 2134 /// ... 2135 /// call or jmp *%MethodReg 2136 /// 2137 /// Form 2: 2138 /// mov METHOD_OFFSET(%VtableReg), %MethodReg 2139 /// ... 2140 /// call or jmp *%MethodReg 2141 /// 2142 /// Form 3: 2143 /// ... 2144 /// call or jmp *METHOD_OFFSET(%VtableReg) 2145 /// 2146 bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, 2147 InstructionIterator ForwardEnd, 2148 std::vector<MCInst *> &MethodFetchInsns, 2149 unsigned &VtableRegNum, unsigned &MethodRegNum, 2150 uint64_t &MethodOffset) const override { 2151 VtableRegNum = X86::NoRegister; 2152 MethodRegNum = X86::NoRegister; 2153 MethodOffset = 0; 2154 2155 std::reverse_iterator<InstructionIterator> Itr(ForwardEnd); 2156 std::reverse_iterator<InstructionIterator> End(ForwardBegin); 2157 2158 MCInst &CallInst = *Itr++; 2159 assert(isIndirectBranch(CallInst) || isCall(CallInst)); 2160 2161 // The call can just be jmp offset(reg) 2162 if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) { 2163 if (!MO->DispExpr && MO->BaseRegNum != X86::RIP && 2164 MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) { 2165 MethodRegNum = MO->BaseRegNum; 2166 if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister && 2167 MO->SegRegNum == X86::NoRegister) { 2168 VtableRegNum = MethodRegNum; 2169 MethodOffset = MO->DispImm; 2170 MethodFetchInsns.push_back(&CallInst); 2171 return true; 2172 } 2173 } 2174 return false; 2175 } 2176 if (CallInst.getOperand(0).isReg()) 2177 MethodRegNum = CallInst.getOperand(0).getReg(); 2178 else 2179 return false; 2180 2181 if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { 2182 VtableRegNum = X86::NoRegister; 2183 MethodRegNum = X86::NoRegister; 2184 return false; 2185 } 2186 2187 // find load from vtable, this may or may not include the method offset 2188 while (Itr != End) { 2189 MCInst &CurInst = *Itr++; 2190 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2191 if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { 2192 if (!mayLoad(CurInst)) 2193 return false; 2194 if (std::optional<X86MemOperand> MO = 2195 evaluateX86MemoryOperand(CurInst)) { 2196 if (!MO->DispExpr && MO->ScaleImm == 1 && 2197 MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP && 2198 MO->BaseRegNum != X86::NoRegister && 2199 MO->IndexRegNum == X86::NoRegister && 2200 MO->SegRegNum == X86::NoRegister) { 2201 VtableRegNum = MO->BaseRegNum; 2202 MethodOffset = MO->DispImm; 2203 MethodFetchInsns.push_back(&CurInst); 2204 if (MethodOffset != 0) 2205 return true; 2206 break; 2207 } 2208 } 2209 return false; 2210 } 2211 } 2212 2213 if (!VtableRegNum) 2214 return false; 2215 2216 // look for any adds affecting the method register. 2217 while (Itr != End) { 2218 MCInst &CurInst = *Itr++; 2219 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2220 if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { 2221 if (isADDri(CurInst)) { 2222 assert(!MethodOffset); 2223 MethodOffset = CurInst.getOperand(2).getImm(); 2224 MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); 2225 break; 2226 } 2227 } 2228 } 2229 2230 return true; 2231 } 2232 2233 void createStackPointerIncrement(MCInst &Inst, int Size, 2234 bool NoFlagsClobber) const override { 2235 if (NoFlagsClobber) { 2236 Inst.setOpcode(X86::LEA64r); 2237 Inst.clear(); 2238 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2239 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2240 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2241 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2242 Inst.addOperand(MCOperand::createImm(-Size)); // Displacement 2243 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2244 return; 2245 } 2246 Inst.setOpcode(X86::SUB64ri8); 2247 Inst.clear(); 2248 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2249 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2250 Inst.addOperand(MCOperand::createImm(Size)); 2251 } 2252 2253 void createStackPointerDecrement(MCInst &Inst, int Size, 2254 bool NoFlagsClobber) const override { 2255 if (NoFlagsClobber) { 2256 Inst.setOpcode(X86::LEA64r); 2257 Inst.clear(); 2258 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2259 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2260 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2261 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2262 Inst.addOperand(MCOperand::createImm(Size)); // Displacement 2263 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2264 return; 2265 } 2266 Inst.setOpcode(X86::ADD64ri8); 2267 Inst.clear(); 2268 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2269 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2270 Inst.addOperand(MCOperand::createImm(Size)); 2271 } 2272 2273 void createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, 2274 const MCPhysReg &SrcReg, int Size) const override { 2275 unsigned NewOpcode; 2276 switch (Size) { 2277 default: 2278 llvm_unreachable("Invalid operand size"); 2279 return; 2280 case 2: NewOpcode = X86::MOV16mr; break; 2281 case 4: NewOpcode = X86::MOV32mr; break; 2282 case 8: NewOpcode = X86::MOV64mr; break; 2283 } 2284 Inst.setOpcode(NewOpcode); 2285 Inst.clear(); 2286 Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg 2287 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2288 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2289 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2290 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2291 Inst.addOperand(MCOperand::createReg(SrcReg)); 2292 } 2293 2294 void createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, 2295 int Offset, const MCPhysReg &DstReg, 2296 int Size) const override { 2297 return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, 2298 Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister, 2299 DstReg, Size); 2300 } 2301 2302 void createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale, 2303 const MCPhysReg &IndexReg, int64_t Offset, 2304 const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg, 2305 const MCPhysReg &DstReg, int Size) const override { 2306 unsigned NewOpcode; 2307 switch (Size) { 2308 default: 2309 llvm_unreachable("Invalid operand size"); 2310 return; 2311 case 2: NewOpcode = X86::MOV16rm; break; 2312 case 4: NewOpcode = X86::MOV32rm; break; 2313 case 8: NewOpcode = X86::MOV64rm; break; 2314 } 2315 Inst.setOpcode(NewOpcode); 2316 Inst.clear(); 2317 Inst.addOperand(MCOperand::createReg(DstReg)); 2318 Inst.addOperand(MCOperand::createReg(BaseReg)); 2319 Inst.addOperand(MCOperand::createImm(Scale)); 2320 Inst.addOperand(MCOperand::createReg(IndexReg)); 2321 if (OffsetExpr) 2322 Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement 2323 else 2324 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2325 Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg 2326 } 2327 2328 InstructionListType createLoadImmediate(const MCPhysReg Dest, 2329 uint64_t Imm) const override { 2330 InstructionListType Insts; 2331 Insts.emplace_back(); 2332 Insts.back().setOpcode(X86::MOV64ri32); 2333 Insts.back().clear(); 2334 Insts.back().addOperand(MCOperand::createReg(Dest)); 2335 Insts.back().addOperand(MCOperand::createImm(Imm)); 2336 return Insts; 2337 } 2338 2339 void createIJmp32Frag(SmallVectorImpl<MCInst> &Insts, 2340 const MCOperand &BaseReg, const MCOperand &Scale, 2341 const MCOperand &IndexReg, const MCOperand &Offset, 2342 const MCOperand &TmpReg) const override { 2343 // The code fragment we emit here is: 2344 // 2345 // mov32 (%base, %index, scale), %tmpreg 2346 // ijmp *(%tmpreg) 2347 // 2348 MCInst IJmp; 2349 IJmp.setOpcode(X86::JMP64r); 2350 IJmp.addOperand(TmpReg); 2351 2352 MCInst Load; 2353 Load.setOpcode(X86::MOV32rm); 2354 Load.addOperand(TmpReg); 2355 Load.addOperand(BaseReg); 2356 Load.addOperand(Scale); 2357 Load.addOperand(IndexReg); 2358 Load.addOperand(Offset); 2359 Load.addOperand(MCOperand::createReg(X86::NoRegister)); 2360 2361 Insts.push_back(Load); 2362 Insts.push_back(IJmp); 2363 } 2364 2365 void createNoop(MCInst &Inst) const override { 2366 Inst.setOpcode(X86::NOOP); 2367 Inst.clear(); 2368 } 2369 2370 void createReturn(MCInst &Inst) const override { 2371 Inst.setOpcode(X86::RET64); 2372 Inst.clear(); 2373 } 2374 2375 InstructionListType createInlineMemcpy(bool ReturnEnd) const override { 2376 InstructionListType Code; 2377 if (ReturnEnd) 2378 Code.emplace_back(MCInstBuilder(X86::LEA64r) 2379 .addReg(X86::RAX) 2380 .addReg(X86::RDI) 2381 .addImm(1) 2382 .addReg(X86::RDX) 2383 .addImm(0) 2384 .addReg(X86::NoRegister)); 2385 else 2386 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2387 .addReg(X86::RAX) 2388 .addReg(X86::RDI)); 2389 2390 Code.emplace_back(MCInstBuilder(X86::MOV32rr) 2391 .addReg(X86::ECX) 2392 .addReg(X86::EDX)); 2393 Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64)); 2394 2395 return Code; 2396 } 2397 2398 InstructionListType createOneByteMemcpy() const override { 2399 InstructionListType Code; 2400 Code.emplace_back(MCInstBuilder(X86::MOV8rm) 2401 .addReg(X86::CL) 2402 .addReg(X86::RSI) 2403 .addImm(0) 2404 .addReg(X86::NoRegister) 2405 .addImm(0) 2406 .addReg(X86::NoRegister)); 2407 Code.emplace_back(MCInstBuilder(X86::MOV8mr) 2408 .addReg(X86::RDI) 2409 .addImm(0) 2410 .addReg(X86::NoRegister) 2411 .addImm(0) 2412 .addReg(X86::NoRegister) 2413 .addReg(X86::CL)); 2414 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2415 .addReg(X86::RAX) 2416 .addReg(X86::RDI)); 2417 return Code; 2418 } 2419 2420 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 2421 const MCSymbol *Target, 2422 MCContext *Ctx) const override { 2423 InstructionListType Code; 2424 Code.emplace_back(MCInstBuilder(X86::CMP64ri8) 2425 .addReg(RegNo) 2426 .addImm(Imm)); 2427 Code.emplace_back(MCInstBuilder(X86::JCC_1) 2428 .addExpr(MCSymbolRefExpr::create( 2429 Target, MCSymbolRefExpr::VK_None, *Ctx)) 2430 .addImm(X86::COND_E)); 2431 return Code; 2432 } 2433 2434 std::optional<Relocation> 2435 createRelocation(const MCFixup &Fixup, 2436 const MCAsmBackend &MAB) const override { 2437 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2438 2439 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2440 const uint64_t RelOffset = Fixup.getOffset(); 2441 2442 uint64_t RelType; 2443 if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2444 switch (FKI.TargetSize) { 2445 default: 2446 return std::nullopt; 2447 case 8: RelType = ELF::R_X86_64_PC8; break; 2448 case 16: RelType = ELF::R_X86_64_PC16; break; 2449 case 32: RelType = ELF::R_X86_64_PC32; break; 2450 case 64: RelType = ELF::R_X86_64_PC64; break; 2451 } 2452 } else { 2453 switch (FKI.TargetSize) { 2454 default: 2455 return std::nullopt; 2456 case 8: RelType = ELF::R_X86_64_8; break; 2457 case 16: RelType = ELF::R_X86_64_16; break; 2458 case 32: RelType = ELF::R_X86_64_32; break; 2459 case 64: RelType = ELF::R_X86_64_64; break; 2460 } 2461 } 2462 2463 auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup); 2464 2465 return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0}); 2466 } 2467 2468 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 2469 int64_t Addend, MCContext *Ctx, int64_t &Value, 2470 uint64_t RelType) const override { 2471 unsigned ImmOpNo = -1U; 2472 2473 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 2474 ++Index) { 2475 if (Inst.getOperand(Index).isImm()) { 2476 ImmOpNo = Index; 2477 // TODO: this is a bit hacky. It finds the correct operand by 2478 // searching for a specific immediate value. If no value is 2479 // provided it defaults to the last immediate operand found. 2480 // This could lead to unexpected results if the instruction 2481 // has more than one immediate with the same value. 2482 if (Inst.getOperand(ImmOpNo).getImm() == Value) 2483 break; 2484 } 2485 } 2486 2487 if (ImmOpNo == -1U) 2488 return false; 2489 2490 Value = Inst.getOperand(ImmOpNo).getImm(); 2491 2492 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 2493 2494 return true; 2495 } 2496 2497 bool replaceRegWithImm(MCInst &Inst, unsigned Register, 2498 int64_t Imm) const override { 2499 2500 enum CheckSignExt : uint8_t { 2501 NOCHECK = 0, 2502 CHECK8, 2503 CHECK32, 2504 }; 2505 2506 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 2507 struct InstInfo { 2508 // Size in bytes that Inst loads from memory. 2509 uint8_t DataSize; 2510 2511 // True when the target operand has to be duplicated because the opcode 2512 // expects a LHS operand. 2513 bool HasLHS; 2514 2515 // List of checks and corresponding opcodes to be used. We try to use the 2516 // smallest possible immediate value when various sizes are available, 2517 // hence we may need to check whether a larger constant fits in a smaller 2518 // immediate. 2519 CheckList Checks; 2520 }; 2521 2522 InstInfo I; 2523 2524 switch (Inst.getOpcode()) { 2525 default: { 2526 switch (getPushSize(Inst)) { 2527 2528 case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break; 2529 case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break; 2530 case 8: I = {8, false, {{CHECK8, X86::PUSH64i8}, 2531 {CHECK32, X86::PUSH64i32}, 2532 {NOCHECK, Inst.getOpcode()}}}; break; 2533 default: return false; 2534 } 2535 break; 2536 } 2537 2538 // MOV 2539 case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 2540 case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 2541 case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 2542 case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32}, 2543 {NOCHECK, X86::MOV64ri}}}; break; 2544 2545 case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break; 2546 case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break; 2547 case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break; 2548 case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32}, 2549 {NOCHECK, X86::MOV64mr}}}; break; 2550 2551 // MOVZX 2552 case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 2553 case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 2554 case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 2555 2556 // CMP 2557 case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 2558 case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8}, 2559 {NOCHECK, X86::CMP16ri}}}; break; 2560 case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8}, 2561 {NOCHECK, X86::CMP32ri}}}; break; 2562 case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8}, 2563 {CHECK32, X86::CMP64ri32}, 2564 {NOCHECK, X86::CMP64rr}}}; break; 2565 2566 // TEST 2567 case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 2568 case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 2569 case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 2570 case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 2571 {NOCHECK, X86::TEST64rr}}}; break; 2572 2573 // ADD 2574 case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 2575 case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8}, 2576 {NOCHECK, X86::ADD16ri}}}; break; 2577 case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8}, 2578 {NOCHECK, X86::ADD32ri}}}; break; 2579 case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8}, 2580 {CHECK32, X86::ADD64ri32}, 2581 {NOCHECK, X86::ADD64rr}}}; break; 2582 2583 // SUB 2584 case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 2585 case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8}, 2586 {NOCHECK, X86::SUB16ri}}}; break; 2587 case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8}, 2588 {NOCHECK, X86::SUB32ri}}}; break; 2589 case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8}, 2590 {CHECK32, X86::SUB64ri32}, 2591 {NOCHECK, X86::SUB64rr}}}; break; 2592 2593 // AND 2594 case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 2595 case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8}, 2596 {NOCHECK, X86::AND16ri}}}; break; 2597 case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8}, 2598 {NOCHECK, X86::AND32ri}}}; break; 2599 case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8}, 2600 {CHECK32, X86::AND64ri32}, 2601 {NOCHECK, X86::AND64rr}}}; break; 2602 2603 // OR 2604 case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 2605 case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8}, 2606 {NOCHECK, X86::OR16ri}}}; break; 2607 case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8}, 2608 {NOCHECK, X86::OR32ri}}}; break; 2609 case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8}, 2610 {CHECK32, X86::OR64ri32}, 2611 {NOCHECK, X86::OR64rr}}}; break; 2612 2613 // XOR 2614 case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 2615 case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8}, 2616 {NOCHECK, X86::XOR16ri}}}; break; 2617 case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8}, 2618 {NOCHECK, X86::XOR32ri}}}; break; 2619 case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8}, 2620 {CHECK32, X86::XOR64ri32}, 2621 {NOCHECK, X86::XOR64rr}}}; break; 2622 } 2623 2624 // Compute the new opcode. 2625 unsigned NewOpcode = 0; 2626 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 2627 NewOpcode = Check.second; 2628 if (Check.first == NOCHECK) 2629 break; 2630 if (Check.first == CHECK8 && isInt<8>(Imm)) 2631 break; 2632 if (Check.first == CHECK32 && isInt<32>(Imm)) 2633 break; 2634 } 2635 if (NewOpcode == Inst.getOpcode()) 2636 return false; 2637 2638 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2639 2640 unsigned NumFound = 0; 2641 for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0), 2642 E = InstDesc.getNumOperands(); 2643 Index != E; ++Index) 2644 if (Inst.getOperand(Index).isReg() && 2645 Inst.getOperand(Index).getReg() == Register) 2646 NumFound++; 2647 2648 if (NumFound != 1) 2649 return false; 2650 2651 MCOperand TargetOp = Inst.getOperand(0); 2652 Inst.clear(); 2653 Inst.setOpcode(NewOpcode); 2654 Inst.addOperand(TargetOp); 2655 if (I.HasLHS) 2656 Inst.addOperand(TargetOp); 2657 Inst.addOperand(MCOperand::createImm(Imm)); 2658 2659 return true; 2660 } 2661 2662 bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace, 2663 unsigned ReplaceWith) const override { 2664 2665 // Get the HasLHS value so that iteration can be done 2666 bool HasLHS; 2667 if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) || 2668 X86::isSUB(Inst.getOpcode())) { 2669 HasLHS = true; 2670 } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) || 2671 X86::isTEST(Inst.getOpcode())) { 2672 HasLHS = false; 2673 } else { 2674 switch (Inst.getOpcode()) { 2675 case X86::MOV8rr: 2676 case X86::MOV8rm: 2677 case X86::MOV8mr: 2678 case X86::MOV8ri: 2679 case X86::MOV16rr: 2680 case X86::MOV16rm: 2681 case X86::MOV16mr: 2682 case X86::MOV16ri: 2683 case X86::MOV32rr: 2684 case X86::MOV32rm: 2685 case X86::MOV32mr: 2686 case X86::MOV32ri: 2687 case X86::MOV64rr: 2688 case X86::MOV64rm: 2689 case X86::MOV64mr: 2690 case X86::MOV64ri: 2691 case X86::MOVZX16rr8: 2692 case X86::MOVZX32rr8: 2693 case X86::MOVZX32rr16: 2694 case X86::MOVSX32rm8: 2695 case X86::MOVSX32rr8: 2696 case X86::MOVSX64rm32: 2697 case X86::LEA64r: 2698 HasLHS = false; 2699 break; 2700 default: 2701 return false; 2702 } 2703 } 2704 2705 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2706 2707 bool FoundOne = false; 2708 2709 // Iterate only through src operands that arent also dest operands 2710 for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0), 2711 E = InstDesc.getNumOperands(); 2712 Index != E; ++Index) { 2713 BitVector RegAliases = getAliases(ToReplace, true); 2714 if (!Inst.getOperand(Index).isReg() || 2715 !RegAliases.test(Inst.getOperand(Index).getReg())) 2716 continue; 2717 // Resize register if needed 2718 unsigned SizedReplaceWith = getAliasSized( 2719 ReplaceWith, getRegSize(Inst.getOperand(Index).getReg())); 2720 MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith); 2721 Inst.getOperand(Index) = NewOperand; 2722 FoundOne = true; 2723 } 2724 2725 // Return true if at least one operand was replaced 2726 return FoundOne; 2727 } 2728 2729 void createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 2730 MCContext *Ctx) const override { 2731 Inst.setOpcode(X86::JMP_1); 2732 Inst.clear(); 2733 Inst.addOperand(MCOperand::createExpr( 2734 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); 2735 } 2736 2737 void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target, 2738 MCContext *Ctx) const override { 2739 Inst.setOpcode(X86::JMP_4); 2740 Inst.clear(); 2741 Inst.addOperand(MCOperand::createExpr( 2742 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2743 } 2744 2745 void createCall(MCInst &Inst, const MCSymbol *Target, 2746 MCContext *Ctx) override { 2747 Inst.setOpcode(X86::CALL64pcrel32); 2748 Inst.clear(); 2749 Inst.addOperand(MCOperand::createExpr( 2750 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2751 } 2752 2753 void createTailCall(MCInst &Inst, const MCSymbol *Target, 2754 MCContext *Ctx) override { 2755 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 2756 } 2757 2758 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 2759 MCContext *Ctx) override { 2760 Seq.clear(); 2761 Seq.emplace_back(); 2762 createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true); 2763 } 2764 2765 void createTrap(MCInst &Inst) const override { 2766 Inst.clear(); 2767 Inst.setOpcode(X86::TRAP); 2768 } 2769 2770 void createCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC, 2771 MCContext *Ctx) const override { 2772 Inst.setOpcode(X86::JCC_1); 2773 Inst.clear(); 2774 Inst.addOperand(MCOperand::createExpr( 2775 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2776 Inst.addOperand(MCOperand::createImm(CC)); 2777 } 2778 2779 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 2780 MCContext *Ctx) const override { 2781 unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); 2782 assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); 2783 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); 2784 Inst.getOperand(0) = MCOperand::createExpr( 2785 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2786 return true; 2787 } 2788 2789 bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, 2790 unsigned CC) const override { 2791 if (CC == X86::COND_INVALID) 2792 return false; 2793 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC); 2794 Inst.getOperand(0) = MCOperand::createExpr( 2795 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2796 return true; 2797 } 2798 2799 unsigned getCanonicalBranchCondCode(unsigned CC) const override { 2800 switch (CC) { 2801 default: return X86::COND_INVALID; 2802 2803 case X86::COND_E: return X86::COND_E; 2804 case X86::COND_NE: return X86::COND_E; 2805 2806 case X86::COND_L: return X86::COND_L; 2807 case X86::COND_GE: return X86::COND_L; 2808 2809 case X86::COND_LE: return X86::COND_G; 2810 case X86::COND_G: return X86::COND_G; 2811 2812 case X86::COND_B: return X86::COND_B; 2813 case X86::COND_AE: return X86::COND_B; 2814 2815 case X86::COND_BE: return X86::COND_A; 2816 case X86::COND_A: return X86::COND_A; 2817 2818 case X86::COND_S: return X86::COND_S; 2819 case X86::COND_NS: return X86::COND_S; 2820 2821 case X86::COND_P: return X86::COND_P; 2822 case X86::COND_NP: return X86::COND_P; 2823 2824 case X86::COND_O: return X86::COND_O; 2825 case X86::COND_NO: return X86::COND_O; 2826 } 2827 } 2828 2829 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 2830 MCContext *Ctx) const override { 2831 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 2832 "Invalid instruction"); 2833 Inst.getOperand(0) = MCOperand::createExpr( 2834 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2835 return true; 2836 } 2837 2838 MCPhysReg getX86R11() const override { return X86::R11; } 2839 2840 unsigned getShortBranchOpcode(unsigned Opcode) const override { 2841 switch (Opcode) { 2842 default: 2843 return Opcode; 2844 case X86::JMP_2: 2845 return X86::JMP_1; 2846 case X86::JMP_4: 2847 return X86::JMP_1; 2848 case X86::JCC_2: 2849 return X86::JCC_1; 2850 case X86::JCC_4: 2851 return X86::JCC_1; 2852 } 2853 } 2854 2855 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 2856 // FIXME: this should depend on the calling convention. 2857 switch (ArgNo) { 2858 case 0: return X86::RDI; 2859 case 1: return X86::RSI; 2860 case 2: return X86::RDX; 2861 case 3: return X86::RCX; 2862 case 4: return X86::R8; 2863 case 5: return X86::R9; 2864 default: return getNoRegister(); 2865 } 2866 } 2867 2868 void createPause(MCInst &Inst) const override { 2869 Inst.clear(); 2870 Inst.setOpcode(X86::PAUSE); 2871 } 2872 2873 void createLfence(MCInst &Inst) const override { 2874 Inst.clear(); 2875 Inst.setOpcode(X86::LFENCE); 2876 } 2877 2878 void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 2879 bool IsTailCall) override { 2880 Inst.clear(); 2881 Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 2882 Inst.addOperand(MCOperand::createExpr( 2883 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2884 if (IsTailCall) 2885 setTailCall(Inst); 2886 } 2887 2888 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 2889 MCContext *Ctx, bool IsTailCall) override { 2890 Seq.clear(); 2891 MCInst Inst; 2892 Inst.setOpcode(X86::JMP_1); 2893 Inst.addOperand(MCOperand::createExpr( 2894 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2895 if (IsTailCall) 2896 setTailCall(Inst); 2897 Seq.emplace_back(Inst); 2898 } 2899 2900 bool isConditionalMove(const MCInst &Inst) const override { 2901 unsigned OpCode = Inst.getOpcode(); 2902 return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr || 2903 OpCode == X86::CMOV64rr); 2904 } 2905 2906 bool isBranchOnMem(const MCInst &Inst) const override { 2907 unsigned OpCode = Inst.getOpcode(); 2908 if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) || 2909 OpCode == X86::JMP64m) 2910 return true; 2911 2912 return false; 2913 } 2914 2915 bool isBranchOnReg(const MCInst &Inst) const override { 2916 unsigned OpCode = Inst.getOpcode(); 2917 if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) || 2918 OpCode == X86::JMP64r) 2919 return true; 2920 2921 return false; 2922 } 2923 2924 void createPushRegister(MCInst &Inst, MCPhysReg Reg, 2925 unsigned Size) const override { 2926 Inst.clear(); 2927 unsigned NewOpcode = 0; 2928 if (Reg == X86::EFLAGS) { 2929 switch (Size) { 2930 case 2: NewOpcode = X86::PUSHF16; break; 2931 case 4: NewOpcode = X86::PUSHF32; break; 2932 case 8: NewOpcode = X86::PUSHF64; break; 2933 default: 2934 llvm_unreachable("Unexpected size"); 2935 } 2936 Inst.setOpcode(NewOpcode); 2937 return; 2938 } 2939 switch (Size) { 2940 case 2: NewOpcode = X86::PUSH16r; break; 2941 case 4: NewOpcode = X86::PUSH32r; break; 2942 case 8: NewOpcode = X86::PUSH64r; break; 2943 default: 2944 llvm_unreachable("Unexpected size"); 2945 } 2946 Inst.setOpcode(NewOpcode); 2947 Inst.addOperand(MCOperand::createReg(Reg)); 2948 } 2949 2950 void createPopRegister(MCInst &Inst, MCPhysReg Reg, 2951 unsigned Size) const override { 2952 Inst.clear(); 2953 unsigned NewOpcode = 0; 2954 if (Reg == X86::EFLAGS) { 2955 switch (Size) { 2956 case 2: NewOpcode = X86::POPF16; break; 2957 case 4: NewOpcode = X86::POPF32; break; 2958 case 8: NewOpcode = X86::POPF64; break; 2959 default: 2960 llvm_unreachable("Unexpected size"); 2961 } 2962 Inst.setOpcode(NewOpcode); 2963 return; 2964 } 2965 switch (Size) { 2966 case 2: NewOpcode = X86::POP16r; break; 2967 case 4: NewOpcode = X86::POP32r; break; 2968 case 8: NewOpcode = X86::POP64r; break; 2969 default: 2970 llvm_unreachable("Unexpected size"); 2971 } 2972 Inst.setOpcode(NewOpcode); 2973 Inst.addOperand(MCOperand::createReg(Reg)); 2974 } 2975 2976 void createPushFlags(MCInst &Inst, unsigned Size) const override { 2977 return createPushRegister(Inst, X86::EFLAGS, Size); 2978 } 2979 2980 void createPopFlags(MCInst &Inst, unsigned Size) const override { 2981 return createPopRegister(Inst, X86::EFLAGS, Size); 2982 } 2983 2984 void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value, 2985 unsigned Size) const { 2986 unsigned int Opcode; 2987 switch (Size) { 2988 case 1: Opcode = X86::ADD8ri; break; 2989 case 2: Opcode = X86::ADD16ri; break; 2990 case 4: Opcode = X86::ADD32ri; break; 2991 default: 2992 llvm_unreachable("Unexpected size"); 2993 } 2994 Inst.setOpcode(Opcode); 2995 Inst.clear(); 2996 Inst.addOperand(MCOperand::createReg(Reg)); 2997 Inst.addOperand(MCOperand::createReg(Reg)); 2998 Inst.addOperand(MCOperand::createImm(Value)); 2999 } 3000 3001 void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg, 3002 unsigned Size) const { 3003 unsigned int Opcode; 3004 switch (Size) { 3005 case 1: Opcode = X86::MOV8ri; break; 3006 case 2: Opcode = X86::MOV16ri; break; 3007 case 4: Opcode = X86::MOV32ri; break; 3008 // Writing to a 32-bit register always zeros the upper 32 bits of the 3009 // full-width register 3010 case 8: 3011 Opcode = X86::MOV32ri; 3012 Reg = getAliasSized(Reg, 4); 3013 break; 3014 default: 3015 llvm_unreachable("Unexpected size"); 3016 } 3017 Inst.setOpcode(Opcode); 3018 Inst.clear(); 3019 Inst.addOperand(MCOperand::createReg(Reg)); 3020 Inst.addOperand(MCOperand::createImm(0)); 3021 } 3022 3023 void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const { 3024 Inst.setOpcode(X86::SETCCr); 3025 Inst.clear(); 3026 Inst.addOperand(MCOperand::createReg(Reg)); 3027 Inst.addOperand(MCOperand::createImm(X86::COND_O)); 3028 } 3029 3030 void createX86Lahf(MCInst &Inst) const { 3031 Inst.setOpcode(X86::LAHF); 3032 Inst.clear(); 3033 } 3034 3035 void createX86Sahf(MCInst &Inst) const { 3036 Inst.setOpcode(X86::SAHF); 3037 Inst.clear(); 3038 } 3039 3040 InstructionListType 3041 createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf, 3042 unsigned CodePointerSize) const override { 3043 InstructionListType Instrs(IsLeaf ? 13 : 11); 3044 unsigned int I = 0; 3045 3046 // Don't clobber application red zone (ABI dependent) 3047 if (IsLeaf) 3048 createStackPointerIncrement(Instrs[I++], 128, 3049 /*NoFlagsClobber=*/true); 3050 3051 // Performance improvements based on the optimization discussed at 3052 // https://reviews.llvm.org/D6629 3053 // LAHF/SAHF are used instead of PUSHF/POPF 3054 // PUSHF 3055 createPushRegister(Instrs[I++], X86::RAX, 8); 3056 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3057 createX86Lahf(Instrs[I++]); 3058 createPushRegister(Instrs[I++], X86::RAX, 8); 3059 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3060 createX86SaveOVFlagToRegister(Instrs[I++], X86::AL); 3061 // LOCK INC 3062 InstructionListType IncMem = createIncMemory(Target, Ctx); 3063 assert(IncMem.size() == 1 && "Invalid IncMem size"); 3064 std::copy(IncMem.begin(), IncMem.end(), Instrs.begin() + I); 3065 I += IncMem.size(); 3066 // POPF 3067 createAddRegImm(Instrs[I++], X86::AL, 127, 1); 3068 createPopRegister(Instrs[I++], X86::RAX, 8); 3069 createX86Sahf(Instrs[I++]); 3070 createPopRegister(Instrs[I++], X86::RAX, 8); 3071 3072 if (IsLeaf) 3073 createStackPointerDecrement(Instrs[I], 128, 3074 /*NoFlagsClobber=*/true); 3075 return Instrs; 3076 } 3077 3078 void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg, 3079 int64_t Disp) const { 3080 Inst.setOpcode(X86::XCHG64rm); 3081 Inst.clear(); 3082 Inst.addOperand(MCOperand::createReg(Source)); 3083 Inst.addOperand(MCOperand::createReg(Source)); 3084 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3085 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3086 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3087 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3088 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3089 } 3090 3091 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 3092 int64_t Disp) const { 3093 Inst.setOpcode(X86::JMP64m); 3094 Inst.clear(); 3095 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3096 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3097 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3098 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3099 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3100 } 3101 3102 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 3103 MCSymbol *HandlerFuncAddr, 3104 int CallSiteID, 3105 MCContext *Ctx) override { 3106 // Check if the target address expression used in the original indirect call 3107 // uses the stack pointer, which we are going to clobber. 3108 static BitVector SPAliases(getAliases(X86::RSP)); 3109 bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) { 3110 return Op.isReg() && SPAliases[Op.getReg()]; 3111 }); 3112 3113 InstructionListType Insts; 3114 MCPhysReg TempReg = getIntArgRegister(0); 3115 // Code sequence used to enter indirect call instrumentation helper: 3116 // push %rdi 3117 // add $8, %rsp ;; $rsp may be used in target, so fix it to prev val 3118 // movq target, %rdi ;; via convertIndirectCallTargetToLoad 3119 // sub $8, %rsp ;; restore correct stack value 3120 // push %rdi 3121 // movq $CallSiteID, %rdi 3122 // push %rdi 3123 // callq/jmp HandlerFuncAddr 3124 Insts.emplace_back(); 3125 createPushRegister(Insts.back(), TempReg, 8); 3126 if (UsesSP) { // Only adjust SP if we really need to 3127 Insts.emplace_back(); 3128 createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3129 } 3130 Insts.emplace_back(CallInst); 3131 // Insts.back() and CallInst now share the same annotation instruction. 3132 // Strip it from Insts.back(), only preserving tail call annotation. 3133 stripAnnotations(Insts.back(), /*KeepTC=*/true); 3134 convertIndirectCallToLoad(Insts.back(), TempReg); 3135 if (UsesSP) { 3136 Insts.emplace_back(); 3137 createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3138 } 3139 Insts.emplace_back(); 3140 createPushRegister(Insts.back(), TempReg, 8); 3141 InstructionListType LoadImm = createLoadImmediate(TempReg, CallSiteID); 3142 Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end()); 3143 Insts.emplace_back(); 3144 createPushRegister(Insts.back(), TempReg, 8); 3145 3146 MCInst &NewCallInst = Insts.emplace_back(); 3147 createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); 3148 3149 // Carry over metadata including tail call marker if present. 3150 stripAnnotations(NewCallInst); 3151 moveAnnotations(std::move(CallInst), NewCallInst); 3152 3153 return Insts; 3154 } 3155 3156 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 3157 const MCPhysReg TempReg = getIntArgRegister(0); 3158 // We just need to undo the sequence created for every ind call in 3159 // instrumentIndirectTarget(), which can be accomplished minimally with: 3160 // popfq 3161 // pop %rdi 3162 // add $16, %rsp 3163 // xchg (%rsp), %rdi 3164 // jmp *-8(%rsp) 3165 InstructionListType Insts(5); 3166 createPopFlags(Insts[0], 8); 3167 createPopRegister(Insts[1], TempReg, 8); 3168 createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); 3169 createSwap(Insts[3], TempReg, X86::RSP, 0); 3170 createIndirectBranch(Insts[4], X86::RSP, -8); 3171 return Insts; 3172 } 3173 3174 InstructionListType 3175 createInstrumentedIndTailCallHandlerExitBB() const override { 3176 const MCPhysReg TempReg = getIntArgRegister(0); 3177 // Same thing as above, but for tail calls 3178 // popfq 3179 // add $16, %rsp 3180 // pop %rdi 3181 // jmp *-16(%rsp) 3182 InstructionListType Insts(4); 3183 createPopFlags(Insts[0], 8); 3184 createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); 3185 createPopRegister(Insts[2], TempReg, 8); 3186 createIndirectBranch(Insts[3], X86::RSP, -16); 3187 return Insts; 3188 } 3189 3190 InstructionListType 3191 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 3192 const MCSymbol *IndCallHandler, 3193 MCContext *Ctx) override { 3194 const MCPhysReg TempReg = getIntArgRegister(0); 3195 // Code sequence used to check whether InstrTampoline was initialized 3196 // and call it if so, returns via IndCallHandler. 3197 // pushfq 3198 // mov InstrTrampoline,%rdi 3199 // cmp $0x0,%rdi 3200 // je IndCallHandler 3201 // callq *%rdi 3202 // jmpq IndCallHandler 3203 InstructionListType Insts; 3204 Insts.emplace_back(); 3205 createPushFlags(Insts.back(), 8); 3206 Insts.emplace_back(); 3207 createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); 3208 InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); 3209 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 3210 Insts.emplace_back(); 3211 Insts.back().setOpcode(X86::CALL64r); 3212 Insts.back().addOperand(MCOperand::createReg(TempReg)); 3213 Insts.emplace_back(); 3214 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 3215 return Insts; 3216 } 3217 3218 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 3219 InstructionListType Insts(2); 3220 MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); 3221 createMove(Insts[0], NumLocs, X86::EAX, Ctx); 3222 createReturn(Insts[1]); 3223 return Insts; 3224 } 3225 3226 InstructionListType 3227 createInstrLocationsGetter(MCContext *Ctx) const override { 3228 InstructionListType Insts(2); 3229 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); 3230 createLea(Insts[0], Locs, X86::EAX, Ctx); 3231 createReturn(Insts[1]); 3232 return Insts; 3233 } 3234 3235 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 3236 InstructionListType Insts(2); 3237 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); 3238 createLea(Insts[0], Locs, X86::EAX, Ctx); 3239 createReturn(Insts[1]); 3240 return Insts; 3241 } 3242 3243 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 3244 InstructionListType Insts(2); 3245 MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); 3246 createMove(Insts[0], NumFuncs, X86::EAX, Ctx); 3247 createReturn(Insts[1]); 3248 return Insts; 3249 } 3250 3251 InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, 3252 MCContext *Ctx) override { 3253 InstructionListType Insts(1); 3254 createUncondBranch(Insts[0], TgtSym, Ctx); 3255 return Insts; 3256 } 3257 3258 InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { 3259 InstructionListType Insts(1); 3260 createReturn(Insts[0]); 3261 return Insts; 3262 } 3263 3264 BlocksVectorTy indirectCallPromotion( 3265 const MCInst &CallInst, 3266 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3267 const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms, 3268 const std::vector<MCInst *> &MethodFetchInsns, 3269 const bool MinimizeCodeSize, MCContext *Ctx) override { 3270 const bool IsTailCall = isTailCall(CallInst); 3271 const bool IsJumpTable = getJumpTable(CallInst) != 0; 3272 BlocksVectorTy Results; 3273 3274 // Label for the current code block. 3275 MCSymbol *NextTarget = nullptr; 3276 3277 // The join block which contains all the instructions following CallInst. 3278 // MergeBlock remains null if CallInst is a tail call. 3279 MCSymbol *MergeBlock = nullptr; 3280 3281 unsigned FuncAddrReg = X86::R10; 3282 3283 const bool LoadElim = !VtableSyms.empty(); 3284 assert((!LoadElim || VtableSyms.size() == Targets.size()) && 3285 "There must be a vtable entry for every method " 3286 "in the targets vector."); 3287 3288 if (MinimizeCodeSize && !LoadElim) { 3289 std::set<unsigned> UsedRegs; 3290 3291 for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) { 3292 const MCOperand &Op = CallInst.getOperand(I); 3293 if (Op.isReg()) 3294 UsedRegs.insert(Op.getReg()); 3295 } 3296 3297 if (UsedRegs.count(X86::R10) == 0) 3298 FuncAddrReg = X86::R10; 3299 else if (UsedRegs.count(X86::R11) == 0) 3300 FuncAddrReg = X86::R11; 3301 else 3302 return Results; 3303 } 3304 3305 const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { 3306 assert(MergeBlock); 3307 NewCall.push_back(CallInst); 3308 MCInst &Merge = NewCall.back(); 3309 Merge.clear(); 3310 createUncondBranch(Merge, MergeBlock, Ctx); 3311 }; 3312 3313 for (unsigned int i = 0; i < Targets.size(); ++i) { 3314 Results.emplace_back(NextTarget, InstructionListType()); 3315 InstructionListType *NewCall = &Results.back().second; 3316 3317 if (MinimizeCodeSize && !LoadElim) { 3318 // Load the call target into FuncAddrReg. 3319 NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc 3320 MCInst &Target = NewCall->back(); 3321 Target.clear(); 3322 Target.setOpcode(X86::MOV64ri32); 3323 Target.addOperand(MCOperand::createReg(FuncAddrReg)); 3324 if (Targets[i].first) { 3325 // Is this OK? 3326 Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3327 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3328 } else { 3329 const uint64_t Addr = Targets[i].second; 3330 // Immediate address is out of sign extended 32 bit range. 3331 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3332 return BlocksVectorTy(); 3333 3334 Target.addOperand(MCOperand::createImm(Addr)); 3335 } 3336 3337 // Compare current call target to a specific address. 3338 NewCall->push_back(CallInst); 3339 MCInst &Compare = NewCall->back(); 3340 Compare.clear(); 3341 if (isBranchOnReg(CallInst)) 3342 Compare.setOpcode(X86::CMP64rr); 3343 else if (CallInst.getOpcode() == X86::CALL64pcrel32) 3344 Compare.setOpcode(X86::CMP64ri32); 3345 else 3346 Compare.setOpcode(X86::CMP64rm); 3347 3348 Compare.addOperand(MCOperand::createReg(FuncAddrReg)); 3349 3350 // TODO: Would be preferable to only load this value once. 3351 for (unsigned i = 0; 3352 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3353 if (!CallInst.getOperand(i).isInst()) 3354 Compare.addOperand(CallInst.getOperand(i)); 3355 } else { 3356 // Compare current call target to a specific address. 3357 NewCall->push_back(CallInst); 3358 MCInst &Compare = NewCall->back(); 3359 Compare.clear(); 3360 if (isBranchOnReg(CallInst)) 3361 Compare.setOpcode(X86::CMP64ri32); 3362 else 3363 Compare.setOpcode(X86::CMP64mi32); 3364 3365 // Original call address. 3366 for (unsigned i = 0; 3367 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3368 if (!CallInst.getOperand(i).isInst()) 3369 Compare.addOperand(CallInst.getOperand(i)); 3370 3371 // Target address. 3372 if (Targets[i].first || LoadElim) { 3373 const MCSymbol *Sym = 3374 LoadElim ? VtableSyms[i].first : Targets[i].first; 3375 const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0; 3376 const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx); 3377 if (Addend) 3378 Expr = MCBinaryExpr::createAdd( 3379 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx); 3380 Compare.addOperand(MCOperand::createExpr(Expr)); 3381 } else { 3382 const uint64_t Addr = Targets[i].second; 3383 // Immediate address is out of sign extended 32 bit range. 3384 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3385 return BlocksVectorTy(); 3386 3387 Compare.addOperand(MCOperand::createImm(Addr)); 3388 } 3389 } 3390 3391 // jump to next target compare. 3392 NextTarget = 3393 Ctx->createNamedTempSymbol(); // generate label for the next block 3394 NewCall->push_back(CallInst); 3395 3396 if (IsJumpTable) { 3397 MCInst &Je = NewCall->back(); 3398 3399 // Jump to next compare if target addresses don't match. 3400 Je.clear(); 3401 Je.setOpcode(X86::JCC_1); 3402 if (Targets[i].first) 3403 Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3404 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3405 else 3406 Je.addOperand(MCOperand::createImm(Targets[i].second)); 3407 3408 Je.addOperand(MCOperand::createImm(X86::COND_E)); 3409 assert(!isInvoke(CallInst)); 3410 } else { 3411 MCInst &Jne = NewCall->back(); 3412 3413 // Jump to next compare if target addresses don't match. 3414 Jne.clear(); 3415 Jne.setOpcode(X86::JCC_1); 3416 Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3417 NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); 3418 Jne.addOperand(MCOperand::createImm(X86::COND_NE)); 3419 3420 // Call specific target directly. 3421 Results.emplace_back(Ctx->createNamedTempSymbol(), 3422 InstructionListType()); 3423 NewCall = &Results.back().second; 3424 NewCall->push_back(CallInst); 3425 MCInst &CallOrJmp = NewCall->back(); 3426 3427 CallOrJmp.clear(); 3428 3429 if (MinimizeCodeSize && !LoadElim) { 3430 CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r); 3431 CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); 3432 } else { 3433 CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 3434 3435 if (Targets[i].first) 3436 CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3437 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3438 else 3439 CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); 3440 } 3441 if (IsTailCall) 3442 setTailCall(CallOrJmp); 3443 3444 if (CallOrJmp.getOpcode() == X86::CALL64r || 3445 CallOrJmp.getOpcode() == X86::CALL64pcrel32) { 3446 if (std::optional<uint32_t> Offset = getOffset(CallInst)) 3447 // Annotated as duplicated call 3448 setOffset(CallOrJmp, *Offset); 3449 } 3450 3451 if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { 3452 // Copy over any EH or GNU args size information from the original 3453 // call. 3454 std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst); 3455 if (EHInfo) 3456 addEHInfo(CallOrJmp, *EHInfo); 3457 int64_t GnuArgsSize = getGnuArgsSize(CallInst); 3458 if (GnuArgsSize >= 0) 3459 addGnuArgsSize(CallOrJmp, GnuArgsSize); 3460 } 3461 3462 if (!IsTailCall) { 3463 // The fallthrough block for the most common target should be 3464 // the merge block. 3465 if (i == 0) { 3466 // Fallthrough to merge block. 3467 MergeBlock = Ctx->createNamedTempSymbol(); 3468 } else { 3469 // Insert jump to the merge block if we are not doing a fallthrough. 3470 jumpToMergeBlock(*NewCall); 3471 } 3472 } 3473 } 3474 } 3475 3476 // Cold call block. 3477 Results.emplace_back(NextTarget, InstructionListType()); 3478 InstructionListType &NewCall = Results.back().second; 3479 for (const MCInst *Inst : MethodFetchInsns) 3480 if (Inst != &CallInst) 3481 NewCall.push_back(*Inst); 3482 NewCall.push_back(CallInst); 3483 3484 // Jump to merge block from cold call block 3485 if (!IsTailCall && !IsJumpTable) { 3486 jumpToMergeBlock(NewCall); 3487 3488 // Record merge block 3489 Results.emplace_back(MergeBlock, InstructionListType()); 3490 } 3491 3492 return Results; 3493 } 3494 3495 BlocksVectorTy jumpTablePromotion( 3496 const MCInst &IJmpInst, 3497 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3498 const std::vector<MCInst *> &TargetFetchInsns, 3499 MCContext *Ctx) const override { 3500 assert(getJumpTable(IJmpInst) != 0); 3501 uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg"); 3502 if (IndexReg == 0) 3503 return BlocksVectorTy(); 3504 3505 BlocksVectorTy Results; 3506 3507 // Label for the current code block. 3508 MCSymbol *NextTarget = nullptr; 3509 3510 for (unsigned int i = 0; i < Targets.size(); ++i) { 3511 Results.emplace_back(NextTarget, InstructionListType()); 3512 InstructionListType *CurBB = &Results.back().second; 3513 3514 // Compare current index to a specific index. 3515 CurBB->emplace_back(MCInst()); 3516 MCInst &CompareInst = CurBB->back(); 3517 CompareInst.setLoc(IJmpInst.getLoc()); 3518 CompareInst.setOpcode(X86::CMP64ri32); 3519 CompareInst.addOperand(MCOperand::createReg(IndexReg)); 3520 3521 const uint64_t CaseIdx = Targets[i].second; 3522 // Immediate address is out of sign extended 32 bit range. 3523 if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) 3524 return BlocksVectorTy(); 3525 3526 CompareInst.addOperand(MCOperand::createImm(CaseIdx)); 3527 shortenInstruction(CompareInst, *Ctx->getSubtargetInfo()); 3528 3529 // jump to next target compare. 3530 NextTarget = 3531 Ctx->createNamedTempSymbol(); // generate label for the next block 3532 CurBB->push_back(MCInst()); 3533 3534 MCInst &JEInst = CurBB->back(); 3535 JEInst.setLoc(IJmpInst.getLoc()); 3536 3537 // Jump to target if indices match 3538 JEInst.setOpcode(X86::JCC_1); 3539 JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3540 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3541 JEInst.addOperand(MCOperand::createImm(X86::COND_E)); 3542 } 3543 3544 // Cold call block. 3545 Results.emplace_back(NextTarget, InstructionListType()); 3546 InstructionListType &CurBB = Results.back().second; 3547 for (const MCInst *Inst : TargetFetchInsns) 3548 if (Inst != &IJmpInst) 3549 CurBB.push_back(*Inst); 3550 3551 CurBB.push_back(IJmpInst); 3552 3553 return Results; 3554 } 3555 3556 private: 3557 void createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3558 MCContext *Ctx) const { 3559 Inst.setOpcode(X86::MOV64rm); 3560 Inst.clear(); 3561 Inst.addOperand(MCOperand::createReg(Reg)); 3562 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3563 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3564 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3565 Inst.addOperand(MCOperand::createExpr( 3566 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3567 *Ctx))); // Displacement 3568 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3569 } 3570 3571 void createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3572 MCContext *Ctx) const { 3573 Inst.setOpcode(X86::LEA64r); 3574 Inst.clear(); 3575 Inst.addOperand(MCOperand::createReg(Reg)); 3576 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3577 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3578 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3579 Inst.addOperand(MCOperand::createExpr( 3580 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3581 *Ctx))); // Displacement 3582 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3583 } 3584 }; 3585 3586 } // namespace 3587 3588 namespace llvm { 3589 namespace bolt { 3590 3591 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, 3592 const MCInstrInfo *Info, 3593 const MCRegisterInfo *RegInfo, 3594 const MCSubtargetInfo *STI) { 3595 return new X86MCPlusBuilder(Analysis, Info, RegInfo, STI); 3596 } 3597 3598 } // namespace bolt 3599 } // namespace llvm 3600