1 //===- bolt/Target/AArch64/AArch64MCPlusBuilder.cpp -----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides AArch64-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/AArch64AddressingModes.h" 14 #include "MCTargetDesc/AArch64FixupKinds.h" 15 #include "MCTargetDesc/AArch64MCExpr.h" 16 #include "MCTargetDesc/AArch64MCTargetDesc.h" 17 #include "Utils/AArch64BaseInfo.h" 18 #include "bolt/Core/BinaryBasicBlock.h" 19 #include "bolt/Core/BinaryFunction.h" 20 #include "bolt/Core/MCPlusBuilder.h" 21 #include "llvm/BinaryFormat/ELF.h" 22 #include "llvm/MC/MCContext.h" 23 #include "llvm/MC/MCFixupKindInfo.h" 24 #include "llvm/MC/MCInstBuilder.h" 25 #include "llvm/MC/MCInstrInfo.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/DataExtractor.h" 28 #include "llvm/Support/Debug.h" 29 #include "llvm/Support/ErrorHandling.h" 30 31 #define DEBUG_TYPE "mcplus" 32 33 using namespace llvm; 34 using namespace bolt; 35 36 namespace { 37 38 static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) { 39 Inst.setOpcode(AArch64::MRS); 40 Inst.clear(); 41 Inst.addOperand(MCOperand::createReg(RegName)); 42 Inst.addOperand(MCOperand::createImm(AArch64SysReg::NZCV)); 43 } 44 45 static void setSystemFlag(MCInst &Inst, MCPhysReg RegName) { 46 Inst.setOpcode(AArch64::MSR); 47 Inst.clear(); 48 Inst.addOperand(MCOperand::createImm(AArch64SysReg::NZCV)); 49 Inst.addOperand(MCOperand::createReg(RegName)); 50 } 51 52 static void createPushRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) { 53 Inst.clear(); 54 unsigned NewOpcode = AArch64::STPXpre; 55 Inst.setOpcode(NewOpcode); 56 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 57 Inst.addOperand(MCOperand::createReg(Reg1)); 58 Inst.addOperand(MCOperand::createReg(Reg2)); 59 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 60 Inst.addOperand(MCOperand::createImm(-2)); 61 } 62 63 static void createPopRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) { 64 Inst.clear(); 65 unsigned NewOpcode = AArch64::LDPXpost; 66 Inst.setOpcode(NewOpcode); 67 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 68 Inst.addOperand(MCOperand::createReg(Reg1)); 69 Inst.addOperand(MCOperand::createReg(Reg2)); 70 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 71 Inst.addOperand(MCOperand::createImm(2)); 72 } 73 74 static void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From) { 75 Inst.setOpcode(AArch64::LDRXui); 76 Inst.clear(); 77 if (From == AArch64::SP) { 78 Inst.setOpcode(AArch64::LDRXpost); 79 Inst.addOperand(MCOperand::createReg(From)); 80 Inst.addOperand(MCOperand::createReg(To)); 81 Inst.addOperand(MCOperand::createReg(From)); 82 Inst.addOperand(MCOperand::createImm(16)); 83 } else { 84 Inst.addOperand(MCOperand::createReg(To)); 85 Inst.addOperand(MCOperand::createReg(From)); 86 Inst.addOperand(MCOperand::createImm(0)); 87 } 88 } 89 90 static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) { 91 Inst.setOpcode(AArch64::STRXui); 92 Inst.clear(); 93 if (To == AArch64::SP) { 94 Inst.setOpcode(AArch64::STRXpre); 95 Inst.addOperand(MCOperand::createReg(To)); 96 Inst.addOperand(MCOperand::createReg(From)); 97 Inst.addOperand(MCOperand::createReg(To)); 98 Inst.addOperand(MCOperand::createImm(-16)); 99 } else { 100 Inst.addOperand(MCOperand::createReg(From)); 101 Inst.addOperand(MCOperand::createReg(To)); 102 Inst.addOperand(MCOperand::createImm(0)); 103 } 104 } 105 106 static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) { 107 // NOTE: Supports only ARM with LSE extension 108 Inst.setOpcode(AArch64::LDADDX); 109 Inst.clear(); 110 Inst.addOperand(MCOperand::createReg(AArch64::XZR)); 111 Inst.addOperand(MCOperand::createReg(RegCnt)); 112 Inst.addOperand(MCOperand::createReg(RegTo)); 113 } 114 115 static void createMovz(MCInst &Inst, MCPhysReg Reg, uint64_t Imm) { 116 assert(Imm <= UINT16_MAX && "Invalid Imm size"); 117 Inst.clear(); 118 Inst.setOpcode(AArch64::MOVZXi); 119 Inst.addOperand(MCOperand::createReg(Reg)); 120 Inst.addOperand(MCOperand::createImm(Imm & 0xFFFF)); 121 Inst.addOperand(MCOperand::createImm(0)); 122 } 123 124 static InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegTmp) { 125 InstructionListType Insts; 126 Insts.emplace_back(); 127 createMovz(Insts.back(), RegTmp, 1); 128 Insts.emplace_back(); 129 atomicAdd(Insts.back(), RegTo, RegTmp); 130 return Insts; 131 } 132 class AArch64MCPlusBuilder : public MCPlusBuilder { 133 public: 134 using MCPlusBuilder::MCPlusBuilder; 135 136 MCPhysReg getStackPointer() const override { return AArch64::SP; } 137 MCPhysReg getFramePointer() const override { return AArch64::FP; } 138 139 bool isPush(const MCInst &Inst) const override { 140 return isStoreToStack(Inst); 141 }; 142 143 bool isPop(const MCInst &Inst) const override { 144 return isLoadFromStack(Inst); 145 }; 146 147 void createCall(MCInst &Inst, const MCSymbol *Target, 148 MCContext *Ctx) override { 149 createDirectCall(Inst, Target, Ctx, false); 150 } 151 152 bool convertTailCallToCall(MCInst &Inst) override { 153 int NewOpcode; 154 switch (Inst.getOpcode()) { 155 default: 156 return false; 157 case AArch64::B: 158 NewOpcode = AArch64::BL; 159 break; 160 case AArch64::BR: 161 NewOpcode = AArch64::BLR; 162 break; 163 } 164 165 Inst.setOpcode(NewOpcode); 166 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 167 clearOffset(Inst); 168 return true; 169 } 170 171 bool equals(const MCTargetExpr &A, const MCTargetExpr &B, 172 CompFuncTy Comp) const override { 173 const auto &AArch64ExprA = cast<AArch64MCExpr>(A); 174 const auto &AArch64ExprB = cast<AArch64MCExpr>(B); 175 if (AArch64ExprA.getKind() != AArch64ExprB.getKind()) 176 return false; 177 178 return MCPlusBuilder::equals(*AArch64ExprA.getSubExpr(), 179 *AArch64ExprB.getSubExpr(), Comp); 180 } 181 182 bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override { 183 return false; 184 } 185 186 bool isADRP(const MCInst &Inst) const override { 187 return Inst.getOpcode() == AArch64::ADRP; 188 } 189 190 bool isADR(const MCInst &Inst) const override { 191 return Inst.getOpcode() == AArch64::ADR; 192 } 193 194 bool isAddXri(const MCInst &Inst) const { 195 return Inst.getOpcode() == AArch64::ADDXri; 196 } 197 198 void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const override { 199 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction"); 200 assert(MCPlus::getNumPrimeOperands(Inst) != 0 && 201 "No operands for ADR instruction"); 202 assert(Inst.getOperand(0).isReg() && 203 "Unexpected operand in ADR instruction"); 204 RegName = Inst.getOperand(0).getReg(); 205 } 206 207 bool isTB(const MCInst &Inst) const { 208 return (Inst.getOpcode() == AArch64::TBNZW || 209 Inst.getOpcode() == AArch64::TBNZX || 210 Inst.getOpcode() == AArch64::TBZW || 211 Inst.getOpcode() == AArch64::TBZX); 212 } 213 214 bool isCB(const MCInst &Inst) const { 215 return (Inst.getOpcode() == AArch64::CBNZW || 216 Inst.getOpcode() == AArch64::CBNZX || 217 Inst.getOpcode() == AArch64::CBZW || 218 Inst.getOpcode() == AArch64::CBZX); 219 } 220 221 bool isMOVW(const MCInst &Inst) const { 222 return (Inst.getOpcode() == AArch64::MOVKWi || 223 Inst.getOpcode() == AArch64::MOVKXi || 224 Inst.getOpcode() == AArch64::MOVNWi || 225 Inst.getOpcode() == AArch64::MOVNXi || 226 Inst.getOpcode() == AArch64::MOVZXi || 227 Inst.getOpcode() == AArch64::MOVZWi); 228 } 229 230 bool isADD(const MCInst &Inst) const { 231 return (Inst.getOpcode() == AArch64::ADDSWri || 232 Inst.getOpcode() == AArch64::ADDSWrr || 233 Inst.getOpcode() == AArch64::ADDSWrs || 234 Inst.getOpcode() == AArch64::ADDSWrx || 235 Inst.getOpcode() == AArch64::ADDSXri || 236 Inst.getOpcode() == AArch64::ADDSXrr || 237 Inst.getOpcode() == AArch64::ADDSXrs || 238 Inst.getOpcode() == AArch64::ADDSXrx || 239 Inst.getOpcode() == AArch64::ADDSXrx64 || 240 Inst.getOpcode() == AArch64::ADDWri || 241 Inst.getOpcode() == AArch64::ADDWrr || 242 Inst.getOpcode() == AArch64::ADDWrs || 243 Inst.getOpcode() == AArch64::ADDWrx || 244 Inst.getOpcode() == AArch64::ADDXri || 245 Inst.getOpcode() == AArch64::ADDXrr || 246 Inst.getOpcode() == AArch64::ADDXrs || 247 Inst.getOpcode() == AArch64::ADDXrx || 248 Inst.getOpcode() == AArch64::ADDXrx64); 249 } 250 251 bool isLDRB(const MCInst &Inst) const { 252 const unsigned opcode = Inst.getOpcode(); 253 switch (opcode) { 254 case AArch64::LDRBpost: 255 case AArch64::LDRBBpost: 256 case AArch64::LDRBBpre: 257 case AArch64::LDRBBroW: 258 case AArch64::LDRBroW: 259 case AArch64::LDRBroX: 260 case AArch64::LDRBBroX: 261 case AArch64::LDRBBui: 262 case AArch64::LDRBui: 263 case AArch64::LDRBpre: 264 case AArch64::LDRSBWpost: 265 case AArch64::LDRSBWpre: 266 case AArch64::LDRSBWroW: 267 case AArch64::LDRSBWroX: 268 case AArch64::LDRSBWui: 269 case AArch64::LDRSBXpost: 270 case AArch64::LDRSBXpre: 271 case AArch64::LDRSBXroW: 272 case AArch64::LDRSBXroX: 273 case AArch64::LDRSBXui: 274 case AArch64::LDURBi: 275 case AArch64::LDURBBi: 276 case AArch64::LDURSBWi: 277 case AArch64::LDURSBXi: 278 case AArch64::LDTRBi: 279 case AArch64::LDTRSBWi: 280 case AArch64::LDTRSBXi: 281 return true; 282 default: 283 break; 284 } 285 286 return false; 287 } 288 289 bool isLDRH(const MCInst &Inst) const { 290 const unsigned opcode = Inst.getOpcode(); 291 switch (opcode) { 292 case AArch64::LDRHpost: 293 case AArch64::LDRHHpost: 294 case AArch64::LDRHHpre: 295 case AArch64::LDRHroW: 296 case AArch64::LDRHHroW: 297 case AArch64::LDRHroX: 298 case AArch64::LDRHHroX: 299 case AArch64::LDRHHui: 300 case AArch64::LDRHui: 301 case AArch64::LDRHpre: 302 case AArch64::LDRSHWpost: 303 case AArch64::LDRSHWpre: 304 case AArch64::LDRSHWroW: 305 case AArch64::LDRSHWroX: 306 case AArch64::LDRSHWui: 307 case AArch64::LDRSHXpost: 308 case AArch64::LDRSHXpre: 309 case AArch64::LDRSHXroW: 310 case AArch64::LDRSHXroX: 311 case AArch64::LDRSHXui: 312 case AArch64::LDURHi: 313 case AArch64::LDURHHi: 314 case AArch64::LDURSHWi: 315 case AArch64::LDURSHXi: 316 case AArch64::LDTRHi: 317 case AArch64::LDTRSHWi: 318 case AArch64::LDTRSHXi: 319 return true; 320 default: 321 break; 322 } 323 324 return false; 325 } 326 327 bool isLDRW(const MCInst &Inst) const { 328 const unsigned opcode = Inst.getOpcode(); 329 switch (opcode) { 330 case AArch64::LDRWpost: 331 case AArch64::LDRWpre: 332 case AArch64::LDRWroW: 333 case AArch64::LDRWroX: 334 case AArch64::LDRWui: 335 case AArch64::LDRWl: 336 case AArch64::LDRSWl: 337 case AArch64::LDURWi: 338 case AArch64::LDRSWpost: 339 case AArch64::LDRSWpre: 340 case AArch64::LDRSWroW: 341 case AArch64::LDRSWroX: 342 case AArch64::LDRSWui: 343 case AArch64::LDURSWi: 344 case AArch64::LDTRWi: 345 case AArch64::LDTRSWi: 346 case AArch64::LDPWi: 347 case AArch64::LDPWpost: 348 case AArch64::LDPWpre: 349 case AArch64::LDPSWi: 350 case AArch64::LDPSWpost: 351 case AArch64::LDPSWpre: 352 case AArch64::LDNPWi: 353 return true; 354 default: 355 break; 356 } 357 358 return false; 359 } 360 361 bool isLDRX(const MCInst &Inst) const { 362 const unsigned opcode = Inst.getOpcode(); 363 switch (opcode) { 364 case AArch64::LDRXpost: 365 case AArch64::LDRXpre: 366 case AArch64::LDRXroW: 367 case AArch64::LDRXroX: 368 case AArch64::LDRXui: 369 case AArch64::LDRXl: 370 case AArch64::LDURXi: 371 case AArch64::LDTRXi: 372 case AArch64::LDNPXi: 373 case AArch64::LDPXi: 374 case AArch64::LDPXpost: 375 case AArch64::LDPXpre: 376 return true; 377 default: 378 break; 379 } 380 381 return false; 382 } 383 384 bool isLDRS(const MCInst &Inst) const { 385 const unsigned opcode = Inst.getOpcode(); 386 switch (opcode) { 387 case AArch64::LDRSl: 388 case AArch64::LDRSui: 389 case AArch64::LDRSroW: 390 case AArch64::LDRSroX: 391 case AArch64::LDURSi: 392 case AArch64::LDPSi: 393 case AArch64::LDNPSi: 394 case AArch64::LDRSpre: 395 case AArch64::LDRSpost: 396 case AArch64::LDPSpost: 397 case AArch64::LDPSpre: 398 return true; 399 default: 400 break; 401 } 402 403 return false; 404 } 405 406 bool isLDRD(const MCInst &Inst) const { 407 const unsigned opcode = Inst.getOpcode(); 408 switch (opcode) { 409 case AArch64::LDRDl: 410 case AArch64::LDRDui: 411 case AArch64::LDRDpre: 412 case AArch64::LDRDpost: 413 case AArch64::LDRDroW: 414 case AArch64::LDRDroX: 415 case AArch64::LDURDi: 416 case AArch64::LDPDi: 417 case AArch64::LDNPDi: 418 case AArch64::LDPDpost: 419 case AArch64::LDPDpre: 420 return true; 421 default: 422 break; 423 } 424 425 return false; 426 } 427 428 bool isLDRQ(const MCInst &Inst) const { 429 const unsigned opcode = Inst.getOpcode(); 430 switch (opcode) { 431 case AArch64::LDRQui: 432 case AArch64::LDRQl: 433 case AArch64::LDRQpre: 434 case AArch64::LDRQpost: 435 case AArch64::LDRQroW: 436 case AArch64::LDRQroX: 437 case AArch64::LDURQi: 438 case AArch64::LDPQi: 439 case AArch64::LDNPQi: 440 case AArch64::LDPQpost: 441 case AArch64::LDPQpre: 442 return true; 443 default: 444 break; 445 } 446 447 return false; 448 } 449 450 bool mayLoad(const MCInst &Inst) const override { 451 return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst) || 452 isLDRQ(Inst) || isLDRD(Inst) || isLDRS(Inst); 453 } 454 455 bool isAArch64ExclusiveLoad(const MCInst &Inst) const override { 456 return (Inst.getOpcode() == AArch64::LDXPX || 457 Inst.getOpcode() == AArch64::LDXPW || 458 Inst.getOpcode() == AArch64::LDXRX || 459 Inst.getOpcode() == AArch64::LDXRW || 460 Inst.getOpcode() == AArch64::LDXRH || 461 Inst.getOpcode() == AArch64::LDXRB || 462 Inst.getOpcode() == AArch64::LDAXPX || 463 Inst.getOpcode() == AArch64::LDAXPW || 464 Inst.getOpcode() == AArch64::LDAXRX || 465 Inst.getOpcode() == AArch64::LDAXRW || 466 Inst.getOpcode() == AArch64::LDAXRH || 467 Inst.getOpcode() == AArch64::LDAXRB); 468 } 469 470 bool isAArch64ExclusiveStore(const MCInst &Inst) const override { 471 return (Inst.getOpcode() == AArch64::STXPX || 472 Inst.getOpcode() == AArch64::STXPW || 473 Inst.getOpcode() == AArch64::STXRX || 474 Inst.getOpcode() == AArch64::STXRW || 475 Inst.getOpcode() == AArch64::STXRH || 476 Inst.getOpcode() == AArch64::STXRB || 477 Inst.getOpcode() == AArch64::STLXPX || 478 Inst.getOpcode() == AArch64::STLXPW || 479 Inst.getOpcode() == AArch64::STLXRX || 480 Inst.getOpcode() == AArch64::STLXRW || 481 Inst.getOpcode() == AArch64::STLXRH || 482 Inst.getOpcode() == AArch64::STLXRB); 483 } 484 485 bool isAArch64ExclusiveClear(const MCInst &Inst) const override { 486 return (Inst.getOpcode() == AArch64::CLREX); 487 } 488 489 bool isLoadFromStack(const MCInst &Inst) const { 490 if (!mayLoad(Inst)) 491 return false; 492 for (const MCOperand &Operand : useOperands(Inst)) { 493 if (!Operand.isReg()) 494 continue; 495 unsigned Reg = Operand.getReg(); 496 if (Reg == AArch64::SP || Reg == AArch64::WSP) 497 return true; 498 } 499 return false; 500 } 501 502 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 503 MCPhysReg &To) const override { 504 if (Inst.getOpcode() == AArch64::FMOVDXr) { 505 From = Inst.getOperand(1).getReg(); 506 To = Inst.getOperand(0).getReg(); 507 return true; 508 } 509 510 if (Inst.getOpcode() != AArch64::ORRXrs) 511 return false; 512 if (Inst.getOperand(1).getReg() != AArch64::XZR) 513 return false; 514 if (Inst.getOperand(3).getImm() != 0) 515 return false; 516 From = Inst.getOperand(2).getReg(); 517 To = Inst.getOperand(0).getReg(); 518 return true; 519 } 520 521 bool isIndirectCall(const MCInst &Inst) const override { 522 return Inst.getOpcode() == AArch64::BLR; 523 } 524 525 MCPhysReg getSpRegister(int Size) const { 526 switch (Size) { 527 case 4: 528 return AArch64::WSP; 529 case 8: 530 return AArch64::SP; 531 default: 532 llvm_unreachable("Unexpected size"); 533 } 534 } 535 536 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 537 switch (ArgNo) { 538 case 0: 539 return AArch64::X0; 540 case 1: 541 return AArch64::X1; 542 case 2: 543 return AArch64::X2; 544 case 3: 545 return AArch64::X3; 546 case 4: 547 return AArch64::X4; 548 case 5: 549 return AArch64::X5; 550 case 6: 551 return AArch64::X6; 552 case 7: 553 return AArch64::X7; 554 default: 555 return getNoRegister(); 556 } 557 } 558 559 bool hasPCRelOperand(const MCInst &Inst) const override { 560 // ADRP is blacklisted and is an exception. Even though it has a 561 // PC-relative operand, this operand is not a complete symbol reference 562 // and BOLT shouldn't try to process it in isolation. 563 if (isADRP(Inst)) 564 return false; 565 566 if (isADR(Inst)) 567 return true; 568 569 // Look for literal addressing mode (see C1-143 ARM DDI 0487B.a) 570 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 571 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) 572 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL) 573 return true; 574 575 return false; 576 } 577 578 bool evaluateADR(const MCInst &Inst, int64_t &Imm, 579 const MCExpr **DispExpr) const { 580 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction"); 581 582 const MCOperand &Label = Inst.getOperand(1); 583 if (!Label.isImm()) { 584 assert(Label.isExpr() && "Unexpected ADR operand"); 585 assert(DispExpr && "DispExpr must be set"); 586 *DispExpr = Label.getExpr(); 587 return false; 588 } 589 590 if (Inst.getOpcode() == AArch64::ADR) { 591 Imm = Label.getImm(); 592 return true; 593 } 594 Imm = Label.getImm() << 12; 595 return true; 596 } 597 598 bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm, 599 const MCExpr **DispExpr = nullptr) const { 600 if (isADR(Inst) || isADRP(Inst)) 601 return evaluateADR(Inst, DispImm, DispExpr); 602 603 // Literal addressing mode 604 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 605 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { 606 if (MCII.operands()[I].OperandType != MCOI::OPERAND_PCREL) 607 continue; 608 609 if (!Inst.getOperand(I).isImm()) { 610 assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand"); 611 assert(DispExpr && "DispExpr must be set"); 612 *DispExpr = Inst.getOperand(I).getExpr(); 613 return true; 614 } 615 616 DispImm = Inst.getOperand(I).getImm() * 4; 617 return true; 618 } 619 return false; 620 } 621 622 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 623 uint64_t Address, 624 uint64_t Size) const override { 625 int64_t DispValue; 626 const MCExpr *DispExpr = nullptr; 627 if (!evaluateAArch64MemoryOperand(Inst, DispValue, &DispExpr)) 628 return false; 629 630 // Make sure it's a well-formed addressing we can statically evaluate. 631 if (DispExpr) 632 return false; 633 634 Target = DispValue; 635 if (Inst.getOpcode() == AArch64::ADRP) 636 Target += Address & ~0xFFFULL; 637 else 638 Target += Address; 639 return true; 640 } 641 642 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 643 MCInst::iterator OI = Inst.begin(); 644 if (isADR(Inst) || isADRP(Inst)) { 645 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 && 646 "Unexpected number of operands"); 647 return ++OI; 648 } 649 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 650 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { 651 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL) 652 break; 653 ++OI; 654 } 655 assert(OI != Inst.end() && "Literal operand not found"); 656 return OI; 657 } 658 659 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 660 MCInst::iterator OI = getMemOperandDisp(Inst); 661 *OI = Operand; 662 return true; 663 } 664 665 void getCalleeSavedRegs(BitVector &Regs) const override { 666 Regs |= getAliases(AArch64::X18); 667 Regs |= getAliases(AArch64::X19); 668 Regs |= getAliases(AArch64::X20); 669 Regs |= getAliases(AArch64::X21); 670 Regs |= getAliases(AArch64::X22); 671 Regs |= getAliases(AArch64::X23); 672 Regs |= getAliases(AArch64::X24); 673 Regs |= getAliases(AArch64::X25); 674 Regs |= getAliases(AArch64::X26); 675 Regs |= getAliases(AArch64::X27); 676 Regs |= getAliases(AArch64::X28); 677 Regs |= getAliases(AArch64::LR); 678 Regs |= getAliases(AArch64::FP); 679 } 680 681 const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr, 682 MCContext &Ctx, 683 uint64_t RelType) const override { 684 685 if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 || 686 RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) { 687 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx); 688 } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 || 689 RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC || 690 RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 || 691 RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 || 692 RelType == ELF::R_AARCH64_ADR_GOT_PAGE) { 693 // Never emit a GOT reloc, we handled this in 694 // RewriteInstance::readRelocations(). 695 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); 696 } else { 697 switch (RelType) { 698 case ELF::R_AARCH64_ADD_ABS_LO12_NC: 699 case ELF::R_AARCH64_LD64_GOT_LO12_NC: 700 case ELF::R_AARCH64_LDST8_ABS_LO12_NC: 701 case ELF::R_AARCH64_LDST16_ABS_LO12_NC: 702 case ELF::R_AARCH64_LDST32_ABS_LO12_NC: 703 case ELF::R_AARCH64_LDST64_ABS_LO12_NC: 704 case ELF::R_AARCH64_LDST128_ABS_LO12_NC: 705 case ELF::R_AARCH64_TLSDESC_ADD_LO12: 706 case ELF::R_AARCH64_TLSDESC_LD64_LO12: 707 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 708 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 709 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx); 710 case ELF::R_AARCH64_MOVW_UABS_G3: 711 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G3, Ctx); 712 case ELF::R_AARCH64_MOVW_UABS_G2: 713 case ELF::R_AARCH64_MOVW_UABS_G2_NC: 714 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G2_NC, Ctx); 715 case ELF::R_AARCH64_MOVW_UABS_G1: 716 case ELF::R_AARCH64_MOVW_UABS_G1_NC: 717 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G1_NC, Ctx); 718 case ELF::R_AARCH64_MOVW_UABS_G0: 719 case ELF::R_AARCH64_MOVW_UABS_G0_NC: 720 return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G0_NC, Ctx); 721 default: 722 break; 723 } 724 } 725 return Expr; 726 } 727 728 bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const { 729 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 730 return false; 731 732 // Auto-select correct operand number 733 if (OpNum == 0) { 734 if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst) || 735 isMOVW(Inst)) 736 OpNum = 1; 737 if (isTB(Inst) || isAddXri(Inst)) 738 OpNum = 2; 739 } 740 741 return true; 742 } 743 744 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 745 auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr); 746 if (AArchExpr && AArchExpr->getSubExpr()) 747 return getTargetSymbol(AArchExpr->getSubExpr()); 748 749 auto *BinExpr = dyn_cast<MCBinaryExpr>(Expr); 750 if (BinExpr) 751 return getTargetSymbol(BinExpr->getLHS()); 752 753 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Expr); 754 if (SymExpr && SymExpr->getKind() == MCSymbolRefExpr::VK_None) 755 return &SymExpr->getSymbol(); 756 757 return nullptr; 758 } 759 760 const MCSymbol *getTargetSymbol(const MCInst &Inst, 761 unsigned OpNum = 0) const override { 762 if (!getSymbolRefOperandNum(Inst, OpNum)) 763 return nullptr; 764 765 const MCOperand &Op = Inst.getOperand(OpNum); 766 if (!Op.isExpr()) 767 return nullptr; 768 769 return getTargetSymbol(Op.getExpr()); 770 } 771 772 int64_t getTargetAddend(const MCExpr *Expr) const override { 773 auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr); 774 if (AArchExpr && AArchExpr->getSubExpr()) 775 return getTargetAddend(AArchExpr->getSubExpr()); 776 777 auto *BinExpr = dyn_cast<MCBinaryExpr>(Expr); 778 if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add) 779 return getTargetAddend(BinExpr->getRHS()); 780 781 auto *ConstExpr = dyn_cast<MCConstantExpr>(Expr); 782 if (ConstExpr) 783 return ConstExpr->getValue(); 784 785 return 0; 786 } 787 788 int64_t getTargetAddend(const MCInst &Inst, 789 unsigned OpNum = 0) const override { 790 if (!getSymbolRefOperandNum(Inst, OpNum)) 791 return 0; 792 793 const MCOperand &Op = Inst.getOperand(OpNum); 794 if (!Op.isExpr()) 795 return 0; 796 797 return getTargetAddend(Op.getExpr()); 798 } 799 800 void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 801 MCContext *Ctx) const override { 802 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 803 "Invalid instruction"); 804 assert(MCPlus::getNumPrimeOperands(Inst) >= 1 && 805 "Invalid number of operands"); 806 MCInst::iterator OI = Inst.begin(); 807 808 if (isConditionalBranch(Inst)) { 809 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 && 810 "Invalid number of operands"); 811 ++OI; 812 } 813 814 if (isTB(Inst)) { 815 assert(MCPlus::getNumPrimeOperands(Inst) >= 3 && 816 "Invalid number of operands"); 817 OI = Inst.begin() + 2; 818 } 819 820 *OI = MCOperand::createExpr( 821 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 822 } 823 824 /// Matches indirect branch patterns in AArch64 related to a jump table (JT), 825 /// helping us to build the complete CFG. A typical indirect branch to 826 /// a jump table entry in AArch64 looks like the following: 827 /// 828 /// adrp x1, #-7585792 # Get JT Page location 829 /// add x1, x1, #692 # Complement with JT Page offset 830 /// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry 831 /// adr x1, #12 # Get PC + 12 (end of this BB) used next 832 /// add x0, x1, w0, sxth #2 # Finish building branch target 833 /// # (entries in JT are relative to the end 834 /// # of this BB) 835 /// br x0 # Indirect jump instruction 836 /// 837 /// Return true on successful jump table instruction sequence match, false 838 /// otherwise. 839 bool analyzeIndirectBranchFragment( 840 const MCInst &Inst, 841 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain, 842 const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue, 843 MCInst *&PCRelBase) const { 844 // Expect AArch64 BR 845 assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode"); 846 847 JumpTable = nullptr; 848 849 // Match the indirect branch pattern for aarch64 850 SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst]; 851 if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr) 852 return false; 853 854 const MCInst *DefAdd = UsesRoot[0]; 855 856 // Now we match an ADD 857 if (!isADD(*DefAdd)) { 858 // If the address is not broken up in two parts, this is not branching 859 // according to a jump table entry. Fail. 860 return false; 861 } 862 if (DefAdd->getOpcode() == AArch64::ADDXri) { 863 // This can happen when there is no offset, but a direct jump that was 864 // transformed into an indirect one (indirect tail call) : 865 // ADRP x2, Perl_re_compiler 866 // ADD x2, x2, :lo12:Perl_re_compiler 867 // BR x2 868 return false; 869 } 870 if (DefAdd->getOpcode() == AArch64::ADDXrs) { 871 // Covers the less common pattern where JT entries are relative to 872 // the JT itself (like x86). Seems less efficient since we can't 873 // assume the JT is aligned at 4B boundary and thus drop 2 bits from 874 // JT values. 875 // cde264: 876 // adrp x12, #21544960 ; 216a000 877 // add x12, x12, #1696 ; 216a6a0 (JT object in .rodata) 878 // ldrsw x8, [x12, x8, lsl #2] --> loads e.g. 0xfeb73bd8 879 // * add x8, x8, x12 --> = cde278, next block 880 // br x8 881 // cde278: 882 // 883 // Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0 884 return false; 885 } 886 if (DefAdd->getOpcode() != AArch64::ADDXrx) 887 return false; 888 889 // Validate ADD operands 890 int64_t OperandExtension = DefAdd->getOperand(3).getImm(); 891 unsigned ShiftVal = AArch64_AM::getArithShiftValue(OperandExtension); 892 AArch64_AM::ShiftExtendType ExtendType = 893 AArch64_AM::getArithExtendType(OperandExtension); 894 if (ShiftVal != 2) { 895 // TODO: Handle the patten where ShiftVal != 2. 896 // The following code sequence below has no shift amount, 897 // the range could be 0 to 4. 898 // The pattern comes from libc, it occurs when the binary is static. 899 // adr x6, 0x219fb0 <sigall_set+0x88> 900 // add x6, x6, x14, lsl #2 901 // ldr w7, [x6] 902 // add x6, x6, w7, sxtw => no shift amount 903 // br x6 904 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: " 905 "failed to match indirect branch: ShiftVAL != 2\n"); 906 return false; 907 } 908 909 if (ExtendType == AArch64_AM::SXTB) 910 ScaleValue = 1LL; 911 else if (ExtendType == AArch64_AM::SXTH) 912 ScaleValue = 2LL; 913 else if (ExtendType == AArch64_AM::SXTW) 914 ScaleValue = 4LL; 915 else 916 return false; 917 918 // Match an ADR to load base address to be used when addressing JT targets 919 SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd]; 920 if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) { 921 // This happens when we don't have enough context about this jump table 922 // because the jumping code sequence was split in multiple basic blocks. 923 // This was observed in the wild in HHVM code (dispatchImpl). 924 return false; 925 } 926 MCInst *DefBaseAddr = UsesAdd[1]; 927 if (DefBaseAddr->getOpcode() != AArch64::ADR) 928 return false; 929 930 PCRelBase = DefBaseAddr; 931 // Match LOAD to load the jump table (relative) target 932 const MCInst *DefLoad = UsesAdd[2]; 933 if (!mayLoad(*DefLoad) || (ScaleValue == 1LL && !isLDRB(*DefLoad)) || 934 (ScaleValue == 2LL && !isLDRH(*DefLoad))) 935 return false; 936 937 // Match ADD that calculates the JumpTable Base Address (not the offset) 938 SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad]; 939 const MCInst *DefJTBaseAdd = UsesLoad[1]; 940 MCPhysReg From, To; 941 if (DefJTBaseAdd == nullptr || isLoadFromStack(*DefJTBaseAdd) || 942 isRegToRegMove(*DefJTBaseAdd, From, To)) { 943 // Sometimes base address may have been defined in another basic block 944 // (hoisted). Return with no jump table info. 945 return true; 946 } 947 948 if (DefJTBaseAdd->getOpcode() == AArch64::ADR) { 949 // TODO: Handle the pattern where there is no adrp/add pair. 950 // It also occurs when the binary is static. 951 // adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50> 952 // ldrh w13, [x13, w12, uxtw #1] 953 // adr x12, 0x247b30 <__gettextparse+0x5b0> 954 // add x13, x12, w13, sxth #2 955 // br x13 956 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match indirect branch: " 957 "nop/adr instead of adrp/add\n"); 958 return false; 959 } 960 961 if (DefJTBaseAdd->getOpcode() != AArch64::ADDXri) { 962 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match jump table base " 963 "address pattern! (1)\n"); 964 return false; 965 } 966 967 if (DefJTBaseAdd->getOperand(2).isImm()) 968 Offset = DefJTBaseAdd->getOperand(2).getImm(); 969 SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd]; 970 const MCInst *DefJTBasePage = UsesJTBaseAdd[1]; 971 if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) { 972 return true; 973 } 974 if (DefJTBasePage->getOpcode() != AArch64::ADRP) 975 return false; 976 977 if (DefJTBasePage->getOperand(1).isExpr()) 978 JumpTable = DefJTBasePage->getOperand(1).getExpr(); 979 return true; 980 } 981 982 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> 983 computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin, 984 InstructionIterator End) const { 985 DenseMap<int, MCInst *> RegAliasTable; 986 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> Uses; 987 988 auto addInstrOperands = [&](const MCInst &Instr) { 989 // Update Uses table 990 for (const MCOperand &Operand : MCPlus::primeOperands(Instr)) { 991 if (!Operand.isReg()) 992 continue; 993 unsigned Reg = Operand.getReg(); 994 MCInst *AliasInst = RegAliasTable[Reg]; 995 Uses[&Instr].push_back(AliasInst); 996 LLVM_DEBUG({ 997 dbgs() << "Adding reg operand " << Reg << " refs "; 998 if (AliasInst != nullptr) 999 AliasInst->dump(); 1000 else 1001 dbgs() << "\n"; 1002 }); 1003 } 1004 }; 1005 1006 LLVM_DEBUG(dbgs() << "computeLocalUDChain\n"); 1007 bool TerminatorSeen = false; 1008 for (auto II = Begin; II != End; ++II) { 1009 MCInst &Instr = *II; 1010 // Ignore nops and CFIs 1011 if (isPseudo(Instr) || isNoop(Instr)) 1012 continue; 1013 if (TerminatorSeen) { 1014 RegAliasTable.clear(); 1015 Uses.clear(); 1016 } 1017 1018 LLVM_DEBUG(dbgs() << "Now updating for:\n "); 1019 LLVM_DEBUG(Instr.dump()); 1020 addInstrOperands(Instr); 1021 1022 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 1023 getWrittenRegs(Instr, Regs); 1024 1025 // Update register definitions after this point 1026 for (int Idx : Regs.set_bits()) { 1027 RegAliasTable[Idx] = &Instr; 1028 LLVM_DEBUG(dbgs() << "Setting reg " << Idx 1029 << " def to current instr.\n"); 1030 } 1031 1032 TerminatorSeen = isTerminator(Instr); 1033 } 1034 1035 // Process the last instruction, which is not currently added into the 1036 // instruction stream 1037 if (CurInstr) 1038 addInstrOperands(*CurInstr); 1039 1040 return Uses; 1041 } 1042 1043 IndirectBranchType 1044 analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin, 1045 InstructionIterator End, const unsigned PtrSize, 1046 MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 1047 unsigned &IndexRegNumOut, int64_t &DispValueOut, 1048 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut, 1049 MCInst *&FixedEntryLoadInstr) const override { 1050 MemLocInstrOut = nullptr; 1051 BaseRegNumOut = AArch64::NoRegister; 1052 IndexRegNumOut = AArch64::NoRegister; 1053 DispValueOut = 0; 1054 DispExprOut = nullptr; 1055 FixedEntryLoadInstr = nullptr; 1056 1057 // An instruction referencing memory used by jump instruction (directly or 1058 // via register). This location could be an array of function pointers 1059 // in case of indirect tail call, or a jump table. 1060 MCInst *MemLocInstr = nullptr; 1061 1062 // Analyze the memory location. 1063 int64_t ScaleValue, DispValue; 1064 const MCExpr *DispExpr; 1065 1066 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain = 1067 computeLocalUDChain(&Instruction, Begin, End); 1068 MCInst *PCRelBase; 1069 if (!analyzeIndirectBranchFragment(Instruction, UDChain, DispExpr, 1070 DispValue, ScaleValue, PCRelBase)) 1071 return IndirectBranchType::UNKNOWN; 1072 1073 MemLocInstrOut = MemLocInstr; 1074 DispValueOut = DispValue; 1075 DispExprOut = DispExpr; 1076 PCRelBaseOut = PCRelBase; 1077 return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE; 1078 } 1079 1080 /// Matches PLT entry pattern and returns the associated GOT entry address. 1081 /// Typical PLT entry looks like the following: 1082 /// 1083 /// adrp x16, 230000 1084 /// ldr x17, [x16, #3040] 1085 /// add x16, x16, #0xbe0 1086 /// br x17 1087 /// 1088 /// The other type of trampolines are located in .plt.got, that are used for 1089 /// non-lazy bindings so doesn't use x16 arg to transfer .got entry address: 1090 /// 1091 /// adrp x16, 230000 1092 /// ldr x17, [x16, #3040] 1093 /// br x17 1094 /// nop 1095 /// 1096 uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin, 1097 InstructionIterator End, 1098 uint64_t BeginPC) const override { 1099 // Check branch instruction 1100 MCInst *Branch = &Instruction; 1101 assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode"); 1102 1103 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain = 1104 computeLocalUDChain(Branch, Begin, End); 1105 1106 // Match ldr instruction 1107 SmallVector<MCInst *, 4> &BranchUses = UDChain[Branch]; 1108 if (BranchUses.size() < 1 || BranchUses[0] == nullptr) 1109 return 0; 1110 1111 // Check ldr instruction 1112 const MCInst *Ldr = BranchUses[0]; 1113 if (Ldr->getOpcode() != AArch64::LDRXui) 1114 return 0; 1115 1116 // Get ldr value 1117 const unsigned ScaleLdr = 8; // LDRX operates on 8 bytes segments 1118 assert(Ldr->getOperand(2).isImm() && "Unexpected ldr operand"); 1119 const uint64_t Offset = Ldr->getOperand(2).getImm() * ScaleLdr; 1120 1121 // Match adrp instruction 1122 SmallVector<MCInst *, 4> &LdrUses = UDChain[Ldr]; 1123 if (LdrUses.size() < 2 || LdrUses[1] == nullptr) 1124 return 0; 1125 1126 // Check adrp instruction 1127 MCInst *Adrp = LdrUses[1]; 1128 if (Adrp->getOpcode() != AArch64::ADRP) 1129 return 0; 1130 1131 // Get adrp instruction PC 1132 const unsigned InstSize = 4; 1133 uint64_t AdrpPC = BeginPC; 1134 for (InstructionIterator It = Begin; It != End; ++It) { 1135 if (&(*It) == Adrp) 1136 break; 1137 AdrpPC += InstSize; 1138 } 1139 1140 // Get adrp value 1141 uint64_t Base; 1142 assert(Adrp->getOperand(1).isImm() && "Unexpected adrp operand"); 1143 bool Ret = evaluateMemOperandTarget(*Adrp, Base, AdrpPC, InstSize); 1144 assert(Ret && "Failed to evaluate adrp"); 1145 (void)Ret; 1146 1147 return Base + Offset; 1148 } 1149 1150 unsigned getInvertedBranchOpcode(unsigned Opcode) const { 1151 switch (Opcode) { 1152 default: 1153 llvm_unreachable("Failed to invert branch opcode"); 1154 return Opcode; 1155 case AArch64::TBZW: return AArch64::TBNZW; 1156 case AArch64::TBZX: return AArch64::TBNZX; 1157 case AArch64::TBNZW: return AArch64::TBZW; 1158 case AArch64::TBNZX: return AArch64::TBZX; 1159 case AArch64::CBZW: return AArch64::CBNZW; 1160 case AArch64::CBZX: return AArch64::CBNZX; 1161 case AArch64::CBNZW: return AArch64::CBZW; 1162 case AArch64::CBNZX: return AArch64::CBZX; 1163 } 1164 } 1165 1166 unsigned getCondCode(const MCInst &Inst) const override { 1167 // AArch64 does not use conditional codes, so we just return the opcode 1168 // of the conditional branch here. 1169 return Inst.getOpcode(); 1170 } 1171 1172 unsigned getCanonicalBranchCondCode(unsigned Opcode) const override { 1173 switch (Opcode) { 1174 default: 1175 return Opcode; 1176 case AArch64::TBNZW: return AArch64::TBZW; 1177 case AArch64::TBNZX: return AArch64::TBZX; 1178 case AArch64::CBNZW: return AArch64::CBZW; 1179 case AArch64::CBNZX: return AArch64::CBZX; 1180 } 1181 } 1182 1183 void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 1184 MCContext *Ctx) const override { 1185 if (isTB(Inst) || isCB(Inst)) { 1186 Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode())); 1187 assert(Inst.getOpcode() != 0 && "Invalid branch instruction"); 1188 } else if (Inst.getOpcode() == AArch64::Bcc) { 1189 Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode( 1190 static_cast<AArch64CC::CondCode>(Inst.getOperand(0).getImm()))); 1191 assert(Inst.getOperand(0).getImm() != AArch64CC::AL && 1192 Inst.getOperand(0).getImm() != AArch64CC::NV && 1193 "Can't reverse ALWAYS cond code"); 1194 } else { 1195 LLVM_DEBUG(Inst.dump()); 1196 llvm_unreachable("Unrecognized branch instruction"); 1197 } 1198 replaceBranchTarget(Inst, TBB, Ctx); 1199 } 1200 1201 int getPCRelEncodingSize(const MCInst &Inst) const override { 1202 switch (Inst.getOpcode()) { 1203 default: 1204 llvm_unreachable("Failed to get pcrel encoding size"); 1205 return 0; 1206 case AArch64::TBZW: return 16; 1207 case AArch64::TBZX: return 16; 1208 case AArch64::TBNZW: return 16; 1209 case AArch64::TBNZX: return 16; 1210 case AArch64::CBZW: return 21; 1211 case AArch64::CBZX: return 21; 1212 case AArch64::CBNZW: return 21; 1213 case AArch64::CBNZX: return 21; 1214 case AArch64::B: return 28; 1215 case AArch64::BL: return 28; 1216 case AArch64::Bcc: return 21; 1217 } 1218 } 1219 1220 int getShortJmpEncodingSize() const override { return 33; } 1221 1222 int getUncondBranchEncodingSize() const override { return 28; } 1223 1224 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 1225 const MCSymbol *Target, 1226 MCContext *Ctx) const override { 1227 InstructionListType Code; 1228 Code.emplace_back(MCInstBuilder(AArch64::SUBSXri) 1229 .addReg(RegNo) 1230 .addReg(RegNo) 1231 .addImm(Imm) 1232 .addImm(0)); 1233 Code.emplace_back(MCInstBuilder(AArch64::Bcc) 1234 .addImm(Imm) 1235 .addExpr(MCSymbolRefExpr::create( 1236 Target, MCSymbolRefExpr::VK_None, *Ctx))); 1237 return Code; 1238 } 1239 1240 void createTailCall(MCInst &Inst, const MCSymbol *Target, 1241 MCContext *Ctx) override { 1242 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 1243 } 1244 1245 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 1246 MCContext *Ctx) override { 1247 createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true); 1248 } 1249 1250 void createTrap(MCInst &Inst) const override { 1251 Inst.clear(); 1252 Inst.setOpcode(AArch64::BRK); 1253 Inst.addOperand(MCOperand::createImm(1)); 1254 } 1255 1256 bool convertJmpToTailCall(MCInst &Inst) override { 1257 setTailCall(Inst); 1258 return true; 1259 } 1260 1261 bool convertTailCallToJmp(MCInst &Inst) override { 1262 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1263 clearOffset(Inst); 1264 if (getConditionalTailCall(Inst)) 1265 unsetConditionalTailCall(Inst); 1266 return true; 1267 } 1268 1269 InstructionListType createIndirectPLTCall(MCInst &&DirectCall, 1270 const MCSymbol *TargetLocation, 1271 MCContext *Ctx) override { 1272 const bool IsTailCall = isTailCall(DirectCall); 1273 assert((DirectCall.getOpcode() == AArch64::BL || 1274 (DirectCall.getOpcode() == AArch64::B && IsTailCall)) && 1275 "64-bit direct (tail) call instruction expected"); 1276 1277 InstructionListType Code; 1278 // Code sequence for indirect plt call: 1279 // adrp x16 <symbol> 1280 // ldr x17, [x16, #<offset>] 1281 // blr x17 ; or 'br' for tail calls 1282 1283 MCInst InstAdrp; 1284 InstAdrp.setOpcode(AArch64::ADRP); 1285 InstAdrp.addOperand(MCOperand::createReg(AArch64::X16)); 1286 InstAdrp.addOperand(MCOperand::createImm(0)); 1287 setOperandToSymbolRef(InstAdrp, /* OpNum */ 1, TargetLocation, 1288 /* Addend */ 0, Ctx, ELF::R_AARCH64_ADR_GOT_PAGE); 1289 Code.emplace_back(InstAdrp); 1290 1291 MCInst InstLoad; 1292 InstLoad.setOpcode(AArch64::LDRXui); 1293 InstLoad.addOperand(MCOperand::createReg(AArch64::X17)); 1294 InstLoad.addOperand(MCOperand::createReg(AArch64::X16)); 1295 InstLoad.addOperand(MCOperand::createImm(0)); 1296 setOperandToSymbolRef(InstLoad, /* OpNum */ 2, TargetLocation, 1297 /* Addend */ 0, Ctx, ELF::R_AARCH64_LD64_GOT_LO12_NC); 1298 Code.emplace_back(InstLoad); 1299 1300 MCInst InstCall; 1301 InstCall.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR); 1302 InstCall.addOperand(MCOperand::createReg(AArch64::X17)); 1303 moveAnnotations(std::move(DirectCall), InstCall); 1304 Code.emplace_back(InstCall); 1305 1306 return Code; 1307 } 1308 1309 bool lowerTailCall(MCInst &Inst) override { 1310 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1311 if (getConditionalTailCall(Inst)) 1312 unsetConditionalTailCall(Inst); 1313 return true; 1314 } 1315 1316 bool isNoop(const MCInst &Inst) const override { 1317 return Inst.getOpcode() == AArch64::HINT && 1318 Inst.getOperand(0).getImm() == 0; 1319 } 1320 1321 void createNoop(MCInst &Inst) const override { 1322 Inst.setOpcode(AArch64::HINT); 1323 Inst.clear(); 1324 Inst.addOperand(MCOperand::createImm(0)); 1325 } 1326 1327 bool isStorePair(const MCInst &Inst) const { 1328 const unsigned opcode = Inst.getOpcode(); 1329 1330 auto isStorePairImmOffset = [&]() { 1331 switch (opcode) { 1332 case AArch64::STPWi: 1333 case AArch64::STPXi: 1334 case AArch64::STPSi: 1335 case AArch64::STPDi: 1336 case AArch64::STPQi: 1337 case AArch64::STNPWi: 1338 case AArch64::STNPXi: 1339 case AArch64::STNPSi: 1340 case AArch64::STNPDi: 1341 case AArch64::STNPQi: 1342 return true; 1343 default: 1344 break; 1345 } 1346 1347 return false; 1348 }; 1349 1350 auto isStorePairPostIndex = [&]() { 1351 switch (opcode) { 1352 case AArch64::STPWpost: 1353 case AArch64::STPXpost: 1354 case AArch64::STPSpost: 1355 case AArch64::STPDpost: 1356 case AArch64::STPQpost: 1357 return true; 1358 default: 1359 break; 1360 } 1361 1362 return false; 1363 }; 1364 1365 auto isStorePairPreIndex = [&]() { 1366 switch (opcode) { 1367 case AArch64::STPWpre: 1368 case AArch64::STPXpre: 1369 case AArch64::STPSpre: 1370 case AArch64::STPDpre: 1371 case AArch64::STPQpre: 1372 return true; 1373 default: 1374 break; 1375 } 1376 1377 return false; 1378 }; 1379 1380 return isStorePairImmOffset() || isStorePairPostIndex() || 1381 isStorePairPreIndex(); 1382 } 1383 1384 bool isStoreReg(const MCInst &Inst) const { 1385 const unsigned opcode = Inst.getOpcode(); 1386 1387 auto isStoreRegUnscaleImm = [&]() { 1388 switch (opcode) { 1389 case AArch64::STURBi: 1390 case AArch64::STURBBi: 1391 case AArch64::STURHi: 1392 case AArch64::STURHHi: 1393 case AArch64::STURWi: 1394 case AArch64::STURXi: 1395 case AArch64::STURSi: 1396 case AArch64::STURDi: 1397 case AArch64::STURQi: 1398 return true; 1399 default: 1400 break; 1401 } 1402 1403 return false; 1404 }; 1405 1406 auto isStoreRegScaledImm = [&]() { 1407 switch (opcode) { 1408 case AArch64::STRBui: 1409 case AArch64::STRBBui: 1410 case AArch64::STRHui: 1411 case AArch64::STRHHui: 1412 case AArch64::STRWui: 1413 case AArch64::STRXui: 1414 case AArch64::STRSui: 1415 case AArch64::STRDui: 1416 case AArch64::STRQui: 1417 return true; 1418 default: 1419 break; 1420 } 1421 1422 return false; 1423 }; 1424 1425 auto isStoreRegImmPostIndexed = [&]() { 1426 switch (opcode) { 1427 case AArch64::STRBpost: 1428 case AArch64::STRBBpost: 1429 case AArch64::STRHpost: 1430 case AArch64::STRHHpost: 1431 case AArch64::STRWpost: 1432 case AArch64::STRXpost: 1433 case AArch64::STRSpost: 1434 case AArch64::STRDpost: 1435 case AArch64::STRQpost: 1436 return true; 1437 default: 1438 break; 1439 } 1440 1441 return false; 1442 }; 1443 1444 auto isStoreRegImmPreIndexed = [&]() { 1445 switch (opcode) { 1446 case AArch64::STRBpre: 1447 case AArch64::STRBBpre: 1448 case AArch64::STRHpre: 1449 case AArch64::STRHHpre: 1450 case AArch64::STRWpre: 1451 case AArch64::STRXpre: 1452 case AArch64::STRSpre: 1453 case AArch64::STRDpre: 1454 case AArch64::STRQpre: 1455 return true; 1456 default: 1457 break; 1458 } 1459 1460 return false; 1461 }; 1462 1463 auto isStoreRegUnscaleUnpriv = [&]() { 1464 switch (opcode) { 1465 case AArch64::STTRBi: 1466 case AArch64::STTRHi: 1467 case AArch64::STTRWi: 1468 case AArch64::STTRXi: 1469 return true; 1470 default: 1471 break; 1472 } 1473 1474 return false; 1475 }; 1476 1477 auto isStoreRegTrunc = [&]() { 1478 switch (opcode) { 1479 case AArch64::STRBBroW: 1480 case AArch64::STRBBroX: 1481 case AArch64::STRBroW: 1482 case AArch64::STRBroX: 1483 case AArch64::STRDroW: 1484 case AArch64::STRDroX: 1485 case AArch64::STRHHroW: 1486 case AArch64::STRHHroX: 1487 case AArch64::STRHroW: 1488 case AArch64::STRHroX: 1489 case AArch64::STRQroW: 1490 case AArch64::STRQroX: 1491 case AArch64::STRSroW: 1492 case AArch64::STRSroX: 1493 case AArch64::STRWroW: 1494 case AArch64::STRWroX: 1495 case AArch64::STRXroW: 1496 case AArch64::STRXroX: 1497 return true; 1498 default: 1499 break; 1500 } 1501 1502 return false; 1503 }; 1504 1505 return isStoreRegUnscaleImm() || isStoreRegScaledImm() || 1506 isStoreRegImmPreIndexed() || isStoreRegImmPostIndexed() || 1507 isStoreRegUnscaleUnpriv() || isStoreRegTrunc(); 1508 } 1509 1510 bool mayStore(const MCInst &Inst) const override { 1511 return isStorePair(Inst) || isStoreReg(Inst) || 1512 isAArch64ExclusiveStore(Inst); 1513 } 1514 1515 bool isStoreToStack(const MCInst &Inst) const { 1516 if (!mayStore(Inst)) 1517 return false; 1518 1519 for (const MCOperand &Operand : useOperands(Inst)) { 1520 if (!Operand.isReg()) 1521 continue; 1522 1523 unsigned Reg = Operand.getReg(); 1524 if (Reg == AArch64::SP || Reg == AArch64::WSP) 1525 return true; 1526 } 1527 1528 return false; 1529 } 1530 1531 void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 1532 bool IsTailCall) override { 1533 Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL); 1534 Inst.clear(); 1535 Inst.addOperand(MCOperand::createExpr(getTargetExprFor( 1536 Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), 1537 *Ctx, 0))); 1538 if (IsTailCall) 1539 convertJmpToTailCall(Inst); 1540 } 1541 1542 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1543 const MCSymbol *&TBB, const MCSymbol *&FBB, 1544 MCInst *&CondBranch, 1545 MCInst *&UncondBranch) const override { 1546 auto I = End; 1547 1548 while (I != Begin) { 1549 --I; 1550 1551 // Ignore nops and CFIs 1552 if (isPseudo(*I) || isNoop(*I)) 1553 continue; 1554 1555 // Stop when we find the first non-terminator 1556 if (!isTerminator(*I) || isTailCall(*I) || !isBranch(*I)) 1557 break; 1558 1559 // Handle unconditional branches. 1560 if (isUnconditionalBranch(*I)) { 1561 // If any code was seen after this unconditional branch, we've seen 1562 // unreachable code. Ignore them. 1563 CondBranch = nullptr; 1564 UncondBranch = &*I; 1565 const MCSymbol *Sym = getTargetSymbol(*I); 1566 assert(Sym != nullptr && 1567 "Couldn't extract BB symbol from jump operand"); 1568 TBB = Sym; 1569 continue; 1570 } 1571 1572 // Handle conditional branches and ignore indirect branches 1573 if (isIndirectBranch(*I)) 1574 return false; 1575 1576 if (CondBranch == nullptr) { 1577 const MCSymbol *TargetBB = getTargetSymbol(*I); 1578 if (TargetBB == nullptr) { 1579 // Unrecognized branch target 1580 return false; 1581 } 1582 FBB = TBB; 1583 TBB = TargetBB; 1584 CondBranch = &*I; 1585 continue; 1586 } 1587 1588 llvm_unreachable("multiple conditional branches in one BB"); 1589 } 1590 return true; 1591 } 1592 1593 void createLongJmp(InstructionListType &Seq, const MCSymbol *Target, 1594 MCContext *Ctx, bool IsTailCall) override { 1595 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call 1596 // Standard for the ARM 64-bit Architecture (AArch64)". 1597 // The sequence of instructions we create here is the following: 1598 // movz ip0, #:abs_g3:<addr> 1599 // movk ip0, #:abs_g2_nc:<addr> 1600 // movk ip0, #:abs_g1_nc:<addr> 1601 // movk ip0, #:abs_g0_nc:<addr> 1602 // br ip0 1603 MCInst Inst; 1604 Inst.setOpcode(AArch64::MOVZXi); 1605 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1606 Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( 1607 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), 1608 AArch64MCExpr::VK_ABS_G3, *Ctx))); 1609 Inst.addOperand(MCOperand::createImm(0x30)); 1610 Seq.emplace_back(Inst); 1611 1612 Inst.clear(); 1613 Inst.setOpcode(AArch64::MOVKXi); 1614 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1615 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1616 Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( 1617 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), 1618 AArch64MCExpr::VK_ABS_G2_NC, *Ctx))); 1619 Inst.addOperand(MCOperand::createImm(0x20)); 1620 Seq.emplace_back(Inst); 1621 1622 Inst.clear(); 1623 Inst.setOpcode(AArch64::MOVKXi); 1624 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1625 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1626 Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( 1627 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), 1628 AArch64MCExpr::VK_ABS_G1_NC, *Ctx))); 1629 Inst.addOperand(MCOperand::createImm(0x10)); 1630 Seq.emplace_back(Inst); 1631 1632 Inst.clear(); 1633 Inst.setOpcode(AArch64::MOVKXi); 1634 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1635 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1636 Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( 1637 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), 1638 AArch64MCExpr::VK_ABS_G0_NC, *Ctx))); 1639 Inst.addOperand(MCOperand::createImm(0)); 1640 Seq.emplace_back(Inst); 1641 1642 Inst.clear(); 1643 Inst.setOpcode(AArch64::BR); 1644 Inst.addOperand(MCOperand::createReg(AArch64::X16)); 1645 if (IsTailCall) 1646 setTailCall(Inst); 1647 Seq.emplace_back(Inst); 1648 } 1649 1650 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 1651 MCContext *Ctx, bool IsTailCall) override { 1652 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call 1653 // Standard for the ARM 64-bit Architecture (AArch64)". 1654 // The sequence of instructions we create here is the following: 1655 // adrp ip0, imm 1656 // add ip0, ip0, imm 1657 // br ip0 1658 MCPhysReg Reg = AArch64::X16; 1659 InstructionListType Insts = materializeAddress(Target, Ctx, Reg); 1660 Insts.emplace_back(); 1661 MCInst &Inst = Insts.back(); 1662 Inst.clear(); 1663 Inst.setOpcode(AArch64::BR); 1664 Inst.addOperand(MCOperand::createReg(Reg)); 1665 if (IsTailCall) 1666 setTailCall(Inst); 1667 Seq.swap(Insts); 1668 } 1669 1670 /// Matching pattern here is 1671 /// 1672 /// ADRP x16, imm 1673 /// ADD x16, x16, imm 1674 /// BR x16 1675 /// 1676 uint64_t matchLinkerVeneer(InstructionIterator Begin, InstructionIterator End, 1677 uint64_t Address, const MCInst &CurInst, 1678 MCInst *&TargetHiBits, MCInst *&TargetLowBits, 1679 uint64_t &Target) const override { 1680 if (CurInst.getOpcode() != AArch64::BR || !CurInst.getOperand(0).isReg() || 1681 CurInst.getOperand(0).getReg() != AArch64::X16) 1682 return 0; 1683 1684 auto I = End; 1685 if (I == Begin) 1686 return 0; 1687 1688 --I; 1689 Address -= 4; 1690 if (I == Begin || I->getOpcode() != AArch64::ADDXri || 1691 MCPlus::getNumPrimeOperands(*I) < 3 || !I->getOperand(0).isReg() || 1692 !I->getOperand(1).isReg() || 1693 I->getOperand(0).getReg() != AArch64::X16 || 1694 I->getOperand(1).getReg() != AArch64::X16 || !I->getOperand(2).isImm()) 1695 return 0; 1696 TargetLowBits = &*I; 1697 uint64_t Addr = I->getOperand(2).getImm() & 0xFFF; 1698 1699 --I; 1700 Address -= 4; 1701 if (I->getOpcode() != AArch64::ADRP || 1702 MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() || 1703 !I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16) 1704 return 0; 1705 TargetHiBits = &*I; 1706 Addr |= (Address + ((int64_t)I->getOperand(1).getImm() << 12)) & 1707 0xFFFFFFFFFFFFF000ULL; 1708 Target = Addr; 1709 return 3; 1710 } 1711 1712 /// Match the following pattern: 1713 /// 1714 /// LDR x16, .L1 1715 /// BR x16 1716 /// L1: 1717 /// .quad Target 1718 /// 1719 /// Populate \p TargetAddress with the Target value on successful match. 1720 bool matchAbsLongVeneer(const BinaryFunction &BF, 1721 uint64_t &TargetAddress) const override { 1722 if (BF.size() != 1 || BF.getMaxSize() < 16) 1723 return false; 1724 1725 if (!BF.hasConstantIsland()) 1726 return false; 1727 1728 const BinaryBasicBlock &BB = BF.front(); 1729 if (BB.size() != 2) 1730 return false; 1731 1732 const MCInst &LDRInst = BB.getInstructionAtIndex(0); 1733 if (LDRInst.getOpcode() != AArch64::LDRXl) 1734 return false; 1735 1736 if (!LDRInst.getOperand(0).isReg() || 1737 LDRInst.getOperand(0).getReg() != AArch64::X16) 1738 return false; 1739 1740 const MCSymbol *TargetSym = getTargetSymbol(LDRInst, 1); 1741 if (!TargetSym) 1742 return false; 1743 1744 const MCInst &BRInst = BB.getInstructionAtIndex(1); 1745 if (BRInst.getOpcode() != AArch64::BR) 1746 return false; 1747 if (!BRInst.getOperand(0).isReg() || 1748 BRInst.getOperand(0).getReg() != AArch64::X16) 1749 return false; 1750 1751 const BinaryFunction::IslandInfo &IInfo = BF.getIslandInfo(); 1752 if (IInfo.HasDynamicRelocations) 1753 return false; 1754 1755 auto Iter = IInfo.Offsets.find(8); 1756 if (Iter == IInfo.Offsets.end() || Iter->second != TargetSym) 1757 return false; 1758 1759 // Extract the absolute value stored inside the island. 1760 StringRef SectionContents = BF.getOriginSection()->getContents(); 1761 StringRef FunctionContents = SectionContents.substr( 1762 BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize()); 1763 1764 const BinaryContext &BC = BF.getBinaryContext(); 1765 DataExtractor DE(FunctionContents, BC.AsmInfo->isLittleEndian(), 1766 BC.AsmInfo->getCodePointerSize()); 1767 uint64_t Offset = 8; 1768 TargetAddress = DE.getAddress(&Offset); 1769 1770 return true; 1771 } 1772 1773 bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override { 1774 if (!isADRP(Adrp) || !isAddXri(Add)) 1775 return false; 1776 1777 assert(Adrp.getOperand(0).isReg() && 1778 "Unexpected operand in ADRP instruction"); 1779 MCPhysReg AdrpReg = Adrp.getOperand(0).getReg(); 1780 assert(Add.getOperand(1).isReg() && 1781 "Unexpected operand in ADDXri instruction"); 1782 MCPhysReg AddReg = Add.getOperand(1).getReg(); 1783 return AdrpReg == AddReg; 1784 } 1785 1786 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 1787 int64_t Addend, MCContext *Ctx, int64_t &Value, 1788 uint64_t RelType) const override { 1789 unsigned ImmOpNo = -1U; 1790 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1791 ++Index) { 1792 if (Inst.getOperand(Index).isImm()) { 1793 ImmOpNo = Index; 1794 break; 1795 } 1796 } 1797 if (ImmOpNo == -1U) 1798 return false; 1799 1800 Value = Inst.getOperand(ImmOpNo).getImm(); 1801 1802 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 1803 1804 return true; 1805 } 1806 1807 void createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 1808 MCContext *Ctx) const override { 1809 Inst.setOpcode(AArch64::B); 1810 Inst.clear(); 1811 Inst.addOperand(MCOperand::createExpr(getTargetExprFor( 1812 Inst, MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx), 1813 *Ctx, 0))); 1814 } 1815 1816 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 1817 switch (RelType) { 1818 case ELF::R_AARCH64_ABS64: 1819 case ELF::R_AARCH64_ABS32: 1820 case ELF::R_AARCH64_ABS16: 1821 case ELF::R_AARCH64_ADD_ABS_LO12_NC: 1822 case ELF::R_AARCH64_ADR_GOT_PAGE: 1823 case ELF::R_AARCH64_ADR_PREL_LO21: 1824 case ELF::R_AARCH64_ADR_PREL_PG_HI21: 1825 case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: 1826 case ELF::R_AARCH64_LD64_GOT_LO12_NC: 1827 case ELF::R_AARCH64_LDST8_ABS_LO12_NC: 1828 case ELF::R_AARCH64_LDST16_ABS_LO12_NC: 1829 case ELF::R_AARCH64_LDST32_ABS_LO12_NC: 1830 case ELF::R_AARCH64_LDST64_ABS_LO12_NC: 1831 case ELF::R_AARCH64_LDST128_ABS_LO12_NC: 1832 case ELF::R_AARCH64_TLSDESC_ADD_LO12: 1833 case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: 1834 case ELF::R_AARCH64_TLSDESC_ADR_PREL21: 1835 case ELF::R_AARCH64_TLSDESC_LD64_LO12: 1836 case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 1837 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 1838 case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0: 1839 case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 1840 case ELF::R_AARCH64_MOVW_UABS_G0: 1841 case ELF::R_AARCH64_MOVW_UABS_G0_NC: 1842 case ELF::R_AARCH64_MOVW_UABS_G1: 1843 case ELF::R_AARCH64_MOVW_UABS_G1_NC: 1844 case ELF::R_AARCH64_MOVW_UABS_G2: 1845 case ELF::R_AARCH64_MOVW_UABS_G2_NC: 1846 case ELF::R_AARCH64_MOVW_UABS_G3: 1847 case ELF::R_AARCH64_PREL16: 1848 case ELF::R_AARCH64_PREL32: 1849 case ELF::R_AARCH64_PREL64: 1850 return true; 1851 case ELF::R_AARCH64_CALL26: 1852 case ELF::R_AARCH64_JUMP26: 1853 case ELF::R_AARCH64_TSTBR14: 1854 case ELF::R_AARCH64_CONDBR19: 1855 case ELF::R_AARCH64_TLSDESC_CALL: 1856 case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12: 1857 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 1858 return false; 1859 default: 1860 llvm_unreachable("Unexpected AArch64 relocation type in code"); 1861 } 1862 } 1863 1864 StringRef getTrapFillValue() const override { 1865 return StringRef("\0\0\0\0", 4); 1866 } 1867 1868 void createReturn(MCInst &Inst) const override { 1869 Inst.setOpcode(AArch64::RET); 1870 Inst.clear(); 1871 Inst.addOperand(MCOperand::createReg(AArch64::LR)); 1872 } 1873 1874 void createStackPointerIncrement( 1875 MCInst &Inst, int Size, 1876 bool NoFlagsClobber = false /*unused for AArch64*/) const override { 1877 Inst.setOpcode(AArch64::SUBXri); 1878 Inst.clear(); 1879 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 1880 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 1881 Inst.addOperand(MCOperand::createImm(Size)); 1882 Inst.addOperand(MCOperand::createImm(0)); 1883 } 1884 1885 void createStackPointerDecrement( 1886 MCInst &Inst, int Size, 1887 bool NoFlagsClobber = false /*unused for AArch64*/) const override { 1888 Inst.setOpcode(AArch64::ADDXri); 1889 Inst.clear(); 1890 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 1891 Inst.addOperand(MCOperand::createReg(AArch64::SP)); 1892 Inst.addOperand(MCOperand::createImm(Size)); 1893 Inst.addOperand(MCOperand::createImm(0)); 1894 } 1895 1896 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 1897 int64_t Disp) const { 1898 Inst.setOpcode(AArch64::BR); 1899 Inst.clear(); 1900 Inst.addOperand(MCOperand::createReg(MemBaseReg)); 1901 } 1902 1903 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 1904 InstructionListType Insts(5); 1905 // Code sequence for instrumented indirect call handler: 1906 // msr nzcv, x1 1907 // ldp x0, x1, [sp], #16 1908 // ldr x16, [sp], #16 1909 // ldp x0, x1, [sp], #16 1910 // br x16 1911 setSystemFlag(Insts[0], AArch64::X1); 1912 createPopRegisters(Insts[1], AArch64::X0, AArch64::X1); 1913 // Here we load address of the next function which should be called in the 1914 // original binary to X16 register. Writing to X16 is permitted without 1915 // needing to restore. 1916 loadReg(Insts[2], AArch64::X16, AArch64::SP); 1917 createPopRegisters(Insts[3], AArch64::X0, AArch64::X1); 1918 createIndirectBranch(Insts[4], AArch64::X16, 0); 1919 return Insts; 1920 } 1921 1922 InstructionListType 1923 createInstrumentedIndTailCallHandlerExitBB() const override { 1924 return createInstrumentedIndCallHandlerExitBB(); 1925 } 1926 1927 InstructionListType createGetter(MCContext *Ctx, const char *name) const { 1928 InstructionListType Insts(4); 1929 MCSymbol *Locs = Ctx->getOrCreateSymbol(name); 1930 InstructionListType Addr = materializeAddress(Locs, Ctx, AArch64::X0); 1931 std::copy(Addr.begin(), Addr.end(), Insts.begin()); 1932 assert(Addr.size() == 2 && "Invalid Addr size"); 1933 loadReg(Insts[2], AArch64::X0, AArch64::X0); 1934 createReturn(Insts[3]); 1935 return Insts; 1936 } 1937 1938 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 1939 return createGetter(Ctx, "__bolt_num_counters"); 1940 } 1941 1942 InstructionListType 1943 createInstrLocationsGetter(MCContext *Ctx) const override { 1944 return createGetter(Ctx, "__bolt_instr_locations"); 1945 } 1946 1947 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 1948 return createGetter(Ctx, "__bolt_instr_tables"); 1949 } 1950 1951 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 1952 return createGetter(Ctx, "__bolt_instr_num_funcs"); 1953 } 1954 1955 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1956 bool IsTailCall = isTailCall(Inst); 1957 if (IsTailCall) 1958 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1959 if (Inst.getOpcode() == AArch64::BR || Inst.getOpcode() == AArch64::BLR) { 1960 Inst.setOpcode(AArch64::ORRXrs); 1961 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1962 Inst.insert(Inst.begin() + 1, MCOperand::createReg(AArch64::XZR)); 1963 Inst.insert(Inst.begin() + 3, MCOperand::createImm(0)); 1964 return; 1965 } 1966 llvm_unreachable("not implemented"); 1967 } 1968 1969 InstructionListType createLoadImmediate(const MCPhysReg Dest, 1970 uint64_t Imm) const override { 1971 InstructionListType Insts(4); 1972 int Shift = 48; 1973 for (int I = 0; I < 4; I++, Shift -= 16) { 1974 Insts[I].setOpcode(AArch64::MOVKXi); 1975 Insts[I].addOperand(MCOperand::createReg(Dest)); 1976 Insts[I].addOperand(MCOperand::createReg(Dest)); 1977 Insts[I].addOperand(MCOperand::createImm((Imm >> Shift) & 0xFFFF)); 1978 Insts[I].addOperand(MCOperand::createImm(Shift)); 1979 } 1980 return Insts; 1981 } 1982 1983 void createIndirectCallInst(MCInst &Inst, bool IsTailCall, 1984 MCPhysReg Reg) const { 1985 Inst.clear(); 1986 Inst.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR); 1987 Inst.addOperand(MCOperand::createReg(Reg)); 1988 } 1989 1990 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 1991 MCSymbol *HandlerFuncAddr, 1992 int CallSiteID, 1993 MCContext *Ctx) override { 1994 InstructionListType Insts; 1995 // Code sequence used to enter indirect call instrumentation helper: 1996 // stp x0, x1, [sp, #-16]! createPushRegisters 1997 // mov target x0 convertIndirectCallToLoad -> orr x0 target xzr 1998 // mov x1 CallSiteID createLoadImmediate -> 1999 // movk x1, #0x0, lsl #48 2000 // movk x1, #0x0, lsl #32 2001 // movk x1, #0x0, lsl #16 2002 // movk x1, #0x0 2003 // stp x0, x1, [sp, #-16]! 2004 // bl *HandlerFuncAddr createIndirectCall -> 2005 // adr x0 *HandlerFuncAddr -> adrp + add 2006 // blr x0 2007 Insts.emplace_back(); 2008 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1); 2009 Insts.emplace_back(CallInst); 2010 convertIndirectCallToLoad(Insts.back(), AArch64::X0); 2011 InstructionListType LoadImm = 2012 createLoadImmediate(getIntArgRegister(1), CallSiteID); 2013 Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end()); 2014 Insts.emplace_back(); 2015 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1); 2016 Insts.resize(Insts.size() + 2); 2017 InstructionListType Addr = 2018 materializeAddress(HandlerFuncAddr, Ctx, AArch64::X0); 2019 assert(Addr.size() == 2 && "Invalid Addr size"); 2020 std::copy(Addr.begin(), Addr.end(), Insts.end() - Addr.size()); 2021 Insts.emplace_back(); 2022 createIndirectCallInst(Insts.back(), isTailCall(CallInst), AArch64::X0); 2023 2024 // Carry over metadata including tail call marker if present. 2025 stripAnnotations(Insts.back()); 2026 moveAnnotations(std::move(CallInst), Insts.back()); 2027 2028 return Insts; 2029 } 2030 2031 InstructionListType 2032 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 2033 const MCSymbol *IndCallHandler, 2034 MCContext *Ctx) override { 2035 // Code sequence used to check whether InstrTampoline was initialized 2036 // and call it if so, returns via IndCallHandler 2037 // stp x0, x1, [sp, #-16]! 2038 // mrs x1, nzcv 2039 // adr x0, InstrTrampoline -> adrp + add 2040 // ldr x0, [x0] 2041 // subs x0, x0, #0x0 2042 // b.eq IndCallHandler 2043 // str x30, [sp, #-16]! 2044 // blr x0 2045 // ldr x30, [sp], #16 2046 // b IndCallHandler 2047 InstructionListType Insts; 2048 Insts.emplace_back(); 2049 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1); 2050 Insts.emplace_back(); 2051 getSystemFlag(Insts.back(), getIntArgRegister(1)); 2052 Insts.emplace_back(); 2053 Insts.emplace_back(); 2054 InstructionListType Addr = 2055 materializeAddress(InstrTrampoline, Ctx, AArch64::X0); 2056 std::copy(Addr.begin(), Addr.end(), Insts.end() - Addr.size()); 2057 assert(Addr.size() == 2 && "Invalid Addr size"); 2058 Insts.emplace_back(); 2059 loadReg(Insts.back(), AArch64::X0, AArch64::X0); 2060 InstructionListType cmpJmp = 2061 createCmpJE(AArch64::X0, 0, IndCallHandler, Ctx); 2062 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 2063 Insts.emplace_back(); 2064 storeReg(Insts.back(), AArch64::LR, AArch64::SP); 2065 Insts.emplace_back(); 2066 Insts.back().setOpcode(AArch64::BLR); 2067 Insts.back().addOperand(MCOperand::createReg(AArch64::X0)); 2068 Insts.emplace_back(); 2069 loadReg(Insts.back(), AArch64::LR, AArch64::SP); 2070 Insts.emplace_back(); 2071 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 2072 return Insts; 2073 } 2074 2075 InstructionListType 2076 createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf, 2077 unsigned CodePointerSize) const override { 2078 unsigned int I = 0; 2079 InstructionListType Instrs(IsLeaf ? 12 : 10); 2080 2081 if (IsLeaf) 2082 createStackPointerIncrement(Instrs[I++], 128); 2083 createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1); 2084 getSystemFlag(Instrs[I++], AArch64::X1); 2085 InstructionListType Addr = materializeAddress(Target, Ctx, AArch64::X0); 2086 assert(Addr.size() == 2 && "Invalid Addr size"); 2087 std::copy(Addr.begin(), Addr.end(), Instrs.begin() + I); 2088 I += Addr.size(); 2089 storeReg(Instrs[I++], AArch64::X2, AArch64::SP); 2090 InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X2); 2091 assert(Insts.size() == 2 && "Invalid Insts size"); 2092 std::copy(Insts.begin(), Insts.end(), Instrs.begin() + I); 2093 I += Insts.size(); 2094 loadReg(Instrs[I++], AArch64::X2, AArch64::SP); 2095 setSystemFlag(Instrs[I++], AArch64::X1); 2096 createPopRegisters(Instrs[I++], AArch64::X0, AArch64::X1); 2097 if (IsLeaf) 2098 createStackPointerDecrement(Instrs[I++], 128); 2099 return Instrs; 2100 } 2101 2102 std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym, 2103 MCContext *Ctx) override { 2104 std::vector<MCInst> Insts; 2105 createShortJmp(Insts, TgtSym, Ctx, /*IsTailCall*/ true); 2106 return Insts; 2107 } 2108 2109 InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx, 2110 MCPhysReg RegName, 2111 int64_t Addend = 0) const override { 2112 // Get page-aligned address and add page offset 2113 InstructionListType Insts(2); 2114 Insts[0].setOpcode(AArch64::ADRP); 2115 Insts[0].clear(); 2116 Insts[0].addOperand(MCOperand::createReg(RegName)); 2117 Insts[0].addOperand(MCOperand::createImm(0)); 2118 setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx, 2119 ELF::R_AARCH64_NONE); 2120 Insts[1].setOpcode(AArch64::ADDXri); 2121 Insts[1].clear(); 2122 Insts[1].addOperand(MCOperand::createReg(RegName)); 2123 Insts[1].addOperand(MCOperand::createReg(RegName)); 2124 Insts[1].addOperand(MCOperand::createImm(0)); 2125 Insts[1].addOperand(MCOperand::createImm(0)); 2126 setOperandToSymbolRef(Insts[1], /* OpNum */ 2, Target, Addend, Ctx, 2127 ELF::R_AARCH64_ADD_ABS_LO12_NC); 2128 return Insts; 2129 } 2130 2131 std::optional<Relocation> 2132 createRelocation(const MCFixup &Fixup, 2133 const MCAsmBackend &MAB) const override { 2134 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2135 2136 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2137 const uint64_t RelOffset = Fixup.getOffset(); 2138 2139 uint64_t RelType; 2140 if (Fixup.getKind() == MCFixupKind(AArch64::fixup_aarch64_pcrel_call26)) 2141 RelType = ELF::R_AARCH64_CALL26; 2142 else if (Fixup.getKind() == 2143 MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26)) 2144 RelType = ELF::R_AARCH64_JUMP26; 2145 else if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2146 switch (FKI.TargetSize) { 2147 default: 2148 return std::nullopt; 2149 case 16: 2150 RelType = ELF::R_AARCH64_PREL16; 2151 break; 2152 case 32: 2153 RelType = ELF::R_AARCH64_PREL32; 2154 break; 2155 case 64: 2156 RelType = ELF::R_AARCH64_PREL64; 2157 break; 2158 } 2159 } else { 2160 switch (FKI.TargetSize) { 2161 default: 2162 return std::nullopt; 2163 case 16: 2164 RelType = ELF::R_AARCH64_ABS16; 2165 break; 2166 case 32: 2167 RelType = ELF::R_AARCH64_ABS32; 2168 break; 2169 case 64: 2170 RelType = ELF::R_AARCH64_ABS64; 2171 break; 2172 } 2173 } 2174 2175 auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup); 2176 2177 return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0}); 2178 } 2179 2180 uint16_t getMinFunctionAlignment() const override { return 4; } 2181 2182 std::optional<uint32_t> 2183 getInstructionSize(const MCInst &Inst) const override { 2184 return 4; 2185 } 2186 }; 2187 2188 } // end anonymous namespace 2189 2190 namespace llvm { 2191 namespace bolt { 2192 2193 MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis, 2194 const MCInstrInfo *Info, 2195 const MCRegisterInfo *RegInfo, 2196 const MCSubtargetInfo *STI) { 2197 return new AArch64MCPlusBuilder(Analysis, Info, RegInfo, STI); 2198 } 2199 2200 } // namespace bolt 2201 } // namespace llvm 2202