1 //===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Generic utilities for graphs representing aarch64 objects. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 14 15 #include "llvm/Support/BinaryStreamWriter.h" 16 17 #define DEBUG_TYPE "jitlink" 18 19 namespace llvm { 20 namespace jitlink { 21 namespace aarch64 { 22 23 const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00, 24 0x00, 0x00, 0x00, 0x00}; 25 26 const char PointerJumpStubContent[12] = { 27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21 28 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12] 29 0x00, 0x02, 0x1f, (char)0xd6u // BR x16 30 }; 31 32 const char ReentryTrampolineContent[8] = { 33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8] 34 0x00, 0x00, 0x00, (char)0x94 // BL 35 }; 36 37 const char *getEdgeKindName(Edge::Kind R) { 38 switch (R) { 39 case Pointer64: 40 return "Pointer64"; 41 case Pointer64Authenticated: 42 return "Pointer64Authenticated"; 43 case Pointer32: 44 return "Pointer32"; 45 case Delta64: 46 return "Delta64"; 47 case Delta32: 48 return "Delta32"; 49 case NegDelta64: 50 return "NegDelta64"; 51 case NegDelta32: 52 return "NegDelta32"; 53 case Branch26PCRel: 54 return "Branch26PCRel"; 55 case MoveWide16: 56 return "MoveWide16"; 57 case LDRLiteral19: 58 return "LDRLiteral19"; 59 case TestAndBranch14PCRel: 60 return "TestAndBranch14PCRel"; 61 case CondBranch19PCRel: 62 return "CondBranch19PCRel"; 63 case ADRLiteral21: 64 return "ADRLiteral21"; 65 case Page21: 66 return "Page21"; 67 case PageOffset12: 68 return "PageOffset12"; 69 case GotPageOffset15: 70 return "GotPageOffset15"; 71 case RequestGOTAndTransformToPage21: 72 return "RequestGOTAndTransformToPage21"; 73 case RequestGOTAndTransformToPageOffset12: 74 return "RequestGOTAndTransformToPageOffset12"; 75 case RequestGOTAndTransformToPageOffset15: 76 return "RequestGOTAndTransformToPageOffset15"; 77 case RequestGOTAndTransformToDelta32: 78 return "RequestGOTAndTransformToDelta32"; 79 case RequestTLVPAndTransformToPage21: 80 return "RequestTLVPAndTransformToPage21"; 81 case RequestTLVPAndTransformToPageOffset12: 82 return "RequestTLVPAndTransformToPageOffset12"; 83 case RequestTLSDescEntryAndTransformToPage21: 84 return "RequestTLSDescEntryAndTransformToPage21"; 85 case RequestTLSDescEntryAndTransformToPageOffset12: 86 return "RequestTLSDescEntryAndTransformToPageOffset12"; 87 default: 88 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 89 } 90 } 91 92 // Write a 64-bit GPR -> GPR move. 93 template <typename AppendFtor> 94 static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg, 95 uint64_t SrcReg) { 96 assert(DstReg < 32 && "Dst reg out of range"); 97 assert(SrcReg < 32 && "Src reg out of range"); 98 99 if (DstReg == SrcReg) 100 return Error::success(); 101 102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0; 103 constexpr uint32_t DstRegIndex = 0; 104 constexpr uint32_t SrcRegIndex = 16; 105 uint32_t Instr = MOVGPR64Template; 106 Instr |= DstReg << DstRegIndex; 107 Instr |= SrcReg << SrcRegIndex; 108 return Append(Instr); 109 } 110 111 // Generate a sequence of imm writes to assign the given value. 112 template <typename AppendFtor> 113 static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg, 114 uint64_t Imm) { 115 assert(Reg < 32 && "Invalid register number"); 116 117 constexpr uint32_t MovRegImm64Template = 0xd2800000; 118 constexpr unsigned PreserveBitIndex = 29; 119 constexpr unsigned ShiftBitsIndex = 21; 120 constexpr unsigned ImmBitsIndex = 5; 121 122 bool PreserveRegValue = false; 123 for (unsigned I = 0; I != 4; ++I) { 124 uint32_t ImmBits = Imm & 0xffff; 125 Imm >>= 16; 126 127 // Skip any all-zero immediates after the first one. 128 if (PreserveRegValue && !ImmBits) 129 continue; 130 131 uint32_t Instr = MovRegImm64Template; 132 Instr |= PreserveRegValue << PreserveBitIndex; 133 Instr |= (I << ShiftBitsIndex); 134 Instr |= ImmBits << ImmBitsIndex; 135 Instr |= Reg; 136 if (auto Err = Append(Instr)) 137 return Err; 138 PreserveRegValue = true; 139 } 140 141 return Error::success(); 142 } 143 144 template <typename AppendFtor> 145 static Error 146 writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr, 147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key, 148 uint64_t EncodedDiscriminator, bool AddressDiversify) { 149 assert(DstReg < 32 && "DstReg out of range"); 150 assert(RawAddrReg < 32 && "AddrReg out of range"); 151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range"); 152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range"); 153 154 if (AddressDiversify) { 155 // Move the address into the discriminator register. 156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg)) 157 return Err; 158 // Blend encoded discriminator if there is one. 159 if (EncodedDiscriminator) { 160 constexpr uint32_t MOVKTemplate = 0xf2e00000; 161 constexpr unsigned ImmIndex = 5; 162 uint32_t BlendInstr = MOVKTemplate; 163 BlendInstr |= EncodedDiscriminator << ImmIndex; 164 BlendInstr |= DiscriminatorReg; 165 if (auto Err = Append(BlendInstr)) 166 return Err; 167 } 168 } else if (EncodedDiscriminator) { 169 // Move the encoded discriminator into the discriminator register. 170 if (auto Err = 171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator)) 172 return Err; 173 } else 174 DiscriminatorReg = 31; // WZR 175 176 constexpr uint32_t PACTemplate = 0xdac10000; 177 constexpr unsigned ZBitIndex = 13; 178 constexpr unsigned KeyIndex = 10; 179 constexpr unsigned DiscriminatorRegIndex = 5; 180 181 uint32_t Instr = PACTemplate; 182 Instr |= (DiscriminatorReg == 31) << ZBitIndex; 183 Instr |= Key << KeyIndex; 184 Instr |= DiscriminatorReg << DiscriminatorRegIndex; 185 Instr |= DstReg; 186 187 return Append(Instr); 188 } 189 190 template <typename AppendFtor> 191 static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg, 192 unsigned SrcReg) { 193 assert(DstLocReg < 32 && "DstLocReg out of range"); 194 assert(SrcReg < 32 && "SrcReg out of range"); 195 196 constexpr uint32_t STRTemplate = 0xf9000000; 197 constexpr unsigned DstLocRegIndex = 5; 198 constexpr unsigned SrcRegIndex = 0; 199 200 uint32_t Instr = STRTemplate; 201 Instr |= DstLocReg << DstLocRegIndex; 202 Instr |= SrcReg << SrcRegIndex; 203 204 return Append(Instr); 205 } 206 207 void GOTTableManager::registerExistingEntries() { 208 for (auto *EntrySym : GOTSection->symbols()) { 209 assert(EntrySym->getBlock().edges_size() == 1 && 210 "GOT block edge count != 1"); 211 registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(), 212 *EntrySym); 213 } 214 } 215 216 void PLTTableManager::registerExistingEntries() { 217 for (auto *EntrySym : StubsSection->symbols()) { 218 assert(EntrySym->getBlock().edges_size() == 2 && 219 "PLT block edge count != 2"); 220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget(); 221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1"); 222 registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(), 223 *EntrySym); 224 } 225 } 226 227 const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; } 228 229 /// Creates a pointer signing function section, block, and symbol to reserve 230 /// space for a signing function for this LinkGraph. Clients should insert this 231 /// pass in the post-prune phase, and add the paired 232 /// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase. 233 Error createEmptyPointerSigningFunction(LinkGraph &G) { 234 LLVM_DEBUG({ 235 dbgs() << "Creating empty pointer signing function for " << G.getName() 236 << "\n"; 237 }); 238 239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth 240 // info encoded in the addend -- the only actually unknown quantity is the 241 // fixup location, and we can probably put constraints even on that. 242 size_t NumPtrAuthFixupLocations = 0; 243 for (auto *B : G.blocks()) 244 for (auto &E : B->edges()) 245 NumPtrAuthFixupLocations += 246 E.getKind() == aarch64::Pointer64Authenticated; 247 248 constexpr size_t MaxPtrSignSeqLength = 249 4 + // To materialize the value to sign. 250 4 + // To materialize the fixup location. 251 3 + // To copy, blend discriminator, and sign 252 1; // To store the result. 253 254 // The maximum number of signing instructions required is the maximum per 255 // location, times the number of locations, plus three instructions to 256 // materialize the return value and return. 257 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3; 258 259 // Create signing function section. 260 auto &SigningSection = 261 G.createSection(getPointerSigningFunctionSectionName(), 262 orc::MemProt::Read | orc::MemProt::Exec); 263 SigningSection.setMemLifetime(orc::MemLifetime::Finalize); 264 265 size_t SigningFunctionSize = NumSigningInstrs * 4; 266 auto &SigningFunctionBlock = G.createMutableContentBlock( 267 SigningSection, G.allocateBuffer(SigningFunctionSize), 268 orc::ExecutorAddr(), 4, 0); 269 G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(), 270 true, true); 271 272 LLVM_DEBUG({ 273 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to " 274 << NumSigningInstrs << " instructions required (" 275 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n"; 276 }); 277 278 return Error::success(); 279 } 280 281 /// Given a LinkGraph containing Pointer64Auth edges, transform those edges to 282 /// Pointer64 and add code to sign the pointers in the executor. 283 /// 284 /// This function will add a $__ptrauth_sign section with finalization-lifetime 285 /// containing an anonymous function that will sign all pointers in the graph. 286 /// An allocation action will be added to run this function during finalization. 287 Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) { 288 LLVM_DEBUG({ 289 dbgs() << "Writing pointer signing function for " << G.getName() << "\n"; 290 }); 291 292 constexpr unsigned Reg1 = 8; // Holds pointer value to sign. 293 constexpr unsigned Reg2 = 9; // Holds fixup address. 294 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed. 295 296 // Find the signing function. 297 auto *SigningSection = 298 G.findSectionByName(getPointerSigningFunctionSectionName()); 299 assert(SigningSection && "Siging section missing"); 300 assert(SigningSection->blocks_size() == 1 && 301 "Unexpected number of blocks in signing section"); 302 assert(SigningSection->symbols_size() == 1 && 303 "Unexpected number of symbols in signing section"); 304 305 auto &SigningFunctionSym = **SigningSection->symbols().begin(); 306 auto &SigningFunctionBlock = SigningFunctionSym.getBlock(); 307 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent(); 308 309 // Write the instructions to the block content. 310 BinaryStreamWriter InstrWriter( 311 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()), 312 SigningFunctionBuf.size()}, 313 G.getEndianness()); 314 315 auto AppendInstr = [&](uint32_t Instr) { 316 return InstrWriter.writeInteger(Instr); 317 }; 318 319 for (auto *B : G.blocks()) { 320 for (auto &E : B->edges()) { 321 // We're only concerned with Pointer64Authenticated edges here. 322 if (E.getKind() != aarch64::Pointer64Authenticated) 323 continue; 324 325 uint64_t EncodedInfo = E.getAddend(); 326 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff); 327 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff; 328 bool AddressDiversify = (EncodedInfo >> 48) & 0x1; 329 uint32_t Key = (EncodedInfo >> 49) & 0x3; 330 uint32_t HighBits = EncodedInfo >> 51; 331 auto ValueToSign = E.getTarget().getAddress() + RealAddend; 332 333 if (HighBits != 0x1000) 334 return make_error<JITLinkError>( 335 "Pointer64Auth edge at " + 336 formatv("{0:x}", B->getFixupAddress(E).getValue()) + 337 " has invalid encoded addend " + formatv("{0:x}", EncodedInfo)); 338 339 LLVM_DEBUG({ 340 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"}; 341 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign 342 << " : key = " << KeyNames[Key] << ", discriminator = " 343 << formatv("{0:x4}", InitialDiscriminator) 344 << ", address diversified = " 345 << (AddressDiversify ? "yes" : "no") << "\n"; 346 }); 347 348 // Materialize pointer value. 349 cantFail(writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue())); 350 351 // Materialize fixup pointer. 352 cantFail(writeMovRegImm64Seq(AppendInstr, Reg2, 353 B->getFixupAddress(E).getValue())); 354 355 // Write signing instruction(s). 356 cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3, Key, 357 InitialDiscriminator, AddressDiversify)); 358 359 // Store signed pointer. 360 cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1)); 361 362 // Replace edge with a keep-alive to preserve dependence info. 363 E.setKind(Edge::KeepAlive); 364 } 365 } 366 367 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value. 368 constexpr uint32_t RETInstr = 0xd65f03c0; 369 cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0 370 cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1 371 cantFail(AppendInstr(RETInstr)); // ret 372 373 // Add an allocation action to call the signing function. 374 using namespace orc::shared; 375 G.allocActions().push_back( 376 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>( 377 SigningFunctionSym.getAddress())), 378 {}}); 379 380 return Error::success(); 381 } 382 383 } // namespace aarch64 384 } // namespace jitlink 385 } // namespace llvm 386