1 //===------- ELF_riscv.cpp -JIT linker implementation for ELF/riscv -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/riscv jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" 14 #include "ELFLinkGraphBuilder.h" 15 #include "JITLinkGeneric.h" 16 #include "PerGraphGOTAndPLTStubsBuilder.h" 17 #include "llvm/BinaryFormat/ELF.h" 18 #include "llvm/ExecutionEngine/JITLink/JITLink.h" 19 #include "llvm/ExecutionEngine/JITLink/riscv.h" 20 #include "llvm/Object/ELF.h" 21 #include "llvm/Object/ELFObjectFile.h" 22 #include "llvm/Support/Endian.h" 23 24 #define DEBUG_TYPE "jitlink" 25 using namespace llvm; 26 using namespace llvm::jitlink; 27 using namespace llvm::jitlink::riscv; 28 29 namespace { 30 31 class PerGraphGOTAndPLTStubsBuilder_ELF_riscv 32 : public PerGraphGOTAndPLTStubsBuilder< 33 PerGraphGOTAndPLTStubsBuilder_ELF_riscv> { 34 public: 35 static constexpr size_t StubEntrySize = 16; 36 static const uint8_t NullGOTEntryContent[8]; 37 static const uint8_t RV64StubContent[StubEntrySize]; 38 static const uint8_t RV32StubContent[StubEntrySize]; 39 40 using PerGraphGOTAndPLTStubsBuilder< 41 PerGraphGOTAndPLTStubsBuilder_ELF_riscv>::PerGraphGOTAndPLTStubsBuilder; 42 43 bool isRV64() const { return G.getPointerSize() == 8; } 44 45 bool isGOTEdgeToFix(Edge &E) const { return E.getKind() == R_RISCV_GOT_HI20; } 46 47 Symbol &createGOTEntry(Symbol &Target) { 48 Block &GOTBlock = 49 G.createContentBlock(getGOTSection(), getGOTEntryBlockContent(), 50 orc::ExecutorAddr(), G.getPointerSize(), 0); 51 GOTBlock.addEdge(isRV64() ? R_RISCV_64 : R_RISCV_32, 0, Target, 0); 52 return G.addAnonymousSymbol(GOTBlock, 0, G.getPointerSize(), false, false); 53 } 54 55 Symbol &createPLTStub(Symbol &Target) { 56 Block &StubContentBlock = G.createContentBlock( 57 getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 4, 0); 58 auto &GOTEntrySymbol = getGOTEntry(Target); 59 StubContentBlock.addEdge(R_RISCV_CALL, 0, GOTEntrySymbol, 0); 60 return G.addAnonymousSymbol(StubContentBlock, 0, StubEntrySize, true, 61 false); 62 } 63 64 void fixGOTEdge(Edge &E, Symbol &GOTEntry) { 65 // Replace the relocation pair (R_RISCV_GOT_HI20, R_RISCV_PCREL_LO12) 66 // with (R_RISCV_PCREL_HI20, R_RISCV_PCREL_LO12) 67 // Therefore, here just change the R_RISCV_GOT_HI20 to R_RISCV_PCREL_HI20 68 E.setKind(R_RISCV_PCREL_HI20); 69 E.setTarget(GOTEntry); 70 } 71 72 void fixPLTEdge(Edge &E, Symbol &PLTStubs) { 73 assert((E.getKind() == R_RISCV_CALL || E.getKind() == R_RISCV_CALL_PLT || 74 E.getKind() == CallRelaxable) && 75 "Not a PLT edge?"); 76 E.setKind(R_RISCV_CALL); 77 E.setTarget(PLTStubs); 78 } 79 80 bool isExternalBranchEdge(Edge &E) const { 81 return (E.getKind() == R_RISCV_CALL || E.getKind() == R_RISCV_CALL_PLT || 82 E.getKind() == CallRelaxable) && 83 !E.getTarget().isDefined(); 84 } 85 86 private: 87 Section &getGOTSection() const { 88 if (!GOTSection) 89 GOTSection = &G.createSection("$__GOT", orc::MemProt::Read); 90 return *GOTSection; 91 } 92 93 Section &getStubsSection() const { 94 if (!StubsSection) 95 StubsSection = 96 &G.createSection("$__STUBS", orc::MemProt::Read | orc::MemProt::Exec); 97 return *StubsSection; 98 } 99 100 ArrayRef<char> getGOTEntryBlockContent() { 101 return {reinterpret_cast<const char *>(NullGOTEntryContent), 102 G.getPointerSize()}; 103 } 104 105 ArrayRef<char> getStubBlockContent() { 106 auto StubContent = isRV64() ? RV64StubContent : RV32StubContent; 107 return {reinterpret_cast<const char *>(StubContent), StubEntrySize}; 108 } 109 110 mutable Section *GOTSection = nullptr; 111 mutable Section *StubsSection = nullptr; 112 }; 113 114 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_riscv::NullGOTEntryContent[8] = 115 {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 116 117 const uint8_t 118 PerGraphGOTAndPLTStubsBuilder_ELF_riscv::RV64StubContent[StubEntrySize] = { 119 0x17, 0x0e, 0x00, 0x00, // auipc t3, literal 120 0x03, 0x3e, 0x0e, 0x00, // ld t3, literal(t3) 121 0x67, 0x00, 0x0e, 0x00, // jr t3 122 0x13, 0x00, 0x00, 0x00}; // nop 123 124 const uint8_t 125 PerGraphGOTAndPLTStubsBuilder_ELF_riscv::RV32StubContent[StubEntrySize] = { 126 0x17, 0x0e, 0x00, 0x00, // auipc t3, literal 127 0x03, 0x2e, 0x0e, 0x00, // lw t3, literal(t3) 128 0x67, 0x00, 0x0e, 0x00, // jr t3 129 0x13, 0x00, 0x00, 0x00}; // nop 130 } // namespace 131 namespace llvm { 132 namespace jitlink { 133 134 static Expected<const Edge &> getRISCVPCRelHi20(const Edge &E) { 135 using namespace riscv; 136 assert((E.getKind() == R_RISCV_PCREL_LO12_I || 137 E.getKind() == R_RISCV_PCREL_LO12_S) && 138 "Can only have high relocation for R_RISCV_PCREL_LO12_I or " 139 "R_RISCV_PCREL_LO12_S"); 140 141 const Symbol &Sym = E.getTarget(); 142 const Block &B = Sym.getBlock(); 143 orc::ExecutorAddrDiff Offset = Sym.getOffset(); 144 145 struct Comp { 146 bool operator()(const Edge &Lhs, orc::ExecutorAddrDiff Offset) { 147 return Lhs.getOffset() < Offset; 148 } 149 bool operator()(orc::ExecutorAddrDiff Offset, const Edge &Rhs) { 150 return Offset < Rhs.getOffset(); 151 } 152 }; 153 154 auto Bound = 155 std::equal_range(B.edges().begin(), B.edges().end(), Offset, Comp{}); 156 157 for (auto It = Bound.first; It != Bound.second; ++It) { 158 if (It->getKind() == R_RISCV_PCREL_HI20) 159 return *It; 160 } 161 162 return make_error<JITLinkError>( 163 "No HI20 PCREL relocation type be found for LO12 PCREL relocation type"); 164 } 165 166 static uint32_t extractBits(uint32_t Num, unsigned Low, unsigned Size) { 167 return (Num & (((1ULL << Size) - 1) << Low)) >> Low; 168 } 169 170 static inline bool isAlignmentCorrect(uint64_t Value, int N) { 171 return (Value & (N - 1)) ? false : true; 172 } 173 174 // Requires 0 < N <= 64. 175 static inline bool isInRangeForImm(int64_t Value, int N) { 176 return Value == llvm::SignExtend64(Value, N); 177 } 178 179 class ELFJITLinker_riscv : public JITLinker<ELFJITLinker_riscv> { 180 friend class JITLinker<ELFJITLinker_riscv>; 181 182 public: 183 ELFJITLinker_riscv(std::unique_ptr<JITLinkContext> Ctx, 184 std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) 185 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} 186 187 private: 188 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 189 using namespace riscv; 190 using namespace llvm::support; 191 192 char *BlockWorkingMem = B.getAlreadyMutableContent().data(); 193 char *FixupPtr = BlockWorkingMem + E.getOffset(); 194 orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); 195 switch (E.getKind()) { 196 case R_RISCV_32: { 197 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 198 *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); 199 break; 200 } 201 case R_RISCV_64: { 202 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 203 *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); 204 break; 205 } 206 case R_RISCV_BRANCH: { 207 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 208 if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 12))) 209 return makeTargetOutOfRangeError(G, B, E); 210 if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) 211 return makeAlignmentError(FixupAddress, Value, 2, E); 212 uint32_t Imm12 = extractBits(Value, 12, 1) << 31; 213 uint32_t Imm10_5 = extractBits(Value, 5, 6) << 25; 214 uint32_t Imm4_1 = extractBits(Value, 1, 4) << 8; 215 uint32_t Imm11 = extractBits(Value, 11, 1) << 7; 216 uint32_t RawInstr = *(little32_t *)FixupPtr; 217 *(little32_t *)FixupPtr = 218 (RawInstr & 0x1FFF07F) | Imm12 | Imm10_5 | Imm4_1 | Imm11; 219 break; 220 } 221 case R_RISCV_JAL: { 222 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 223 if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 20))) 224 return makeTargetOutOfRangeError(G, B, E); 225 if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) 226 return makeAlignmentError(FixupAddress, Value, 2, E); 227 uint32_t Imm20 = extractBits(Value, 20, 1) << 31; 228 uint32_t Imm10_1 = extractBits(Value, 1, 10) << 21; 229 uint32_t Imm11 = extractBits(Value, 11, 1) << 20; 230 uint32_t Imm19_12 = extractBits(Value, 12, 8) << 12; 231 uint32_t RawInstr = *(little32_t *)FixupPtr; 232 *(little32_t *)FixupPtr = 233 (RawInstr & 0xFFF) | Imm20 | Imm10_1 | Imm11 | Imm19_12; 234 break; 235 } 236 case CallRelaxable: 237 // Treat as R_RISCV_CALL when the relaxation pass did not run 238 case R_RISCV_CALL_PLT: 239 case R_RISCV_CALL: { 240 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 241 int64_t Hi = Value + 0x800; 242 if (LLVM_UNLIKELY(!isInRangeForImm(Hi, 32))) 243 return makeTargetOutOfRangeError(G, B, E); 244 int32_t Lo = Value & 0xFFF; 245 uint32_t RawInstrAuipc = *(little32_t *)FixupPtr; 246 uint32_t RawInstrJalr = *(little32_t *)(FixupPtr + 4); 247 *(little32_t *)FixupPtr = 248 RawInstrAuipc | (static_cast<uint32_t>(Hi & 0xFFFFF000)); 249 *(little32_t *)(FixupPtr + 4) = 250 RawInstrJalr | (static_cast<uint32_t>(Lo) << 20); 251 break; 252 } 253 // The relocations R_RISCV_CALL_PLT and R_RISCV_GOT_HI20 are handled by 254 // PerGraphGOTAndPLTStubsBuilder_ELF_riscv and are transformed into 255 // R_RISCV_CALL and R_RISCV_PCREL_HI20. 256 case R_RISCV_PCREL_HI20: { 257 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 258 int64_t Hi = Value + 0x800; 259 if (LLVM_UNLIKELY(!isInRangeForImm(Hi, 32))) 260 return makeTargetOutOfRangeError(G, B, E); 261 uint32_t RawInstr = *(little32_t *)FixupPtr; 262 *(little32_t *)FixupPtr = 263 (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); 264 break; 265 } 266 case R_RISCV_PCREL_LO12_I: { 267 // FIXME: We assume that R_RISCV_PCREL_HI20 is present in object code and 268 // pairs with current relocation R_RISCV_PCREL_LO12_I. So here may need a 269 // check. 270 auto RelHI20 = getRISCVPCRelHi20(E); 271 if (!RelHI20) 272 return RelHI20.takeError(); 273 int64_t Value = RelHI20->getTarget().getAddress() + 274 RelHI20->getAddend() - E.getTarget().getAddress(); 275 int64_t Lo = Value & 0xFFF; 276 uint32_t RawInstr = *(little32_t *)FixupPtr; 277 *(little32_t *)FixupPtr = 278 (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); 279 break; 280 } 281 case R_RISCV_PCREL_LO12_S: { 282 // FIXME: We assume that R_RISCV_PCREL_HI20 is present in object code and 283 // pairs with current relocation R_RISCV_PCREL_LO12_S. So here may need a 284 // check. 285 auto RelHI20 = getRISCVPCRelHi20(E); 286 if (!RelHI20) 287 return RelHI20.takeError(); 288 int64_t Value = RelHI20->getTarget().getAddress() + 289 RelHI20->getAddend() - E.getTarget().getAddress(); 290 int64_t Lo = Value & 0xFFF; 291 uint32_t Imm11_5 = extractBits(Lo, 5, 7) << 25; 292 uint32_t Imm4_0 = extractBits(Lo, 0, 5) << 7; 293 uint32_t RawInstr = *(little32_t *)FixupPtr; 294 295 *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm11_5 | Imm4_0; 296 break; 297 } 298 case R_RISCV_HI20: { 299 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 300 int64_t Hi = Value + 0x800; 301 if (LLVM_UNLIKELY(!isInRangeForImm(Hi, 32))) 302 return makeTargetOutOfRangeError(G, B, E); 303 uint32_t RawInstr = *(little32_t *)FixupPtr; 304 *(little32_t *)FixupPtr = 305 (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); 306 break; 307 } 308 case R_RISCV_LO12_I: { 309 // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs 310 // with current relocation R_RISCV_LO12_I. So here may need a check. 311 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 312 int32_t Lo = Value & 0xFFF; 313 uint32_t RawInstr = *(little32_t *)FixupPtr; 314 *(little32_t *)FixupPtr = 315 (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); 316 break; 317 } 318 case R_RISCV_LO12_S: { 319 // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs 320 // with current relocation R_RISCV_LO12_S. So here may need a check. 321 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 322 int64_t Lo = Value & 0xFFF; 323 uint32_t Imm11_5 = extractBits(Lo, 5, 7) << 25; 324 uint32_t Imm4_0 = extractBits(Lo, 0, 5) << 7; 325 uint32_t RawInstr = *(little32_t *)FixupPtr; 326 *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm11_5 | Imm4_0; 327 break; 328 } 329 case R_RISCV_ADD8: { 330 int64_t Value = 331 (E.getTarget().getAddress() + 332 *(reinterpret_cast<const uint8_t *>(FixupPtr)) + E.getAddend()) 333 .getValue(); 334 *FixupPtr = static_cast<uint8_t>(Value); 335 break; 336 } 337 case R_RISCV_ADD16: { 338 int64_t Value = (E.getTarget().getAddress() + 339 support::endian::read16le(FixupPtr) + E.getAddend()) 340 .getValue(); 341 *(little16_t *)FixupPtr = static_cast<uint16_t>(Value); 342 break; 343 } 344 case R_RISCV_ADD32: { 345 int64_t Value = (E.getTarget().getAddress() + 346 support::endian::read32le(FixupPtr) + E.getAddend()) 347 .getValue(); 348 *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); 349 break; 350 } 351 case R_RISCV_ADD64: { 352 int64_t Value = (E.getTarget().getAddress() + 353 support::endian::read64le(FixupPtr) + E.getAddend()) 354 .getValue(); 355 *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); 356 break; 357 } 358 case R_RISCV_SUB8: { 359 int64_t Value = *(reinterpret_cast<const uint8_t *>(FixupPtr)) - 360 E.getTarget().getAddress().getValue() - E.getAddend(); 361 *FixupPtr = static_cast<uint8_t>(Value); 362 break; 363 } 364 case R_RISCV_SUB16: { 365 int64_t Value = support::endian::read16le(FixupPtr) - 366 E.getTarget().getAddress().getValue() - E.getAddend(); 367 *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); 368 break; 369 } 370 case R_RISCV_SUB32: { 371 int64_t Value = support::endian::read32le(FixupPtr) - 372 E.getTarget().getAddress().getValue() - E.getAddend(); 373 *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); 374 break; 375 } 376 case R_RISCV_SUB64: { 377 int64_t Value = support::endian::read64le(FixupPtr) - 378 E.getTarget().getAddress().getValue() - E.getAddend(); 379 *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); 380 break; 381 } 382 case R_RISCV_RVC_BRANCH: { 383 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 384 if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 8))) 385 return makeTargetOutOfRangeError(G, B, E); 386 if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) 387 return makeAlignmentError(FixupAddress, Value, 2, E); 388 uint16_t Imm8 = extractBits(Value, 8, 1) << 12; 389 uint16_t Imm4_3 = extractBits(Value, 3, 2) << 10; 390 uint16_t Imm7_6 = extractBits(Value, 6, 2) << 5; 391 uint16_t Imm2_1 = extractBits(Value, 1, 2) << 3; 392 uint16_t Imm5 = extractBits(Value, 5, 1) << 2; 393 uint16_t RawInstr = *(little16_t *)FixupPtr; 394 *(little16_t *)FixupPtr = 395 (RawInstr & 0xE383) | Imm8 | Imm4_3 | Imm7_6 | Imm2_1 | Imm5; 396 break; 397 } 398 case R_RISCV_RVC_JUMP: { 399 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 400 if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 11))) 401 return makeTargetOutOfRangeError(G, B, E); 402 if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) 403 return makeAlignmentError(FixupAddress, Value, 2, E); 404 uint16_t Imm11 = extractBits(Value, 11, 1) << 12; 405 uint16_t Imm4 = extractBits(Value, 4, 1) << 11; 406 uint16_t Imm9_8 = extractBits(Value, 8, 2) << 9; 407 uint16_t Imm10 = extractBits(Value, 10, 1) << 8; 408 uint16_t Imm6 = extractBits(Value, 6, 1) << 7; 409 uint16_t Imm7 = extractBits(Value, 7, 1) << 6; 410 uint16_t Imm3_1 = extractBits(Value, 1, 3) << 3; 411 uint16_t Imm5 = extractBits(Value, 5, 1) << 2; 412 uint16_t RawInstr = *(little16_t *)FixupPtr; 413 *(little16_t *)FixupPtr = (RawInstr & 0xE003) | Imm11 | Imm4 | Imm9_8 | 414 Imm10 | Imm6 | Imm7 | Imm3_1 | Imm5; 415 break; 416 } 417 case R_RISCV_SUB6: { 418 int64_t Value = *(reinterpret_cast<const uint8_t *>(FixupPtr)) & 0x3f; 419 Value -= E.getTarget().getAddress().getValue() - E.getAddend(); 420 *FixupPtr = (*FixupPtr & 0xc0) | (static_cast<uint8_t>(Value) & 0x3f); 421 break; 422 } 423 case R_RISCV_SET6: { 424 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 425 uint32_t RawData = *(little32_t *)FixupPtr; 426 int64_t Word6 = Value & 0x3f; 427 *(little32_t *)FixupPtr = (RawData & 0xffffffc0) | Word6; 428 break; 429 } 430 case R_RISCV_SET8: { 431 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 432 uint32_t RawData = *(little32_t *)FixupPtr; 433 int64_t Word8 = Value & 0xff; 434 *(little32_t *)FixupPtr = (RawData & 0xffffff00) | Word8; 435 break; 436 } 437 case R_RISCV_SET16: { 438 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 439 uint32_t RawData = *(little32_t *)FixupPtr; 440 int64_t Word16 = Value & 0xffff; 441 *(little32_t *)FixupPtr = (RawData & 0xffff0000) | Word16; 442 break; 443 } 444 case R_RISCV_SET32: { 445 int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); 446 int64_t Word32 = Value & 0xffffffff; 447 *(little32_t *)FixupPtr = Word32; 448 break; 449 } 450 case R_RISCV_32_PCREL: { 451 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 452 int64_t Word32 = Value & 0xffffffff; 453 *(little32_t *)FixupPtr = Word32; 454 break; 455 } 456 case AlignRelaxable: 457 // Ignore when the relaxation pass did not run 458 break; 459 } 460 return Error::success(); 461 } 462 }; 463 464 namespace { 465 466 struct SymbolAnchor { 467 uint64_t Offset; 468 Symbol *Sym; 469 bool End; // true for the anchor of getOffset() + getSize() 470 }; 471 472 struct BlockRelaxAux { 473 // This records symbol start and end offsets which will be adjusted according 474 // to the nearest RelocDeltas element. 475 SmallVector<SymbolAnchor, 0> Anchors; 476 // All edges that either 1) are R_RISCV_ALIGN or 2) have a R_RISCV_RELAX edge 477 // at the same offset. 478 SmallVector<Edge *, 0> RelaxEdges; 479 // For RelaxEdges[I], the actual offset is RelaxEdges[I]->getOffset() - (I ? 480 // RelocDeltas[I - 1] : 0). 481 SmallVector<uint32_t, 0> RelocDeltas; 482 // For RelaxEdges[I], the actual type is EdgeKinds[I]. 483 SmallVector<Edge::Kind, 0> EdgeKinds; 484 // List of rewritten instructions. Contains one raw encoded instruction per 485 // element in EdgeKinds that isn't Invalid or R_RISCV_ALIGN. 486 SmallVector<uint32_t, 0> Writes; 487 }; 488 489 struct RelaxConfig { 490 bool IsRV32; 491 bool HasRVC; 492 }; 493 494 struct RelaxAux { 495 RelaxConfig Config; 496 DenseMap<Block *, BlockRelaxAux> Blocks; 497 }; 498 499 } // namespace 500 501 static bool shouldRelax(const Section &S) { 502 return (S.getMemProt() & orc::MemProt::Exec) != orc::MemProt::None; 503 } 504 505 static bool isRelaxable(const Edge &E) { 506 switch (E.getKind()) { 507 default: 508 return false; 509 case CallRelaxable: 510 case AlignRelaxable: 511 return true; 512 } 513 } 514 515 static RelaxAux initRelaxAux(LinkGraph &G) { 516 RelaxAux Aux; 517 Aux.Config.IsRV32 = G.getTargetTriple().isRISCV32(); 518 const auto &Features = G.getFeatures().getFeatures(); 519 Aux.Config.HasRVC = llvm::is_contained(Features, "+c"); 520 521 for (auto &S : G.sections()) { 522 if (!shouldRelax(S)) 523 continue; 524 for (auto *B : S.blocks()) { 525 auto BlockEmplaceResult = Aux.Blocks.try_emplace(B); 526 assert(BlockEmplaceResult.second && "Block encountered twice"); 527 auto &BlockAux = BlockEmplaceResult.first->second; 528 529 for (auto &E : B->edges()) 530 if (isRelaxable(E)) 531 BlockAux.RelaxEdges.push_back(&E); 532 533 if (BlockAux.RelaxEdges.empty()) { 534 Aux.Blocks.erase(BlockEmplaceResult.first); 535 continue; 536 } 537 538 const auto NumEdges = BlockAux.RelaxEdges.size(); 539 BlockAux.RelocDeltas.resize(NumEdges, 0); 540 BlockAux.EdgeKinds.resize_for_overwrite(NumEdges); 541 542 // Store anchors (offset and offset+size) for symbols. 543 for (auto *Sym : S.symbols()) { 544 if (!Sym->isDefined() || &Sym->getBlock() != B) 545 continue; 546 547 BlockAux.Anchors.push_back({Sym->getOffset(), Sym, false}); 548 BlockAux.Anchors.push_back( 549 {Sym->getOffset() + Sym->getSize(), Sym, true}); 550 } 551 } 552 } 553 554 // Sort anchors by offset so that we can find the closest relocation 555 // efficiently. For a zero size symbol, ensure that its start anchor precedes 556 // its end anchor. For two symbols with anchors at the same offset, their 557 // order does not matter. 558 for (auto &BlockAuxIter : Aux.Blocks) { 559 llvm::sort(BlockAuxIter.second.Anchors, [](auto &A, auto &B) { 560 return std::make_pair(A.Offset, A.End) < std::make_pair(B.Offset, B.End); 561 }); 562 } 563 564 return Aux; 565 } 566 567 static void relaxAlign(orc::ExecutorAddr Loc, const Edge &E, uint32_t &Remove, 568 Edge::Kind &NewEdgeKind) { 569 // E points to the start of the padding bytes. 570 // E + Addend points to the instruction to be aligned by removing padding. 571 // Alignment is the smallest power of 2 strictly greater than Addend. 572 const auto Align = NextPowerOf2(E.getAddend()); 573 const auto DestLoc = alignTo(Loc.getValue(), Align); 574 const auto SrcLoc = Loc.getValue() + E.getAddend(); 575 Remove = SrcLoc - DestLoc; 576 assert(static_cast<int32_t>(Remove) >= 0 && 577 "R_RISCV_ALIGN needs expanding the content"); 578 NewEdgeKind = AlignRelaxable; 579 } 580 581 static void relaxCall(const Block &B, BlockRelaxAux &Aux, 582 const RelaxConfig &Config, orc::ExecutorAddr Loc, 583 const Edge &E, uint32_t &Remove, 584 Edge::Kind &NewEdgeKind) { 585 const auto JALR = 586 support::endian::read32le(B.getContent().data() + E.getOffset() + 4); 587 const auto RD = extractBits(JALR, 7, 5); 588 const auto Dest = E.getTarget().getAddress() + E.getAddend(); 589 const auto Displace = Dest - Loc; 590 591 if (Config.HasRVC && isInt<12>(Displace) && RD == 0) { 592 NewEdgeKind = R_RISCV_RVC_JUMP; 593 Aux.Writes.push_back(0xa001); // c.j 594 Remove = 6; 595 } else if (Config.HasRVC && Config.IsRV32 && isInt<12>(Displace) && RD == 1) { 596 NewEdgeKind = R_RISCV_RVC_JUMP; 597 Aux.Writes.push_back(0x2001); // c.jal 598 Remove = 6; 599 } else if (isInt<21>(Displace)) { 600 NewEdgeKind = R_RISCV_JAL; 601 Aux.Writes.push_back(0x6f | RD << 7); // jal 602 Remove = 4; 603 } else { 604 // Not relaxable 605 NewEdgeKind = R_RISCV_CALL_PLT; 606 Remove = 0; 607 } 608 } 609 610 static bool relaxBlock(LinkGraph &G, Block &Block, BlockRelaxAux &Aux, 611 const RelaxConfig &Config) { 612 const auto BlockAddr = Block.getAddress(); 613 bool Changed = false; 614 ArrayRef<SymbolAnchor> SA = ArrayRef(Aux.Anchors); 615 uint32_t Delta = 0; 616 617 Aux.EdgeKinds.assign(Aux.EdgeKinds.size(), Edge::Invalid); 618 Aux.Writes.clear(); 619 620 for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { 621 const auto Loc = BlockAddr + E->getOffset() - Delta; 622 auto &Cur = Aux.RelocDeltas[I]; 623 uint32_t Remove = 0; 624 switch (E->getKind()) { 625 case AlignRelaxable: 626 relaxAlign(Loc, *E, Remove, Aux.EdgeKinds[I]); 627 break; 628 case CallRelaxable: 629 relaxCall(Block, Aux, Config, Loc, *E, Remove, Aux.EdgeKinds[I]); 630 break; 631 default: 632 llvm_unreachable("Unexpected relaxable edge kind"); 633 } 634 635 // For all anchors whose offsets are <= E->getOffset(), they are preceded by 636 // the previous relocation whose RelocDeltas value equals Delta. 637 // Decrease their offset and update their size. 638 for (; SA.size() && SA[0].Offset <= E->getOffset(); SA = SA.slice(1)) { 639 if (SA[0].End) 640 SA[0].Sym->setSize(SA[0].Offset - Delta - SA[0].Sym->getOffset()); 641 else 642 SA[0].Sym->setOffset(SA[0].Offset - Delta); 643 } 644 645 Delta += Remove; 646 if (Delta != Cur) { 647 Cur = Delta; 648 Changed = true; 649 } 650 } 651 652 for (const SymbolAnchor &A : SA) { 653 if (A.End) 654 A.Sym->setSize(A.Offset - Delta - A.Sym->getOffset()); 655 else 656 A.Sym->setOffset(A.Offset - Delta); 657 } 658 659 return Changed; 660 } 661 662 static bool relaxOnce(LinkGraph &G, RelaxAux &Aux) { 663 bool Changed = false; 664 665 for (auto &[B, BlockAux] : Aux.Blocks) 666 Changed |= relaxBlock(G, *B, BlockAux, Aux.Config); 667 668 return Changed; 669 } 670 671 static void finalizeBlockRelax(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) { 672 auto Contents = Block.getAlreadyMutableContent(); 673 auto *Dest = Contents.data(); 674 auto NextWrite = Aux.Writes.begin(); 675 uint32_t Offset = 0; 676 uint32_t Delta = 0; 677 678 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite 679 // instructions for relaxed relocations. 680 for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { 681 uint32_t Remove = Aux.RelocDeltas[I] - Delta; 682 Delta = Aux.RelocDeltas[I]; 683 if (Remove == 0 && Aux.EdgeKinds[I] == Edge::Invalid) 684 continue; 685 686 // Copy from last location to the current relocated location. 687 const auto Size = E->getOffset() - Offset; 688 std::memmove(Dest, Contents.data() + Offset, Size); 689 Dest += Size; 690 691 uint32_t Skip = 0; 692 switch (Aux.EdgeKinds[I]) { 693 case Edge::Invalid: 694 break; 695 case AlignRelaxable: 696 // For R_RISCV_ALIGN, we will place Offset in a location (among NOPs) to 697 // satisfy the alignment requirement. If both Remove and E->getAddend() 698 // are multiples of 4, it is as if we have skipped some NOPs. Otherwise we 699 // are in the middle of a 4-byte NOP, and we need to rewrite the NOP 700 // sequence. 701 if (Remove % 4 || E->getAddend() % 4) { 702 Skip = E->getAddend() - Remove; 703 uint32_t J = 0; 704 for (; J + 4 <= Skip; J += 4) 705 support::endian::write32le(Dest + J, 0x00000013); // nop 706 if (J != Skip) { 707 assert(J + 2 == Skip); 708 support::endian::write16le(Dest + J, 0x0001); // c.nop 709 } 710 } 711 break; 712 case R_RISCV_RVC_JUMP: 713 Skip = 2; 714 support::endian::write16le(Dest, *NextWrite++); 715 break; 716 case R_RISCV_JAL: 717 Skip = 4; 718 support::endian::write32le(Dest, *NextWrite++); 719 break; 720 } 721 722 Dest += Skip; 723 Offset = E->getOffset() + Skip + Remove; 724 } 725 726 std::memmove(Dest, Contents.data() + Offset, Contents.size() - Offset); 727 728 // Fixup edge offsets and kinds. 729 Delta = 0; 730 size_t I = 0; 731 for (auto &E : Block.edges()) { 732 E.setOffset(E.getOffset() - Delta); 733 734 if (I < Aux.RelaxEdges.size() && Aux.RelaxEdges[I] == &E) { 735 if (Aux.EdgeKinds[I] != Edge::Invalid) 736 E.setKind(Aux.EdgeKinds[I]); 737 738 Delta = Aux.RelocDeltas[I]; 739 ++I; 740 } 741 } 742 743 // Remove AlignRelaxable edges: all other relaxable edges got modified and 744 // will be used later while linking. Alignment is entirely handled here so we 745 // don't need these edges anymore. 746 for (auto IE = Block.edges().begin(); IE != Block.edges().end();) { 747 if (IE->getKind() == AlignRelaxable) 748 IE = Block.removeEdge(IE); 749 else 750 ++IE; 751 } 752 } 753 754 static void finalizeRelax(LinkGraph &G, RelaxAux &Aux) { 755 for (auto &[B, BlockAux] : Aux.Blocks) 756 finalizeBlockRelax(G, *B, BlockAux); 757 } 758 759 static Error relax(LinkGraph &G) { 760 auto Aux = initRelaxAux(G); 761 while (relaxOnce(G, Aux)) { 762 } 763 finalizeRelax(G, Aux); 764 return Error::success(); 765 } 766 767 template <typename ELFT> 768 class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder<ELFT> { 769 private: 770 static Expected<riscv::EdgeKind_riscv> 771 getRelocationKind(const uint32_t Type) { 772 using namespace riscv; 773 switch (Type) { 774 case ELF::R_RISCV_32: 775 return EdgeKind_riscv::R_RISCV_32; 776 case ELF::R_RISCV_64: 777 return EdgeKind_riscv::R_RISCV_64; 778 case ELF::R_RISCV_BRANCH: 779 return EdgeKind_riscv::R_RISCV_BRANCH; 780 case ELF::R_RISCV_JAL: 781 return EdgeKind_riscv::R_RISCV_JAL; 782 case ELF::R_RISCV_CALL: 783 return EdgeKind_riscv::R_RISCV_CALL; 784 case ELF::R_RISCV_CALL_PLT: 785 return EdgeKind_riscv::R_RISCV_CALL_PLT; 786 case ELF::R_RISCV_GOT_HI20: 787 return EdgeKind_riscv::R_RISCV_GOT_HI20; 788 case ELF::R_RISCV_PCREL_HI20: 789 return EdgeKind_riscv::R_RISCV_PCREL_HI20; 790 case ELF::R_RISCV_PCREL_LO12_I: 791 return EdgeKind_riscv::R_RISCV_PCREL_LO12_I; 792 case ELF::R_RISCV_PCREL_LO12_S: 793 return EdgeKind_riscv::R_RISCV_PCREL_LO12_S; 794 case ELF::R_RISCV_HI20: 795 return EdgeKind_riscv::R_RISCV_HI20; 796 case ELF::R_RISCV_LO12_I: 797 return EdgeKind_riscv::R_RISCV_LO12_I; 798 case ELF::R_RISCV_LO12_S: 799 return EdgeKind_riscv::R_RISCV_LO12_S; 800 case ELF::R_RISCV_ADD8: 801 return EdgeKind_riscv::R_RISCV_ADD8; 802 case ELF::R_RISCV_ADD16: 803 return EdgeKind_riscv::R_RISCV_ADD16; 804 case ELF::R_RISCV_ADD32: 805 return EdgeKind_riscv::R_RISCV_ADD32; 806 case ELF::R_RISCV_ADD64: 807 return EdgeKind_riscv::R_RISCV_ADD64; 808 case ELF::R_RISCV_SUB8: 809 return EdgeKind_riscv::R_RISCV_SUB8; 810 case ELF::R_RISCV_SUB16: 811 return EdgeKind_riscv::R_RISCV_SUB16; 812 case ELF::R_RISCV_SUB32: 813 return EdgeKind_riscv::R_RISCV_SUB32; 814 case ELF::R_RISCV_SUB64: 815 return EdgeKind_riscv::R_RISCV_SUB64; 816 case ELF::R_RISCV_RVC_BRANCH: 817 return EdgeKind_riscv::R_RISCV_RVC_BRANCH; 818 case ELF::R_RISCV_RVC_JUMP: 819 return EdgeKind_riscv::R_RISCV_RVC_JUMP; 820 case ELF::R_RISCV_SUB6: 821 return EdgeKind_riscv::R_RISCV_SUB6; 822 case ELF::R_RISCV_SET6: 823 return EdgeKind_riscv::R_RISCV_SET6; 824 case ELF::R_RISCV_SET8: 825 return EdgeKind_riscv::R_RISCV_SET8; 826 case ELF::R_RISCV_SET16: 827 return EdgeKind_riscv::R_RISCV_SET16; 828 case ELF::R_RISCV_SET32: 829 return EdgeKind_riscv::R_RISCV_SET32; 830 case ELF::R_RISCV_32_PCREL: 831 return EdgeKind_riscv::R_RISCV_32_PCREL; 832 case ELF::R_RISCV_ALIGN: 833 return EdgeKind_riscv::AlignRelaxable; 834 } 835 836 return make_error<JITLinkError>( 837 "Unsupported riscv relocation:" + formatv("{0:d}: ", Type) + 838 object::getELFRelocationTypeName(ELF::EM_RISCV, Type)); 839 } 840 841 EdgeKind_riscv getRelaxableRelocationKind(EdgeKind_riscv Kind) { 842 switch (Kind) { 843 default: 844 // Just ignore unsupported relaxations 845 return Kind; 846 case R_RISCV_CALL: 847 case R_RISCV_CALL_PLT: 848 return CallRelaxable; 849 } 850 } 851 852 Error addRelocations() override { 853 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 854 855 using Base = ELFLinkGraphBuilder<ELFT>; 856 using Self = ELFLinkGraphBuilder_riscv<ELFT>; 857 for (const auto &RelSect : Base::Sections) 858 if (Error Err = Base::forEachRelaRelocation(RelSect, this, 859 &Self::addSingleRelocation)) 860 return Err; 861 862 return Error::success(); 863 } 864 865 Error addSingleRelocation(const typename ELFT::Rela &Rel, 866 const typename ELFT::Shdr &FixupSect, 867 Block &BlockToFix) { 868 using Base = ELFLinkGraphBuilder<ELFT>; 869 870 uint32_t Type = Rel.getType(false); 871 int64_t Addend = Rel.r_addend; 872 873 if (Type == ELF::R_RISCV_RELAX) { 874 if (BlockToFix.edges_empty()) 875 return make_error<StringError>( 876 "R_RISCV_RELAX without preceding relocation", 877 inconvertibleErrorCode()); 878 879 auto &PrevEdge = *std::prev(BlockToFix.edges().end()); 880 auto Kind = static_cast<EdgeKind_riscv>(PrevEdge.getKind()); 881 PrevEdge.setKind(getRelaxableRelocationKind(Kind)); 882 return Error::success(); 883 } 884 885 Expected<riscv::EdgeKind_riscv> Kind = getRelocationKind(Type); 886 if (!Kind) 887 return Kind.takeError(); 888 889 uint32_t SymbolIndex = Rel.getSymbol(false); 890 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 891 if (!ObjSymbol) 892 return ObjSymbol.takeError(); 893 894 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 895 if (!GraphSymbol) 896 return make_error<StringError>( 897 formatv("Could not find symbol at given index, did you add it to " 898 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 899 SymbolIndex, (*ObjSymbol)->st_shndx, 900 Base::GraphSymbols.size()), 901 inconvertibleErrorCode()); 902 903 auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; 904 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 905 Edge GE(*Kind, Offset, *GraphSymbol, Addend); 906 LLVM_DEBUG({ 907 dbgs() << " "; 908 printEdge(dbgs(), BlockToFix, GE, riscv::getEdgeKindName(*Kind)); 909 dbgs() << "\n"; 910 }); 911 912 BlockToFix.addEdge(std::move(GE)); 913 return Error::success(); 914 } 915 916 public: 917 ELFLinkGraphBuilder_riscv(StringRef FileName, 918 const object::ELFFile<ELFT> &Obj, Triple TT, 919 SubtargetFeatures Features) 920 : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), 921 FileName, riscv::getEdgeKindName) {} 922 }; 923 924 Expected<std::unique_ptr<LinkGraph>> 925 createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) { 926 LLVM_DEBUG({ 927 dbgs() << "Building jitlink graph for new input " 928 << ObjectBuffer.getBufferIdentifier() << "...\n"; 929 }); 930 931 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 932 if (!ELFObj) 933 return ELFObj.takeError(); 934 935 auto Features = (*ELFObj)->getFeatures(); 936 if (!Features) 937 return Features.takeError(); 938 939 if ((*ELFObj)->getArch() == Triple::riscv64) { 940 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 941 return ELFLinkGraphBuilder_riscv<object::ELF64LE>( 942 (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), 943 (*ELFObj)->makeTriple(), std::move(*Features)) 944 .buildGraph(); 945 } else { 946 assert((*ELFObj)->getArch() == Triple::riscv32 && 947 "Invalid triple for RISCV ELF object file"); 948 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); 949 return ELFLinkGraphBuilder_riscv<object::ELF32LE>( 950 (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), 951 (*ELFObj)->makeTriple(), std::move(*Features)) 952 .buildGraph(); 953 } 954 } 955 956 void link_ELF_riscv(std::unique_ptr<LinkGraph> G, 957 std::unique_ptr<JITLinkContext> Ctx) { 958 PassConfiguration Config; 959 const Triple &TT = G->getTargetTriple(); 960 if (Ctx->shouldAddDefaultTargetPasses(TT)) { 961 if (auto MarkLive = Ctx->getMarkLivePass(TT)) 962 Config.PrePrunePasses.push_back(std::move(MarkLive)); 963 else 964 Config.PrePrunePasses.push_back(markAllSymbolsLive); 965 Config.PostPrunePasses.push_back( 966 PerGraphGOTAndPLTStubsBuilder_ELF_riscv::asPass); 967 Config.PostAllocationPasses.push_back(relax); 968 } 969 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 970 return Ctx->notifyFailed(std::move(Err)); 971 972 ELFJITLinker_riscv::link(std::move(Ctx), std::move(G), std::move(Config)); 973 } 974 975 LinkGraphPassFunction createRelaxationPass_ELF_riscv() { return relax; } 976 977 } // namespace jitlink 978 } // namespace llvm 979