1 //===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the collection of functions and classes used for 10 // emission of code and data into object/binary file. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "bolt/Core/BinaryEmitter.h" 15 #include "bolt/Core/BinaryContext.h" 16 #include "bolt/Core/BinaryFunction.h" 17 #include "bolt/Core/DebugData.h" 18 #include "bolt/Core/FunctionLayout.h" 19 #include "bolt/Utils/CommandLineOpts.h" 20 #include "bolt/Utils/Utils.h" 21 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" 22 #include "llvm/MC/MCSection.h" 23 #include "llvm/MC/MCStreamer.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/LEB128.h" 26 #include "llvm/Support/SMLoc.h" 27 28 #define DEBUG_TYPE "bolt" 29 30 using namespace llvm; 31 using namespace bolt; 32 33 namespace opts { 34 35 extern cl::opt<JumpTableSupportLevel> JumpTables; 36 extern cl::opt<bool> PreserveBlocksAlignment; 37 38 cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"), 39 cl::cat(BoltOptCategory)); 40 41 static cl::list<std::string> 42 BreakFunctionNames("break-funcs", 43 cl::CommaSeparated, 44 cl::desc("list of functions to core dump on (debugging)"), 45 cl::value_desc("func1,func2,func3,..."), 46 cl::Hidden, 47 cl::cat(BoltCategory)); 48 49 static cl::list<std::string> 50 FunctionPadSpec("pad-funcs", cl::CommaSeparated, 51 cl::desc("list of functions to pad with amount of bytes"), 52 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), 53 cl::Hidden, cl::cat(BoltCategory)); 54 55 static cl::list<std::string> FunctionPadBeforeSpec( 56 "pad-funcs-before", cl::CommaSeparated, 57 cl::desc("list of functions to pad with amount of bytes"), 58 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), cl::Hidden, 59 cl::cat(BoltCategory)); 60 61 static cl::opt<bool> MarkFuncs( 62 "mark-funcs", 63 cl::desc("mark function boundaries with break instruction to make " 64 "sure we accidentally don't cross them"), 65 cl::ReallyHidden, cl::cat(BoltCategory)); 66 67 static cl::opt<bool> PrintJumpTables("print-jump-tables", 68 cl::desc("print jump tables"), cl::Hidden, 69 cl::cat(BoltCategory)); 70 71 static cl::opt<bool> 72 X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only", 73 cl::desc("only apply branch boundary alignment in hot code"), 74 cl::init(true), 75 cl::cat(BoltOptCategory)); 76 77 size_t padFunction(std::map<std::string, size_t> &FunctionPadding, 78 const cl::list<std::string> &Spec, 79 const BinaryFunction &Function) { 80 if (FunctionPadding.empty() && !Spec.empty()) { 81 for (const std::string &Spec : Spec) { 82 size_t N = Spec.find(':'); 83 if (N == std::string::npos) 84 continue; 85 std::string Name = Spec.substr(0, N); 86 size_t Padding = std::stoull(Spec.substr(N + 1)); 87 FunctionPadding[Name] = Padding; 88 } 89 } 90 91 for (auto &FPI : FunctionPadding) { 92 std::string Name = FPI.first; 93 size_t Padding = FPI.second; 94 if (Function.hasNameRegex(Name)) 95 return Padding; 96 } 97 98 return 0; 99 } 100 101 size_t padFunctionBefore(const BinaryFunction &Function) { 102 static std::map<std::string, size_t> CacheFunctionPadding; 103 return padFunction(CacheFunctionPadding, FunctionPadBeforeSpec, Function); 104 } 105 size_t padFunctionAfter(const BinaryFunction &Function) { 106 static std::map<std::string, size_t> CacheFunctionPadding; 107 return padFunction(CacheFunctionPadding, FunctionPadSpec, Function); 108 } 109 110 } // namespace opts 111 112 namespace { 113 using JumpTable = bolt::JumpTable; 114 115 class BinaryEmitter { 116 private: 117 BinaryEmitter(const BinaryEmitter &) = delete; 118 BinaryEmitter &operator=(const BinaryEmitter &) = delete; 119 120 MCStreamer &Streamer; 121 BinaryContext &BC; 122 123 public: 124 BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC) 125 : Streamer(Streamer), BC(BC) {} 126 127 /// Emit all code and data. 128 void emitAll(StringRef OrgSecPrefix); 129 130 /// Emit function code. The caller is responsible for emitting function 131 /// symbol(s) and setting the section to emit the code to. 132 void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF, 133 bool EmitCodeOnly = false); 134 135 private: 136 /// Emit function code. 137 void emitFunctions(); 138 139 /// Emit a single function. 140 bool emitFunction(BinaryFunction &BF, FunctionFragment &FF); 141 142 /// Helper for emitFunctionBody to write data inside a function 143 /// (used for AArch64) 144 void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, 145 BinaryFunction *OnBehalfOf = nullptr); 146 147 /// Emit jump tables for the function. 148 void emitJumpTables(const BinaryFunction &BF); 149 150 /// Emit jump table data. Callee supplies sections for the data. 151 void emitJumpTable(const JumpTable &JT, MCSection *HotSection, 152 MCSection *ColdSection); 153 154 void emitCFIInstruction(const MCCFIInstruction &Inst) const; 155 156 /// Emit exception handling ranges for the function fragment. 157 void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF); 158 159 /// Emit line number information corresponding to \p NewLoc. \p PrevLoc 160 /// provides a context for de-duplication of line number info. 161 /// \p FirstInstr indicates if \p NewLoc represents the first instruction 162 /// in a sequence, such as a function fragment. 163 /// 164 /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be 165 /// created and the function will return \p PrevLoc while \p InstrLabel will 166 /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the 167 /// corresponding instruction by emitting \p InstrLabel before it. 168 /// If \p InstrLabel is set by the caller, its value will be used with \p 169 /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer 170 /// to a new temp symbol used with \p NewLoc. 171 /// 172 /// Return new current location which is either \p NewLoc or \p PrevLoc. 173 SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc, 174 bool FirstInstr, MCSymbol *&InstrLabel); 175 176 /// Use \p FunctionEndSymbol to mark the end of the line info sequence. 177 /// Note that it does not automatically result in the insertion of the EOS 178 /// marker in the line table program, but provides one to the DWARF generator 179 /// when it needs it. 180 void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol); 181 182 /// Emit debug line info for unprocessed functions from CUs that include 183 /// emitted functions. 184 void emitDebugLineInfoForOriginalFunctions(); 185 186 /// Emit debug line for CUs that were not modified. 187 void emitDebugLineInfoForUnprocessedCUs(); 188 189 /// Emit data sections that have code references in them. 190 void emitDataSections(StringRef OrgSecPrefix); 191 }; 192 193 } // anonymous namespace 194 195 void BinaryEmitter::emitAll(StringRef OrgSecPrefix) { 196 Streamer.initSections(false, *BC.STI); 197 Streamer.setUseAssemblerInfoForParsing(false); 198 199 if (opts::UpdateDebugSections && BC.isELF()) { 200 // Force the emission of debug line info into allocatable section to ensure 201 // JITLink will process it. 202 // 203 // NB: on MachO all sections are required for execution, hence no need 204 // to change flags/attributes. 205 MCSectionELF *ELFDwarfLineSection = 206 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection()); 207 ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC); 208 MCSectionELF *ELFDwarfLineStrSection = 209 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection()); 210 ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC); 211 } 212 213 if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary()) 214 RtLibrary->emitBinary(BC, Streamer); 215 216 BC.getTextSection()->setAlignment(Align(opts::AlignText)); 217 218 emitFunctions(); 219 220 if (opts::UpdateDebugSections) { 221 emitDebugLineInfoForOriginalFunctions(); 222 DwarfLineTable::emit(BC, Streamer); 223 } 224 225 emitDataSections(OrgSecPrefix); 226 227 // TODO Enable for Mach-O once BinaryContext::getDataSection supports it. 228 if (BC.isELF()) 229 AddressMap::emit(Streamer, BC); 230 Streamer.setUseAssemblerInfoForParsing(true); 231 } 232 233 void BinaryEmitter::emitFunctions() { 234 auto emit = [&](const std::vector<BinaryFunction *> &Functions) { 235 const bool HasProfile = BC.NumProfiledFuncs > 0; 236 const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding(); 237 for (BinaryFunction *Function : Functions) { 238 if (!BC.shouldEmit(*Function)) 239 continue; 240 241 LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function 242 << "\" : " << Function->getFunctionNumber() << '\n'); 243 244 // Was any part of the function emitted. 245 bool Emitted = false; 246 247 // Turn off Intel JCC Erratum mitigation for cold code if requested 248 if (HasProfile && opts::X86AlignBranchBoundaryHotOnly && 249 !Function->hasValidProfile()) 250 Streamer.setAllowAutoPadding(false); 251 252 FunctionLayout &Layout = Function->getLayout(); 253 Emitted |= emitFunction(*Function, Layout.getMainFragment()); 254 255 if (Function->isSplit()) { 256 if (opts::X86AlignBranchBoundaryHotOnly) 257 Streamer.setAllowAutoPadding(false); 258 259 assert((Layout.fragment_size() == 1 || Function->isSimple()) && 260 "Only simple functions can have fragments"); 261 for (FunctionFragment &FF : Layout.getSplitFragments()) { 262 // Skip empty fragments so no symbols and sections for empty fragments 263 // are generated 264 if (FF.empty() && !Function->hasConstantIsland()) 265 continue; 266 Emitted |= emitFunction(*Function, FF); 267 } 268 } 269 270 Streamer.setAllowAutoPadding(OriginalAllowAutoPadding); 271 272 if (Emitted) 273 Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics); 274 } 275 }; 276 277 // Mark the start of hot text. 278 if (opts::HotText) { 279 Streamer.switchSection(BC.getTextSection()); 280 Streamer.emitLabel(BC.getHotTextStartSymbol()); 281 } 282 283 // Emit functions in sorted order. 284 std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions(); 285 emit(SortedFunctions); 286 287 // Emit functions added by BOLT. 288 emit(BC.getInjectedBinaryFunctions()); 289 290 // Mark the end of hot text. 291 if (opts::HotText) { 292 if (BC.HasWarmSection) 293 Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName())); 294 else 295 Streamer.switchSection(BC.getTextSection()); 296 Streamer.emitLabel(BC.getHotTextEndSymbol()); 297 } 298 } 299 300 bool BinaryEmitter::emitFunction(BinaryFunction &Function, 301 FunctionFragment &FF) { 302 if (Function.size() == 0 && !Function.hasIslandsInfo()) 303 return false; 304 305 if (Function.getState() == BinaryFunction::State::Empty) 306 return false; 307 308 // Avoid emitting function without instructions when overwriting the original 309 // function in-place. Otherwise, emit the empty function to define the symbol. 310 if (!BC.HasRelocations && !Function.hasNonPseudoInstructions()) 311 return false; 312 313 MCSection *Section = 314 BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum())); 315 Streamer.switchSection(Section); 316 Section->setHasInstructions(true); 317 BC.Ctx->addGenDwarfSection(Section); 318 319 if (BC.HasRelocations) { 320 // Set section alignment to at least maximum possible object alignment. 321 // We need this to support LongJmp and other passes that calculates 322 // tentative layout. 323 Section->ensureMinAlignment(Align(opts::AlignFunctions)); 324 325 Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI); 326 uint16_t MaxAlignBytes = FF.isSplitFragment() 327 ? Function.getMaxColdAlignmentBytes() 328 : Function.getMaxAlignmentBytes(); 329 if (MaxAlignBytes > 0) 330 Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes); 331 } else { 332 Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI); 333 } 334 335 if (size_t Padding = opts::padFunctionBefore(Function)) { 336 // Handle padFuncsBefore after the above alignment logic but before 337 // symbol addresses are decided. 338 if (!BC.HasRelocations) { 339 BC.errs() << "BOLT-ERROR: -pad-before-funcs is not supported in " 340 << "non-relocation mode\n"; 341 exit(1); 342 } 343 344 // Preserve Function.getMinAlign(). 345 if (!isAligned(Function.getMinAlign(), Padding)) { 346 BC.errs() << "BOLT-ERROR: user-requested " << Padding 347 << " padding bytes before function " << Function 348 << " is not a multiple of the minimum function alignment (" 349 << Function.getMinAlign().value() << ").\n"; 350 exit(1); 351 } 352 353 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding before function " << Function 354 << " with " << Padding << " bytes\n"); 355 356 // Since the padding is not executed, it can be null bytes. 357 Streamer.emitFill(Padding, 0); 358 } 359 360 MCContext &Context = Streamer.getContext(); 361 const MCAsmInfo *MAI = Context.getAsmInfo(); 362 363 MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum()); 364 365 // Emit all symbols associated with the main function entry. 366 if (FF.isMainFragment()) { 367 for (MCSymbol *Symbol : Function.getSymbols()) { 368 Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction); 369 Streamer.emitLabel(Symbol); 370 } 371 } else { 372 Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction); 373 Streamer.emitLabel(StartSymbol); 374 } 375 376 // Emit CFI start 377 if (Function.hasCFI()) { 378 Streamer.emitCFIStartProc(/*IsSimple=*/false); 379 if (Function.getPersonalityFunction() != nullptr) 380 Streamer.emitCFIPersonality(Function.getPersonalityFunction(), 381 Function.getPersonalityEncoding()); 382 MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum()); 383 if (LSDASymbol) 384 Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding); 385 else 386 Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit); 387 // Emit CFI instructions relative to the CIE 388 for (const MCCFIInstruction &CFIInstr : Function.cie()) { 389 // Only write CIE CFI insns that LLVM will not already emit 390 const std::vector<MCCFIInstruction> &FrameInstrs = 391 MAI->getInitialFrameState(); 392 if (!llvm::is_contained(FrameInstrs, CFIInstr)) 393 emitCFIInstruction(CFIInstr); 394 } 395 } 396 397 assert((Function.empty() || !(*Function.begin()).isCold()) && 398 "first basic block should never be cold"); 399 400 // Emit UD2 at the beginning if requested by user. 401 if (!opts::BreakFunctionNames.empty()) { 402 for (std::string &Name : opts::BreakFunctionNames) { 403 if (Function.hasNameRegex(Name)) { 404 Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B 405 break; 406 } 407 } 408 } 409 410 // Emit code. 411 emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false); 412 413 // Emit padding if requested. 414 if (size_t Padding = opts::padFunctionAfter(Function)) { 415 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with " 416 << Padding << " bytes\n"); 417 Streamer.emitFill(Padding, MAI->getTextAlignFillValue()); 418 } 419 420 if (opts::MarkFuncs) 421 Streamer.emitBytes(BC.MIB->getTrapFillValue()); 422 423 // Emit CFI end 424 if (Function.hasCFI()) 425 Streamer.emitCFIEndProc(); 426 427 MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum()); 428 Streamer.emitLabel(EndSymbol); 429 430 if (MAI->hasDotTypeDotSizeDirective()) { 431 const MCExpr *SizeExpr = MCBinaryExpr::createSub( 432 MCSymbolRefExpr::create(EndSymbol, Context), 433 MCSymbolRefExpr::create(StartSymbol, Context), Context); 434 Streamer.emitELFSize(StartSymbol, SizeExpr); 435 } 436 437 if (opts::UpdateDebugSections && Function.getDWARFUnit()) 438 emitLineInfoEnd(Function, EndSymbol); 439 440 // Exception handling info for the function. 441 emitLSDA(Function, FF); 442 443 if (FF.isMainFragment() && opts::JumpTables > JTS_NONE) 444 emitJumpTables(Function); 445 446 return true; 447 } 448 449 void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF, 450 bool EmitCodeOnly) { 451 if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) { 452 assert(BF.getLayout().isHotColdSplit() && 453 "Constant island support only with hot/cold split"); 454 BF.duplicateConstantIslands(); 455 } 456 457 // Track the first emitted instruction with debug info. 458 bool FirstInstr = true; 459 for (BinaryBasicBlock *const BB : FF) { 460 if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) && 461 BB->getAlignment() > 1) 462 Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI, 463 BB->getAlignmentMaxBytes()); 464 Streamer.emitLabel(BB->getLabel()); 465 if (!EmitCodeOnly) { 466 if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB)) 467 Streamer.emitLabel(EntrySymbol); 468 } 469 470 SMLoc LastLocSeen; 471 for (auto I = BB->begin(), E = BB->end(); I != E; ++I) { 472 MCInst &Instr = *I; 473 474 if (EmitCodeOnly && BC.MIB->isPseudo(Instr)) 475 continue; 476 477 // Handle pseudo instructions. 478 if (BC.MIB->isCFI(Instr)) { 479 emitCFIInstruction(*BF.getCFIFor(Instr)); 480 continue; 481 } 482 483 if (!EmitCodeOnly) { 484 // A symbol to be emitted before the instruction to mark its location. 485 MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr); 486 487 if (opts::UpdateDebugSections && BF.getDWARFUnit()) { 488 LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen, 489 FirstInstr, InstrLabel); 490 FirstInstr = false; 491 } 492 493 // Prepare to tag this location with a label if we need to keep track of 494 // the location of calls/returns for BOLT address translation maps 495 if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) { 496 const uint32_t Offset = *BC.MIB->getOffset(Instr); 497 if (!InstrLabel) 498 InstrLabel = BC.Ctx->createTempSymbol(); 499 BB->getLocSyms().emplace_back(Offset, InstrLabel); 500 } 501 502 if (InstrLabel) 503 Streamer.emitLabel(InstrLabel); 504 } 505 506 // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86. 507 // This is a workaround for invalid NOPs handling by asm/disasm layer. 508 if (BC.isX86() && BC.MIB->isNoop(Instr)) { 509 if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) { 510 SmallString<15> Code; 511 raw_svector_ostream VecOS(Code); 512 BC.MAB->writeNopData(VecOS, *Size, BC.STI.get()); 513 Streamer.emitBytes(Code); 514 continue; 515 } 516 } 517 518 Streamer.emitInstruction(Instr, *BC.STI); 519 } 520 } 521 522 if (!EmitCodeOnly) 523 emitConstantIslands(BF, FF.isSplitFragment()); 524 } 525 526 void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, 527 BinaryFunction *OnBehalfOf) { 528 if (!BF.hasIslandsInfo()) 529 return; 530 531 BinaryFunction::IslandInfo &Islands = BF.getIslandInfo(); 532 if (Islands.DataOffsets.empty() && Islands.Dependency.empty()) 533 return; 534 535 // AArch64 requires CI to be aligned to 8 bytes due to access instructions 536 // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes. 537 const uint16_t Alignment = OnBehalfOf 538 ? OnBehalfOf->getConstantIslandAlignment() 539 : BF.getConstantIslandAlignment(); 540 Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI); 541 542 if (!OnBehalfOf) { 543 if (!EmitColdPart) 544 Streamer.emitLabel(BF.getFunctionConstantIslandLabel()); 545 else 546 Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel()); 547 } 548 549 assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) && 550 "spurious OnBehalfOf constant island emission"); 551 552 assert(!BF.isInjected() && 553 "injected functions should not have constant islands"); 554 // Raw contents of the function. 555 StringRef SectionContents = BF.getOriginSection()->getContents(); 556 557 // Raw contents of the function. 558 StringRef FunctionContents = SectionContents.substr( 559 BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize()); 560 561 if (opts::Verbosity && !OnBehalfOf) 562 BC.outs() << "BOLT-INFO: emitting constant island for function " << BF 563 << "\n"; 564 565 // We split the island into smaller blocks and output labels between them. 566 auto IS = Islands.Offsets.begin(); 567 for (auto DataIter = Islands.DataOffsets.begin(); 568 DataIter != Islands.DataOffsets.end(); ++DataIter) { 569 uint64_t FunctionOffset = *DataIter; 570 uint64_t EndOffset = 0ULL; 571 572 // Determine size of this data chunk 573 auto NextData = std::next(DataIter); 574 auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter); 575 if (CodeIter == Islands.CodeOffsets.end() && 576 NextData == Islands.DataOffsets.end()) 577 EndOffset = BF.getMaxSize(); 578 else if (CodeIter == Islands.CodeOffsets.end()) 579 EndOffset = *NextData; 580 else if (NextData == Islands.DataOffsets.end()) 581 EndOffset = *CodeIter; 582 else 583 EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter; 584 585 if (FunctionOffset == EndOffset) 586 continue; // Size is zero, nothing to emit 587 588 auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) { 589 if (FunctionOffset >= EndOffset) 590 return; 591 592 for (auto It = Islands.Relocations.lower_bound(FunctionOffset); 593 It != Islands.Relocations.end(); ++It) { 594 if (It->first >= EndOffset) 595 break; 596 597 const Relocation &Relocation = It->second; 598 if (FunctionOffset < Relocation.Offset) { 599 Streamer.emitBytes( 600 FunctionContents.slice(FunctionOffset, Relocation.Offset)); 601 FunctionOffset = Relocation.Offset; 602 } 603 604 LLVM_DEBUG( 605 dbgs() << "BOLT-DEBUG: emitting constant island relocation" 606 << " for " << BF << " at offset 0x" 607 << Twine::utohexstr(Relocation.Offset) << " with size " 608 << Relocation::getSizeForType(Relocation.Type) << '\n'); 609 610 FunctionOffset += Relocation.emit(&Streamer); 611 } 612 613 assert(FunctionOffset <= EndOffset && "overflow error"); 614 if (FunctionOffset < EndOffset) { 615 Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset)); 616 FunctionOffset = EndOffset; 617 } 618 }; 619 620 // Emit labels, relocs and data 621 while (IS != Islands.Offsets.end() && IS->first < EndOffset) { 622 auto NextLabelOffset = 623 IS == Islands.Offsets.end() ? EndOffset : IS->first; 624 auto NextStop = std::min(NextLabelOffset, EndOffset); 625 assert(NextStop <= EndOffset && "internal overflow error"); 626 emitCI(FunctionOffset, NextStop); 627 if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) { 628 // This is a slightly complex code to decide which label to emit. We 629 // have 4 cases to handle: regular symbol, cold symbol, regular or cold 630 // symbol being emitted on behalf of an external function. 631 if (!OnBehalfOf) { 632 if (!EmitColdPart) { 633 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " 634 << IS->second->getName() << " at offset 0x" 635 << Twine::utohexstr(IS->first) << '\n'); 636 if (IS->second->isUndefined()) 637 Streamer.emitLabel(IS->second); 638 else 639 assert(BF.hasName(std::string(IS->second->getName()))); 640 } else if (Islands.ColdSymbols.count(IS->second) != 0) { 641 LLVM_DEBUG(dbgs() 642 << "BOLT-DEBUG: emitted label " 643 << Islands.ColdSymbols[IS->second]->getName() << '\n'); 644 if (Islands.ColdSymbols[IS->second]->isUndefined()) 645 Streamer.emitLabel(Islands.ColdSymbols[IS->second]); 646 } 647 } else { 648 if (!EmitColdPart) { 649 if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) { 650 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " 651 << Sym->getName() << '\n'); 652 Streamer.emitLabel(Sym); 653 } 654 } else if (MCSymbol *Sym = 655 Islands.ColdProxies[OnBehalfOf][IS->second]) { 656 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName() 657 << '\n'); 658 Streamer.emitLabel(Sym); 659 } 660 } 661 ++IS; 662 } 663 } 664 assert(FunctionOffset <= EndOffset && "overflow error"); 665 emitCI(FunctionOffset, EndOffset); 666 } 667 assert(IS == Islands.Offsets.end() && "some symbols were not emitted!"); 668 669 if (OnBehalfOf) 670 return; 671 // Now emit constant islands from other functions that we may have used in 672 // this function. 673 for (BinaryFunction *ExternalFunc : Islands.Dependency) 674 emitConstantIslands(*ExternalFunc, EmitColdPart, &BF); 675 } 676 677 SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, 678 SMLoc PrevLoc, bool FirstInstr, 679 MCSymbol *&InstrLabel) { 680 DWARFUnit *FunctionCU = BF.getDWARFUnit(); 681 const DWARFDebugLine::LineTable *FunctionLineTable = BF.getDWARFLineTable(); 682 assert(FunctionCU && "cannot emit line info for function without CU"); 683 684 DebugLineTableRowRef RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc); 685 686 // Check if no new line info needs to be emitted. 687 if (RowReference == DebugLineTableRowRef::NULL_ROW || 688 NewLoc.getPointer() == PrevLoc.getPointer()) 689 return PrevLoc; 690 691 unsigned CurrentFilenum = 0; 692 const DWARFDebugLine::LineTable *CurrentLineTable = FunctionLineTable; 693 694 // If the CU id from the current instruction location does not 695 // match the CU id from the current function, it means that we 696 // have come across some inlined code. We must look up the CU 697 // for the instruction's original function and get the line table 698 // from that. 699 const uint64_t FunctionUnitIndex = FunctionCU->getOffset(); 700 const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex; 701 if (CurrentUnitIndex != FunctionUnitIndex) { 702 CurrentLineTable = BC.DwCtx->getLineTableForUnit( 703 BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex)); 704 // Add filename from the inlined function to the current CU. 705 CurrentFilenum = BC.addDebugFilenameToUnit( 706 FunctionUnitIndex, CurrentUnitIndex, 707 CurrentLineTable->Rows[RowReference.RowIndex - 1].File); 708 } 709 710 const DWARFDebugLine::Row &CurrentRow = 711 CurrentLineTable->Rows[RowReference.RowIndex - 1]; 712 if (!CurrentFilenum) 713 CurrentFilenum = CurrentRow.File; 714 715 unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) | 716 (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) | 717 (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) | 718 (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin); 719 720 // Always emit is_stmt at the beginning of function fragment. 721 if (FirstInstr) 722 Flags |= DWARF2_FLAG_IS_STMT; 723 724 BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column, 725 Flags, CurrentRow.Isa, CurrentRow.Discriminator); 726 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc(); 727 BC.Ctx->clearDwarfLocSeen(); 728 729 if (!InstrLabel) 730 InstrLabel = BC.Ctx->createTempSymbol(); 731 732 BC.getDwarfLineTable(FunctionUnitIndex) 733 .getMCLineSections() 734 .addLineEntry(MCDwarfLineEntry(InstrLabel, DwarfLoc), 735 Streamer.getCurrentSectionOnly()); 736 737 return NewLoc; 738 } 739 740 void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF, 741 MCSymbol *FunctionEndLabel) { 742 DWARFUnit *FunctionCU = BF.getDWARFUnit(); 743 assert(FunctionCU && "DWARF unit expected"); 744 BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0); 745 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc(); 746 BC.Ctx->clearDwarfLocSeen(); 747 BC.getDwarfLineTable(FunctionCU->getOffset()) 748 .getMCLineSections() 749 .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc), 750 Streamer.getCurrentSectionOnly()); 751 } 752 753 void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) { 754 MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection(); 755 MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection( 756 ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC); 757 758 if (!BF.hasJumpTables()) 759 return; 760 761 if (opts::PrintJumpTables) 762 BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n"; 763 764 for (auto &JTI : BF.jumpTables()) { 765 JumpTable &JT = *JTI.second; 766 // Only emit shared jump tables once, when processing the first parent 767 if (JT.Parents.size() > 1 && JT.Parents[0] != &BF) 768 continue; 769 if (opts::PrintJumpTables) 770 JT.print(BC.outs()); 771 if (opts::JumpTables == JTS_BASIC) { 772 JT.updateOriginal(); 773 } else { 774 MCSection *HotSection, *ColdSection; 775 if (BF.isSimple()) { 776 HotSection = ReadOnlySection; 777 ColdSection = ReadOnlyColdSection; 778 } else { 779 HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection; 780 ColdSection = HotSection; 781 } 782 emitJumpTable(JT, HotSection, ColdSection); 783 } 784 } 785 } 786 787 void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection, 788 MCSection *ColdSection) { 789 // Pre-process entries for aggressive splitting. 790 // Each label represents a separate switch table and gets its own count 791 // determining its destination. 792 std::map<MCSymbol *, uint64_t> LabelCounts; 793 if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) { 794 auto It = JT.Labels.find(0); 795 assert(It != JT.Labels.end()); 796 MCSymbol *CurrentLabel = It->second; 797 uint64_t CurrentLabelCount = 0; 798 for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) { 799 auto LI = JT.Labels.find(Index * JT.EntrySize); 800 if (LI != JT.Labels.end()) { 801 LabelCounts[CurrentLabel] = CurrentLabelCount; 802 CurrentLabel = LI->second; 803 CurrentLabelCount = 0; 804 } 805 CurrentLabelCount += JT.Counts[Index].Count; 806 } 807 LabelCounts[CurrentLabel] = CurrentLabelCount; 808 } else { 809 Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection); 810 Streamer.emitValueToAlignment(Align(JT.EntrySize)); 811 } 812 MCSymbol *LastLabel = nullptr; 813 uint64_t Offset = 0; 814 for (MCSymbol *Entry : JT.Entries) { 815 auto LI = JT.Labels.find(Offset); 816 if (LI != JT.Labels.end()) { 817 LLVM_DEBUG({ 818 dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName() 819 << " (originally was at address 0x" 820 << Twine::utohexstr(JT.getAddress() + Offset) 821 << (Offset ? ") as part of larger jump table\n" : ")\n"); 822 }); 823 if (!LabelCounts.empty()) { 824 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " 825 << LabelCounts[LI->second] << '\n'); 826 if (LabelCounts[LI->second] > 0) 827 Streamer.switchSection(HotSection); 828 else 829 Streamer.switchSection(ColdSection); 830 Streamer.emitValueToAlignment(Align(JT.EntrySize)); 831 } 832 // Emit all labels registered at the address of this jump table 833 // to sync with our global symbol table. We may have two labels 834 // registered at this address if one label was created via 835 // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing 836 // this location) and another via getOrCreateJumpTable(). This 837 // creates a race where the symbols created by these two 838 // functions may or may not be the same, but they are both 839 // registered in our symbol table at the same address. By 840 // emitting them all here we make sure there is no ambiguity 841 // that depends on the order that these symbols were created, so 842 // whenever this address is referenced in the binary, it is 843 // certain to point to the jump table identified at this 844 // address. 845 if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) { 846 for (MCSymbol *S : BD->getSymbols()) 847 Streamer.emitLabel(S); 848 } else { 849 Streamer.emitLabel(LI->second); 850 } 851 LastLabel = LI->second; 852 } 853 if (JT.Type == JumpTable::JTT_NORMAL) { 854 Streamer.emitSymbolValue(Entry, JT.OutputEntrySize); 855 } else { // JTT_PIC 856 const MCSymbolRefExpr *JTExpr = 857 MCSymbolRefExpr::create(LastLabel, Streamer.getContext()); 858 const MCSymbolRefExpr *E = 859 MCSymbolRefExpr::create(Entry, Streamer.getContext()); 860 const MCBinaryExpr *Value = 861 MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext()); 862 Streamer.emitValue(Value, JT.EntrySize); 863 } 864 Offset += JT.EntrySize; 865 } 866 } 867 868 void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const { 869 switch (Inst.getOperation()) { 870 default: 871 llvm_unreachable("Unexpected instruction"); 872 case MCCFIInstruction::OpDefCfaOffset: 873 Streamer.emitCFIDefCfaOffset(Inst.getOffset()); 874 break; 875 case MCCFIInstruction::OpAdjustCfaOffset: 876 Streamer.emitCFIAdjustCfaOffset(Inst.getOffset()); 877 break; 878 case MCCFIInstruction::OpDefCfa: 879 Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset()); 880 break; 881 case MCCFIInstruction::OpDefCfaRegister: 882 Streamer.emitCFIDefCfaRegister(Inst.getRegister()); 883 break; 884 case MCCFIInstruction::OpOffset: 885 Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset()); 886 break; 887 case MCCFIInstruction::OpRegister: 888 Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2()); 889 break; 890 case MCCFIInstruction::OpWindowSave: 891 Streamer.emitCFIWindowSave(); 892 break; 893 case MCCFIInstruction::OpNegateRAState: 894 Streamer.emitCFINegateRAState(); 895 break; 896 case MCCFIInstruction::OpSameValue: 897 Streamer.emitCFISameValue(Inst.getRegister()); 898 break; 899 case MCCFIInstruction::OpGnuArgsSize: 900 Streamer.emitCFIGnuArgsSize(Inst.getOffset()); 901 break; 902 case MCCFIInstruction::OpEscape: 903 Streamer.AddComment(Inst.getComment()); 904 Streamer.emitCFIEscape(Inst.getValues()); 905 break; 906 case MCCFIInstruction::OpRestore: 907 Streamer.emitCFIRestore(Inst.getRegister()); 908 break; 909 case MCCFIInstruction::OpUndefined: 910 Streamer.emitCFIUndefined(Inst.getRegister()); 911 break; 912 } 913 } 914 915 // The code is based on EHStreamer::emitExceptionTable(). 916 void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) { 917 const BinaryFunction::CallSitesRange Sites = 918 BF.getCallSites(FF.getFragmentNum()); 919 if (Sites.empty()) 920 return; 921 922 Streamer.switchSection(BC.MOFI->getLSDASection()); 923 924 const unsigned TTypeEncoding = BF.getLSDATypeEncoding(); 925 const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding); 926 const uint16_t TTypeAlignment = 4; 927 928 // Type tables have to be aligned at 4 bytes. 929 Streamer.emitValueToAlignment(Align(TTypeAlignment)); 930 931 // Emit the LSDA label. 932 MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum()); 933 assert(LSDASymbol && "no LSDA symbol set"); 934 Streamer.emitLabel(LSDASymbol); 935 936 // Corresponding FDE start. 937 const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum()); 938 939 // Emit the LSDA header. 940 941 // If LPStart is omitted, then the start of the FDE is used as a base for 942 // landing pad displacements. Then, if a cold fragment starts with 943 // a landing pad, this means that the first landing pad offset will be 0. 944 // However, C++ runtime will treat 0 as if there is no landing pad, thus we 945 // cannot emit LP offset as 0. 946 // 947 // As a solution, for fixed-address binaries we set LPStart to 0, and for 948 // position-independent binaries we offset LP start by one byte. 949 bool NeedsLPAdjustment = false; 950 std::function<void(const MCSymbol *)> emitLandingPad; 951 952 // Check if there's a symbol associated with a landing pad fragment. 953 const MCSymbol *LPStartSymbol = BF.getLPStartSymbol(FF.getFragmentNum()); 954 if (!LPStartSymbol) { 955 // Since landing pads are not in the same fragment, we fall back to emitting 956 // absolute addresses for this FDE. 957 if (opts::Verbosity >= 2) { 958 BC.outs() << "BOLT-INFO: falling back to generating absolute-address " 959 << "exception ranges for " << BF << '\n'; 960 } 961 962 assert(BC.HasFixedLoadAddress && 963 "Cannot emit absolute-address landing pads for PIE/DSO"); 964 965 Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format 966 Streamer.emitIntValue(0, 4); // LPStart 967 emitLandingPad = [&](const MCSymbol *LPSymbol) { 968 if (LPSymbol) 969 Streamer.emitSymbolValue(LPSymbol, 4); 970 else 971 Streamer.emitIntValue(0, 4); 972 }; 973 } else { 974 std::optional<FragmentNum> LPFN = BF.getLPFragment(FF.getFragmentNum()); 975 const FunctionFragment &LPFragment = BF.getLayout().getFragment(*LPFN); 976 NeedsLPAdjustment = 977 (!LPFragment.empty() && LPFragment.front()->isLandingPad()); 978 979 // Emit LPStart encoding and optionally LPStart. 980 if (NeedsLPAdjustment || LPStartSymbol != StartSymbol) { 981 Streamer.emitIntValue(dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4, 1); 982 MCSymbol *DotSymbol = BC.Ctx->createTempSymbol("LPBase"); 983 Streamer.emitLabel(DotSymbol); 984 985 const MCExpr *LPStartExpr = MCBinaryExpr::createSub( 986 MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx), 987 MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx); 988 if (NeedsLPAdjustment) 989 LPStartExpr = MCBinaryExpr::createSub( 990 LPStartExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx); 991 Streamer.emitValue(LPStartExpr, 4); 992 } else { 993 // DW_EH_PE_omit means FDE start (StartSymbol) will be used as LPStart. 994 Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); 995 } 996 emitLandingPad = [&](const MCSymbol *LPSymbol) { 997 if (LPSymbol) { 998 const MCExpr *LPOffsetExpr = MCBinaryExpr::createSub( 999 MCSymbolRefExpr::create(LPSymbol, *BC.Ctx), 1000 MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx), *BC.Ctx); 1001 if (NeedsLPAdjustment) 1002 LPOffsetExpr = MCBinaryExpr::createAdd( 1003 LPOffsetExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx); 1004 Streamer.emitULEB128Value(LPOffsetExpr); 1005 } else { 1006 Streamer.emitULEB128IntValue(0); 1007 } 1008 }; 1009 } 1010 1011 Streamer.emitIntValue(TTypeEncoding, 1); // TType format 1012 1013 MCSymbol *TTBaseLabel = nullptr; 1014 if (TTypeEncoding != dwarf::DW_EH_PE_omit) { 1015 TTBaseLabel = BC.Ctx->createTempSymbol("TTBase"); 1016 MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol("TTBaseRef"); 1017 Streamer.emitAbsoluteSymbolDiffAsULEB128(TTBaseLabel, TTBaseRefLabel); 1018 Streamer.emitLabel(TTBaseRefLabel); 1019 } 1020 1021 // Emit encoding of entries in the call site table. The format is used for the 1022 // call site start, length, and corresponding landing pad. 1023 if (!LPStartSymbol) 1024 Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1); 1025 else 1026 Streamer.emitIntValue(dwarf::DW_EH_PE_uleb128, 1); 1027 1028 MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol("CSTStart"); 1029 MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol("CSTEnd"); 1030 Streamer.emitAbsoluteSymbolDiffAsULEB128(CSTEndLabel, CSTStartLabel); 1031 1032 Streamer.emitLabel(CSTStartLabel); 1033 for (const auto &FragmentCallSite : Sites) { 1034 const BinaryFunction::CallSite &CallSite = FragmentCallSite.second; 1035 const MCSymbol *BeginLabel = CallSite.Start; 1036 const MCSymbol *EndLabel = CallSite.End; 1037 1038 assert(BeginLabel && "start EH label expected"); 1039 assert(EndLabel && "end EH label expected"); 1040 1041 // Start of the range is emitted relative to the start of current 1042 // function split part. 1043 if (!LPStartSymbol) { 1044 Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4); 1045 Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4); 1046 } else { 1047 Streamer.emitAbsoluteSymbolDiffAsULEB128(BeginLabel, StartSymbol); 1048 Streamer.emitAbsoluteSymbolDiffAsULEB128(EndLabel, BeginLabel); 1049 } 1050 emitLandingPad(CallSite.LP); 1051 Streamer.emitULEB128IntValue(CallSite.Action); 1052 } 1053 Streamer.emitLabel(CSTEndLabel); 1054 1055 // Write out action, type, and type index tables at the end. 1056 // 1057 // For action and type index tables there's no need to change the original 1058 // table format unless we are doing function splitting, in which case we can 1059 // split and optimize the tables. 1060 // 1061 // For type table we (re-)encode the table using TTypeEncoding matching 1062 // the current assembler mode. 1063 for (uint8_t const &Byte : BF.getLSDAActionTable()) 1064 Streamer.emitIntValue(Byte, 1); 1065 1066 const BinaryFunction::LSDATypeTableTy &TypeTable = 1067 (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable() 1068 : BF.getLSDATypeTable(); 1069 assert(TypeTable.size() == BF.getLSDATypeTable().size() && 1070 "indirect type table size mismatch"); 1071 1072 Streamer.emitValueToAlignment(Align(TTypeAlignment)); 1073 1074 for (int Index = TypeTable.size() - 1; Index >= 0; --Index) { 1075 const uint64_t TypeAddress = TypeTable[Index]; 1076 switch (TTypeEncoding & 0x70) { 1077 default: 1078 llvm_unreachable("unsupported TTypeEncoding"); 1079 case dwarf::DW_EH_PE_absptr: 1080 Streamer.emitIntValue(TypeAddress, TTypeEncodingSize); 1081 break; 1082 case dwarf::DW_EH_PE_pcrel: { 1083 if (TypeAddress) { 1084 const MCSymbol *TypeSymbol = 1085 BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment); 1086 MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol(); 1087 Streamer.emitLabel(DotSymbol); 1088 const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub( 1089 MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx), 1090 MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx); 1091 Streamer.emitValue(SubDotExpr, TTypeEncodingSize); 1092 } else { 1093 Streamer.emitIntValue(0, TTypeEncodingSize); 1094 } 1095 break; 1096 } 1097 } 1098 } 1099 1100 if (TTypeEncoding != dwarf::DW_EH_PE_omit) 1101 Streamer.emitLabel(TTBaseLabel); 1102 1103 for (uint8_t const &Byte : BF.getLSDATypeIndexTable()) 1104 Streamer.emitIntValue(Byte, 1); 1105 } 1106 1107 void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() { 1108 // If a function is in a CU containing at least one processed function, we 1109 // have to rewrite the whole line table for that CU. For unprocessed functions 1110 // we use data from the input line table. 1111 for (auto &It : BC.getBinaryFunctions()) { 1112 const BinaryFunction &Function = It.second; 1113 1114 // If the function was emitted, its line info was emitted with it. 1115 if (Function.isEmitted()) 1116 continue; 1117 1118 const DWARFDebugLine::LineTable *LineTable = Function.getDWARFLineTable(); 1119 if (!LineTable) 1120 continue; // nothing to update for this function 1121 1122 const uint64_t Address = Function.getAddress(); 1123 std::vector<uint32_t> Results; 1124 if (!LineTable->lookupAddressRange( 1125 {Address, object::SectionedAddress::UndefSection}, 1126 Function.getSize(), Results)) 1127 continue; 1128 1129 if (Results.empty()) 1130 continue; 1131 1132 // The first row returned could be the last row matching the start address. 1133 // Find the first row with the same address that is not the end of the 1134 // sequence. 1135 uint64_t FirstRow = Results.front(); 1136 while (FirstRow > 0) { 1137 const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1]; 1138 if (PrevRow.Address.Address != Address || PrevRow.EndSequence) 1139 break; 1140 --FirstRow; 1141 } 1142 1143 const uint64_t EndOfSequenceAddress = 1144 Function.getAddress() + Function.getMaxSize(); 1145 BC.getDwarfLineTable(Function.getDWARFUnit()->getOffset()) 1146 .addLineTableSequence(LineTable, FirstRow, Results.back(), 1147 EndOfSequenceAddress); 1148 } 1149 1150 // For units that are completely unprocessed, use original debug line contents 1151 // eliminating the need to regenerate line info program. 1152 emitDebugLineInfoForUnprocessedCUs(); 1153 } 1154 1155 void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() { 1156 // Sorted list of section offsets provides boundaries for section fragments, 1157 // where each fragment is the unit's contribution to debug line section. 1158 std::vector<uint64_t> StmtListOffsets; 1159 StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits()); 1160 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) { 1161 DWARFDie CUDie = CU->getUnitDIE(); 1162 auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list)); 1163 if (!StmtList) 1164 continue; 1165 1166 StmtListOffsets.push_back(*StmtList); 1167 } 1168 llvm::sort(StmtListOffsets); 1169 1170 // For each CU that was not processed, emit its line info as a binary blob. 1171 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) { 1172 if (BC.ProcessedCUs.count(CU.get())) 1173 continue; 1174 1175 DWARFDie CUDie = CU->getUnitDIE(); 1176 auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list)); 1177 if (!StmtList) 1178 continue; 1179 1180 StringRef DebugLineContents = CU->getLineSection().Data; 1181 1182 const uint64_t Begin = *StmtList; 1183 1184 // Statement list ends where the next unit contribution begins, or at the 1185 // end of the section. 1186 auto It = llvm::upper_bound(StmtListOffsets, Begin); 1187 const uint64_t End = 1188 It == StmtListOffsets.end() ? DebugLineContents.size() : *It; 1189 1190 BC.getDwarfLineTable(CU->getOffset()) 1191 .addRawContents(DebugLineContents.slice(Begin, End)); 1192 } 1193 } 1194 1195 void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) { 1196 for (BinarySection &Section : BC.sections()) { 1197 if (!Section.hasRelocations()) 1198 continue; 1199 1200 StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : ""; 1201 Section.emitAsData(Streamer, Prefix + Section.getName()); 1202 Section.clearRelocations(); 1203 } 1204 } 1205 1206 namespace llvm { 1207 namespace bolt { 1208 1209 void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC, 1210 StringRef OrgSecPrefix) { 1211 BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix); 1212 } 1213 1214 void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF, 1215 FunctionFragment &FF, bool EmitCodeOnly) { 1216 BinaryEmitter(Streamer, BF.getBinaryContext()) 1217 .emitFunctionBody(BF, FF, EmitCodeOnly); 1218 } 1219 1220 } // namespace bolt 1221 } // namespace llvm 1222