1 //===---- MachineOutliner.cpp - Outline instructions -----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// Replaces repeated sequences of instructions with function calls. 11 /// 12 /// This works by placing every instruction from every basic block in a 13 /// suffix tree, and repeatedly querying that tree for repeated sequences of 14 /// instructions. If a sequence of instructions appears often, then it ought 15 /// to be beneficial to pull out into a function. 16 /// 17 /// The MachineOutliner communicates with a given target using hooks defined in 18 /// TargetInstrInfo.h. The target supplies the outliner with information on how 19 /// a specific sequence of instructions should be outlined. This information 20 /// is used to deduce the number of instructions necessary to 21 /// 22 /// * Create an outlined function 23 /// * Call that outlined function 24 /// 25 /// Targets must implement 26 /// * getOutliningCandidateInfo 27 /// * buildOutlinedFrame 28 /// * insertOutlinedCall 29 /// * isFunctionSafeToOutlineFrom 30 /// 31 /// in order to make use of the MachineOutliner. 32 /// 33 /// This was originally presented at the 2016 LLVM Developers' Meeting in the 34 /// talk "Reducing Code Size Using Outlining". For a high-level overview of 35 /// how this pass works, the talk is available on YouTube at 36 /// 37 /// https://www.youtube.com/watch?v=yorld-WSOeU 38 /// 39 /// The slides for the talk are available at 40 /// 41 /// http://www.llvm.org/devmtg/2016-11/Slides/Paquette-Outliner.pdf 42 /// 43 /// The talk provides an overview of how the outliner finds candidates and 44 /// ultimately outlines them. It describes how the main data structure for this 45 /// pass, the suffix tree, is queried and purged for candidates. It also gives 46 /// a simplified suffix tree construction algorithm for suffix trees based off 47 /// of the algorithm actually used here, Ukkonen's algorithm. 48 /// 49 /// For the original RFC for this pass, please see 50 /// 51 /// http://lists.llvm.org/pipermail/llvm-dev/2016-August/104170.html 52 /// 53 /// For more information on the suffix tree data structure, please see 54 /// https://www.cs.helsinki.fi/u/ukkonen/SuffixT1withFigs.pdf 55 /// 56 //===----------------------------------------------------------------------===// 57 #include "llvm/CodeGen/MachineOutliner.h" 58 #include "llvm/ADT/DenseMap.h" 59 #include "llvm/ADT/SmallSet.h" 60 #include "llvm/ADT/Statistic.h" 61 #include "llvm/ADT/Twine.h" 62 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 63 #include "llvm/CodeGen/LivePhysRegs.h" 64 #include "llvm/CodeGen/MachineModuleInfo.h" 65 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" 66 #include "llvm/CodeGen/Passes.h" 67 #include "llvm/CodeGen/TargetInstrInfo.h" 68 #include "llvm/CodeGen/TargetSubtargetInfo.h" 69 #include "llvm/IR/DIBuilder.h" 70 #include "llvm/IR/IRBuilder.h" 71 #include "llvm/IR/Mangler.h" 72 #include "llvm/InitializePasses.h" 73 #include "llvm/Support/CommandLine.h" 74 #include "llvm/Support/Debug.h" 75 #include "llvm/Support/SuffixTree.h" 76 #include "llvm/Support/raw_ostream.h" 77 #include <functional> 78 #include <tuple> 79 #include <vector> 80 81 #define DEBUG_TYPE "machine-outliner" 82 83 using namespace llvm; 84 using namespace ore; 85 using namespace outliner; 86 87 // Statistics for outlined functions. 88 STATISTIC(NumOutlined, "Number of candidates outlined"); 89 STATISTIC(FunctionsCreated, "Number of functions created"); 90 91 // Statistics for instruction mapping. 92 STATISTIC(NumLegalInUnsignedVec, "Outlinable instructions mapped"); 93 STATISTIC(NumIllegalInUnsignedVec, 94 "Unoutlinable instructions mapped + number of sentinel values"); 95 STATISTIC(NumSentinels, "Sentinel values inserted during mapping"); 96 STATISTIC(NumInvisible, 97 "Invisible instructions skipped during mapping"); 98 STATISTIC(UnsignedVecSize, 99 "Total number of instructions mapped and saved to mapping vector"); 100 101 // Set to true if the user wants the outliner to run on linkonceodr linkage 102 // functions. This is false by default because the linker can dedupe linkonceodr 103 // functions. Since the outliner is confined to a single module (modulo LTO), 104 // this is off by default. It should, however, be the default behaviour in 105 // LTO. 106 static cl::opt<bool> EnableLinkOnceODROutlining( 107 "enable-linkonceodr-outlining", cl::Hidden, 108 cl::desc("Enable the machine outliner on linkonceodr functions"), 109 cl::init(false)); 110 111 /// Number of times to re-run the outliner. This is not the total number of runs 112 /// as the outliner will run at least one time. The default value is set to 0, 113 /// meaning the outliner will run one time and rerun zero times after that. 114 static cl::opt<unsigned> OutlinerReruns( 115 "machine-outliner-reruns", cl::init(0), cl::Hidden, 116 cl::desc( 117 "Number of times to rerun the outliner after the initial outline")); 118 119 static cl::opt<unsigned> OutlinerBenefitThreshold( 120 "outliner-benefit-threshold", cl::init(1), cl::Hidden, 121 cl::desc( 122 "The minimum size in bytes before an outlining candidate is accepted")); 123 124 static cl::opt<bool> OutlinerLeafDescendants( 125 "outliner-leaf-descendants", cl::init(true), cl::Hidden, 126 cl::desc("Consider all leaf descendants of internal nodes of the suffix " 127 "tree as candidates for outlining (if false, only leaf children " 128 "are considered)")); 129 130 namespace { 131 132 /// Maps \p MachineInstrs to unsigned integers and stores the mappings. 133 struct InstructionMapper { 134 135 /// The next available integer to assign to a \p MachineInstr that 136 /// cannot be outlined. 137 /// 138 /// Set to -3 for compatability with \p DenseMapInfo<unsigned>. 139 unsigned IllegalInstrNumber = -3; 140 141 /// The next available integer to assign to a \p MachineInstr that can 142 /// be outlined. 143 unsigned LegalInstrNumber = 0; 144 145 /// Correspondence from \p MachineInstrs to unsigned integers. 146 DenseMap<MachineInstr *, unsigned, MachineInstrExpressionTrait> 147 InstructionIntegerMap; 148 149 /// Correspondence between \p MachineBasicBlocks and target-defined flags. 150 DenseMap<MachineBasicBlock *, unsigned> MBBFlagsMap; 151 152 /// The vector of unsigned integers that the module is mapped to. 153 SmallVector<unsigned> UnsignedVec; 154 155 /// Stores the location of the instruction associated with the integer 156 /// at index i in \p UnsignedVec for each index i. 157 SmallVector<MachineBasicBlock::iterator> InstrList; 158 159 // Set if we added an illegal number in the previous step. 160 // Since each illegal number is unique, we only need one of them between 161 // each range of legal numbers. This lets us make sure we don't add more 162 // than one illegal number per range. 163 bool AddedIllegalLastTime = false; 164 165 /// Maps \p *It to a legal integer. 166 /// 167 /// Updates \p CanOutlineWithPrevInstr, \p HaveLegalRange, \p InstrListForMBB, 168 /// \p UnsignedVecForMBB, \p InstructionIntegerMap, and \p LegalInstrNumber. 169 /// 170 /// \returns The integer that \p *It was mapped to. 171 unsigned mapToLegalUnsigned( 172 MachineBasicBlock::iterator &It, bool &CanOutlineWithPrevInstr, 173 bool &HaveLegalRange, unsigned &NumLegalInBlock, 174 SmallVector<unsigned> &UnsignedVecForMBB, 175 SmallVector<MachineBasicBlock::iterator> &InstrListForMBB) { 176 // We added something legal, so we should unset the AddedLegalLastTime 177 // flag. 178 AddedIllegalLastTime = false; 179 180 // If we have at least two adjacent legal instructions (which may have 181 // invisible instructions in between), remember that. 182 if (CanOutlineWithPrevInstr) 183 HaveLegalRange = true; 184 CanOutlineWithPrevInstr = true; 185 186 // Keep track of the number of legal instructions we insert. 187 NumLegalInBlock++; 188 189 // Get the integer for this instruction or give it the current 190 // LegalInstrNumber. 191 InstrListForMBB.push_back(It); 192 MachineInstr &MI = *It; 193 bool WasInserted; 194 DenseMap<MachineInstr *, unsigned, MachineInstrExpressionTrait>::iterator 195 ResultIt; 196 std::tie(ResultIt, WasInserted) = 197 InstructionIntegerMap.insert(std::make_pair(&MI, LegalInstrNumber)); 198 unsigned MINumber = ResultIt->second; 199 200 // There was an insertion. 201 if (WasInserted) 202 LegalInstrNumber++; 203 204 UnsignedVecForMBB.push_back(MINumber); 205 206 // Make sure we don't overflow or use any integers reserved by the DenseMap. 207 if (LegalInstrNumber >= IllegalInstrNumber) 208 report_fatal_error("Instruction mapping overflow!"); 209 210 assert(LegalInstrNumber != DenseMapInfo<unsigned>::getEmptyKey() && 211 "Tried to assign DenseMap tombstone or empty key to instruction."); 212 assert(LegalInstrNumber != DenseMapInfo<unsigned>::getTombstoneKey() && 213 "Tried to assign DenseMap tombstone or empty key to instruction."); 214 215 // Statistics. 216 ++NumLegalInUnsignedVec; 217 return MINumber; 218 } 219 220 /// Maps \p *It to an illegal integer. 221 /// 222 /// Updates \p InstrListForMBB, \p UnsignedVecForMBB, and \p 223 /// IllegalInstrNumber. 224 /// 225 /// \returns The integer that \p *It was mapped to. 226 unsigned mapToIllegalUnsigned( 227 MachineBasicBlock::iterator &It, bool &CanOutlineWithPrevInstr, 228 SmallVector<unsigned> &UnsignedVecForMBB, 229 SmallVector<MachineBasicBlock::iterator> &InstrListForMBB) { 230 // Can't outline an illegal instruction. Set the flag. 231 CanOutlineWithPrevInstr = false; 232 233 // Only add one illegal number per range of legal numbers. 234 if (AddedIllegalLastTime) 235 return IllegalInstrNumber; 236 237 // Remember that we added an illegal number last time. 238 AddedIllegalLastTime = true; 239 unsigned MINumber = IllegalInstrNumber; 240 241 InstrListForMBB.push_back(It); 242 UnsignedVecForMBB.push_back(IllegalInstrNumber); 243 IllegalInstrNumber--; 244 // Statistics. 245 ++NumIllegalInUnsignedVec; 246 247 assert(LegalInstrNumber < IllegalInstrNumber && 248 "Instruction mapping overflow!"); 249 250 assert(IllegalInstrNumber != DenseMapInfo<unsigned>::getEmptyKey() && 251 "IllegalInstrNumber cannot be DenseMap tombstone or empty key!"); 252 253 assert(IllegalInstrNumber != DenseMapInfo<unsigned>::getTombstoneKey() && 254 "IllegalInstrNumber cannot be DenseMap tombstone or empty key!"); 255 256 return MINumber; 257 } 258 259 /// Transforms a \p MachineBasicBlock into a \p vector of \p unsigneds 260 /// and appends it to \p UnsignedVec and \p InstrList. 261 /// 262 /// Two instructions are assigned the same integer if they are identical. 263 /// If an instruction is deemed unsafe to outline, then it will be assigned an 264 /// unique integer. The resulting mapping is placed into a suffix tree and 265 /// queried for candidates. 266 /// 267 /// \param MBB The \p MachineBasicBlock to be translated into integers. 268 /// \param TII \p TargetInstrInfo for the function. 269 void convertToUnsignedVec(MachineBasicBlock &MBB, 270 const TargetInstrInfo &TII) { 271 LLVM_DEBUG(dbgs() << "*** Converting MBB '" << MBB.getName() 272 << "' to unsigned vector ***\n"); 273 unsigned Flags = 0; 274 275 // Don't even map in this case. 276 if (!TII.isMBBSafeToOutlineFrom(MBB, Flags)) 277 return; 278 279 auto OutlinableRanges = TII.getOutlinableRanges(MBB, Flags); 280 LLVM_DEBUG(dbgs() << MBB.getName() << ": " << OutlinableRanges.size() 281 << " outlinable range(s)\n"); 282 if (OutlinableRanges.empty()) 283 return; 284 285 // Store info for the MBB for later outlining. 286 MBBFlagsMap[&MBB] = Flags; 287 288 MachineBasicBlock::iterator It = MBB.begin(); 289 290 // The number of instructions in this block that will be considered for 291 // outlining. 292 unsigned NumLegalInBlock = 0; 293 294 // True if we have at least two legal instructions which aren't separated 295 // by an illegal instruction. 296 bool HaveLegalRange = false; 297 298 // True if we can perform outlining given the last mapped (non-invisible) 299 // instruction. This lets us know if we have a legal range. 300 bool CanOutlineWithPrevInstr = false; 301 302 // FIXME: Should this all just be handled in the target, rather than using 303 // repeated calls to getOutliningType? 304 SmallVector<unsigned> UnsignedVecForMBB; 305 SmallVector<MachineBasicBlock::iterator> InstrListForMBB; 306 307 LLVM_DEBUG(dbgs() << "*** Mapping outlinable ranges ***\n"); 308 for (auto &OutlinableRange : OutlinableRanges) { 309 auto OutlinableRangeBegin = OutlinableRange.first; 310 auto OutlinableRangeEnd = OutlinableRange.second; 311 #ifndef NDEBUG 312 LLVM_DEBUG( 313 dbgs() << "Mapping " 314 << std::distance(OutlinableRangeBegin, OutlinableRangeEnd) 315 << " instruction range\n"); 316 // Everything outside of an outlinable range is illegal. 317 unsigned NumSkippedInRange = 0; 318 #endif 319 for (; It != OutlinableRangeBegin; ++It) { 320 #ifndef NDEBUG 321 ++NumSkippedInRange; 322 #endif 323 mapToIllegalUnsigned(It, CanOutlineWithPrevInstr, UnsignedVecForMBB, 324 InstrListForMBB); 325 } 326 #ifndef NDEBUG 327 LLVM_DEBUG(dbgs() << "Skipped " << NumSkippedInRange 328 << " instructions outside outlinable range\n"); 329 #endif 330 assert(It != MBB.end() && "Should still have instructions?"); 331 // `It` is now positioned at the beginning of a range of instructions 332 // which may be outlinable. Check if each instruction is known to be safe. 333 for (; It != OutlinableRangeEnd; ++It) { 334 // Keep track of where this instruction is in the module. 335 switch (TII.getOutliningType(It, Flags)) { 336 case InstrType::Illegal: 337 mapToIllegalUnsigned(It, CanOutlineWithPrevInstr, UnsignedVecForMBB, 338 InstrListForMBB); 339 break; 340 341 case InstrType::Legal: 342 mapToLegalUnsigned(It, CanOutlineWithPrevInstr, HaveLegalRange, 343 NumLegalInBlock, UnsignedVecForMBB, 344 InstrListForMBB); 345 break; 346 347 case InstrType::LegalTerminator: 348 mapToLegalUnsigned(It, CanOutlineWithPrevInstr, HaveLegalRange, 349 NumLegalInBlock, UnsignedVecForMBB, 350 InstrListForMBB); 351 // The instruction also acts as a terminator, so we have to record 352 // that in the string. 353 mapToIllegalUnsigned(It, CanOutlineWithPrevInstr, UnsignedVecForMBB, 354 InstrListForMBB); 355 break; 356 357 case InstrType::Invisible: 358 // Normally this is set by mapTo(Blah)Unsigned, but we just want to 359 // skip this instruction. So, unset the flag here. 360 ++NumInvisible; 361 AddedIllegalLastTime = false; 362 break; 363 } 364 } 365 } 366 367 LLVM_DEBUG(dbgs() << "HaveLegalRange = " << HaveLegalRange << "\n"); 368 369 // Are there enough legal instructions in the block for outlining to be 370 // possible? 371 if (HaveLegalRange) { 372 // After we're done every insertion, uniquely terminate this part of the 373 // "string". This makes sure we won't match across basic block or function 374 // boundaries since the "end" is encoded uniquely and thus appears in no 375 // repeated substring. 376 mapToIllegalUnsigned(It, CanOutlineWithPrevInstr, UnsignedVecForMBB, 377 InstrListForMBB); 378 ++NumSentinels; 379 append_range(InstrList, InstrListForMBB); 380 append_range(UnsignedVec, UnsignedVecForMBB); 381 } 382 } 383 384 InstructionMapper() { 385 // Make sure that the implementation of DenseMapInfo<unsigned> hasn't 386 // changed. 387 assert(DenseMapInfo<unsigned>::getEmptyKey() == (unsigned)-1 && 388 "DenseMapInfo<unsigned>'s empty key isn't -1!"); 389 assert(DenseMapInfo<unsigned>::getTombstoneKey() == (unsigned)-2 && 390 "DenseMapInfo<unsigned>'s tombstone key isn't -2!"); 391 } 392 }; 393 394 /// An interprocedural pass which finds repeated sequences of 395 /// instructions and replaces them with calls to functions. 396 /// 397 /// Each instruction is mapped to an unsigned integer and placed in a string. 398 /// The resulting mapping is then placed in a \p SuffixTree. The \p SuffixTree 399 /// is then repeatedly queried for repeated sequences of instructions. Each 400 /// non-overlapping repeated sequence is then placed in its own 401 /// \p MachineFunction and each instance is then replaced with a call to that 402 /// function. 403 struct MachineOutliner : public ModulePass { 404 405 static char ID; 406 407 /// Set to true if the outliner should consider functions with 408 /// linkonceodr linkage. 409 bool OutlineFromLinkOnceODRs = false; 410 411 /// The current repeat number of machine outlining. 412 unsigned OutlineRepeatedNum = 0; 413 414 /// Set to true if the outliner should run on all functions in the module 415 /// considered safe for outlining. 416 /// Set to true by default for compatibility with llc's -run-pass option. 417 /// Set when the pass is constructed in TargetPassConfig. 418 bool RunOnAllFunctions = true; 419 420 StringRef getPassName() const override { return "Machine Outliner"; } 421 422 void getAnalysisUsage(AnalysisUsage &AU) const override { 423 AU.addRequired<MachineModuleInfoWrapperPass>(); 424 AU.addPreserved<MachineModuleInfoWrapperPass>(); 425 AU.setPreservesAll(); 426 ModulePass::getAnalysisUsage(AU); 427 } 428 429 MachineOutliner() : ModulePass(ID) { 430 initializeMachineOutlinerPass(*PassRegistry::getPassRegistry()); 431 } 432 433 /// Remark output explaining that not outlining a set of candidates would be 434 /// better than outlining that set. 435 void emitNotOutliningCheaperRemark( 436 unsigned StringLen, std::vector<Candidate> &CandidatesForRepeatedSeq, 437 OutlinedFunction &OF); 438 439 /// Remark output explaining that a function was outlined. 440 void emitOutlinedFunctionRemark(OutlinedFunction &OF); 441 442 /// Find all repeated substrings that satisfy the outlining cost model by 443 /// constructing a suffix tree. 444 /// 445 /// If a substring appears at least twice, then it must be represented by 446 /// an internal node which appears in at least two suffixes. Each suffix 447 /// is represented by a leaf node. To do this, we visit each internal node 448 /// in the tree, using the leaf children of each internal node. If an 449 /// internal node represents a beneficial substring, then we use each of 450 /// its leaf children to find the locations of its substring. 451 /// 452 /// \param Mapper Contains outlining mapping information. 453 /// \param[out] FunctionList Filled with a list of \p OutlinedFunctions 454 /// each type of candidate. 455 void findCandidates(InstructionMapper &Mapper, 456 std::vector<OutlinedFunction> &FunctionList); 457 458 /// Replace the sequences of instructions represented by \p OutlinedFunctions 459 /// with calls to functions. 460 /// 461 /// \param M The module we are outlining from. 462 /// \param FunctionList A list of functions to be inserted into the module. 463 /// \param Mapper Contains the instruction mappings for the module. 464 bool outline(Module &M, std::vector<OutlinedFunction> &FunctionList, 465 InstructionMapper &Mapper, unsigned &OutlinedFunctionNum); 466 467 /// Creates a function for \p OF and inserts it into the module. 468 MachineFunction *createOutlinedFunction(Module &M, OutlinedFunction &OF, 469 InstructionMapper &Mapper, 470 unsigned Name); 471 472 /// Calls 'doOutline()' 1 + OutlinerReruns times. 473 bool runOnModule(Module &M) override; 474 475 /// Construct a suffix tree on the instructions in \p M and outline repeated 476 /// strings from that tree. 477 bool doOutline(Module &M, unsigned &OutlinedFunctionNum); 478 479 /// Return a DISubprogram for OF if one exists, and null otherwise. Helper 480 /// function for remark emission. 481 DISubprogram *getSubprogramOrNull(const OutlinedFunction &OF) { 482 for (const Candidate &C : OF.Candidates) 483 if (MachineFunction *MF = C.getMF()) 484 if (DISubprogram *SP = MF->getFunction().getSubprogram()) 485 return SP; 486 return nullptr; 487 } 488 489 /// Populate and \p InstructionMapper with instruction-to-integer mappings. 490 /// These are used to construct a suffix tree. 491 void populateMapper(InstructionMapper &Mapper, Module &M, 492 MachineModuleInfo &MMI); 493 494 /// Initialize information necessary to output a size remark. 495 /// FIXME: This should be handled by the pass manager, not the outliner. 496 /// FIXME: This is nearly identical to the initSizeRemarkInfo in the legacy 497 /// pass manager. 498 void initSizeRemarkInfo(const Module &M, const MachineModuleInfo &MMI, 499 StringMap<unsigned> &FunctionToInstrCount); 500 501 /// Emit the remark. 502 // FIXME: This should be handled by the pass manager, not the outliner. 503 void 504 emitInstrCountChangedRemark(const Module &M, const MachineModuleInfo &MMI, 505 const StringMap<unsigned> &FunctionToInstrCount); 506 }; 507 } // Anonymous namespace. 508 509 char MachineOutliner::ID = 0; 510 511 namespace llvm { 512 ModulePass *createMachineOutlinerPass(bool RunOnAllFunctions) { 513 MachineOutliner *OL = new MachineOutliner(); 514 OL->RunOnAllFunctions = RunOnAllFunctions; 515 return OL; 516 } 517 518 } // namespace llvm 519 520 INITIALIZE_PASS(MachineOutliner, DEBUG_TYPE, "Machine Function Outliner", false, 521 false) 522 523 void MachineOutliner::emitNotOutliningCheaperRemark( 524 unsigned StringLen, std::vector<Candidate> &CandidatesForRepeatedSeq, 525 OutlinedFunction &OF) { 526 // FIXME: Right now, we arbitrarily choose some Candidate from the 527 // OutlinedFunction. This isn't necessarily fixed, nor does it have to be. 528 // We should probably sort these by function name or something to make sure 529 // the remarks are stable. 530 Candidate &C = CandidatesForRepeatedSeq.front(); 531 MachineOptimizationRemarkEmitter MORE(*(C.getMF()), nullptr); 532 MORE.emit([&]() { 533 MachineOptimizationRemarkMissed R(DEBUG_TYPE, "NotOutliningCheaper", 534 C.front().getDebugLoc(), C.getMBB()); 535 R << "Did not outline " << NV("Length", StringLen) << " instructions" 536 << " from " << NV("NumOccurrences", CandidatesForRepeatedSeq.size()) 537 << " locations." 538 << " Bytes from outlining all occurrences (" 539 << NV("OutliningCost", OF.getOutliningCost()) << ")" 540 << " >= Unoutlined instruction bytes (" 541 << NV("NotOutliningCost", OF.getNotOutlinedCost()) << ")" 542 << " (Also found at: "; 543 544 // Tell the user the other places the candidate was found. 545 for (unsigned i = 1, e = CandidatesForRepeatedSeq.size(); i < e; i++) { 546 R << NV((Twine("OtherStartLoc") + Twine(i)).str(), 547 CandidatesForRepeatedSeq[i].front().getDebugLoc()); 548 if (i != e - 1) 549 R << ", "; 550 } 551 552 R << ")"; 553 return R; 554 }); 555 } 556 557 void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) { 558 MachineBasicBlock *MBB = &*OF.MF->begin(); 559 MachineOptimizationRemarkEmitter MORE(*OF.MF, nullptr); 560 MachineOptimizationRemark R(DEBUG_TYPE, "OutlinedFunction", 561 MBB->findDebugLoc(MBB->begin()), MBB); 562 R << "Saved " << NV("OutliningBenefit", OF.getBenefit()) << " bytes by " 563 << "outlining " << NV("Length", OF.getNumInstrs()) << " instructions " 564 << "from " << NV("NumOccurrences", OF.getOccurrenceCount()) 565 << " locations. " 566 << "(Found at: "; 567 568 // Tell the user the other places the candidate was found. 569 for (size_t i = 0, e = OF.Candidates.size(); i < e; i++) { 570 571 R << NV((Twine("StartLoc") + Twine(i)).str(), 572 OF.Candidates[i].front().getDebugLoc()); 573 if (i != e - 1) 574 R << ", "; 575 } 576 577 R << ")"; 578 579 MORE.emit(R); 580 } 581 582 void MachineOutliner::findCandidates( 583 InstructionMapper &Mapper, std::vector<OutlinedFunction> &FunctionList) { 584 FunctionList.clear(); 585 SuffixTree ST(Mapper.UnsignedVec, OutlinerLeafDescendants); 586 587 // First, find all of the repeated substrings in the tree of minimum length 588 // 2. 589 std::vector<Candidate> CandidatesForRepeatedSeq; 590 LLVM_DEBUG(dbgs() << "*** Discarding overlapping candidates *** \n"); 591 LLVM_DEBUG( 592 dbgs() << "Searching for overlaps in all repeated sequences...\n"); 593 for (SuffixTree::RepeatedSubstring &RS : ST) { 594 CandidatesForRepeatedSeq.clear(); 595 unsigned StringLen = RS.Length; 596 LLVM_DEBUG(dbgs() << " Sequence length: " << StringLen << "\n"); 597 // Debug code to keep track of how many candidates we removed. 598 #ifndef NDEBUG 599 unsigned NumDiscarded = 0; 600 unsigned NumKept = 0; 601 #endif 602 // Sort the start indices so that we can efficiently check if candidates 603 // overlap with the ones we've already found for this sequence. 604 llvm::sort(RS.StartIndices); 605 for (const unsigned &StartIdx : RS.StartIndices) { 606 // Trick: Discard some candidates that would be incompatible with the 607 // ones we've already found for this sequence. This will save us some 608 // work in candidate selection. 609 // 610 // If two candidates overlap, then we can't outline them both. This 611 // happens when we have candidates that look like, say 612 // 613 // AA (where each "A" is an instruction). 614 // 615 // We might have some portion of the module that looks like this: 616 // AAAAAA (6 A's) 617 // 618 // In this case, there are 5 different copies of "AA" in this range, but 619 // at most 3 can be outlined. If only outlining 3 of these is going to 620 // be unbeneficial, then we ought to not bother. 621 // 622 // Note that two things DON'T overlap when they look like this: 623 // start1...end1 .... start2...end2 624 // That is, one must either 625 // * End before the other starts 626 // * Start after the other ends 627 unsigned EndIdx = StartIdx + StringLen - 1; 628 if (!CandidatesForRepeatedSeq.empty() && 629 StartIdx <= CandidatesForRepeatedSeq.back().getEndIdx()) { 630 #ifndef NDEBUG 631 ++NumDiscarded; 632 LLVM_DEBUG(dbgs() << " .. DISCARD candidate @ [" << StartIdx << ", " 633 << EndIdx << "]; overlaps with candidate @ [" 634 << CandidatesForRepeatedSeq.back().getStartIdx() 635 << ", " << CandidatesForRepeatedSeq.back().getEndIdx() 636 << "]\n"); 637 #endif 638 continue; 639 } 640 // It doesn't overlap with anything, so we can outline it. 641 // Each sequence is over [StartIt, EndIt]. 642 // Save the candidate and its location. 643 #ifndef NDEBUG 644 ++NumKept; 645 #endif 646 MachineBasicBlock::iterator StartIt = Mapper.InstrList[StartIdx]; 647 MachineBasicBlock::iterator EndIt = Mapper.InstrList[EndIdx]; 648 MachineBasicBlock *MBB = StartIt->getParent(); 649 CandidatesForRepeatedSeq.emplace_back(StartIdx, StringLen, StartIt, EndIt, 650 MBB, FunctionList.size(), 651 Mapper.MBBFlagsMap[MBB]); 652 } 653 #ifndef NDEBUG 654 LLVM_DEBUG(dbgs() << " Candidates discarded: " << NumDiscarded 655 << "\n"); 656 LLVM_DEBUG(dbgs() << " Candidates kept: " << NumKept << "\n\n"); 657 #endif 658 659 // We've found something we might want to outline. 660 // Create an OutlinedFunction to store it and check if it'd be beneficial 661 // to outline. 662 if (CandidatesForRepeatedSeq.size() < 2) 663 continue; 664 665 // Arbitrarily choose a TII from the first candidate. 666 // FIXME: Should getOutliningCandidateInfo move to TargetMachine? 667 const TargetInstrInfo *TII = 668 CandidatesForRepeatedSeq[0].getMF()->getSubtarget().getInstrInfo(); 669 670 std::optional<OutlinedFunction> OF = 671 TII->getOutliningCandidateInfo(CandidatesForRepeatedSeq); 672 673 // If we deleted too many candidates, then there's nothing worth outlining. 674 // FIXME: This should take target-specified instruction sizes into account. 675 if (!OF || OF->Candidates.size() < 2) 676 continue; 677 678 // Is it better to outline this candidate than not? 679 if (OF->getBenefit() < OutlinerBenefitThreshold) { 680 emitNotOutliningCheaperRemark(StringLen, CandidatesForRepeatedSeq, *OF); 681 continue; 682 } 683 684 FunctionList.push_back(*OF); 685 } 686 } 687 688 MachineFunction *MachineOutliner::createOutlinedFunction( 689 Module &M, OutlinedFunction &OF, InstructionMapper &Mapper, unsigned Name) { 690 691 // Create the function name. This should be unique. 692 // FIXME: We should have a better naming scheme. This should be stable, 693 // regardless of changes to the outliner's cost model/traversal order. 694 std::string FunctionName = "OUTLINED_FUNCTION_"; 695 if (OutlineRepeatedNum > 0) 696 FunctionName += std::to_string(OutlineRepeatedNum + 1) + "_"; 697 FunctionName += std::to_string(Name); 698 LLVM_DEBUG(dbgs() << "NEW FUNCTION: " << FunctionName << "\n"); 699 700 // Create the function using an IR-level function. 701 LLVMContext &C = M.getContext(); 702 Function *F = Function::Create(FunctionType::get(Type::getVoidTy(C), false), 703 Function::ExternalLinkage, FunctionName, M); 704 705 // NOTE: If this is linkonceodr, then we can take advantage of linker deduping 706 // which gives us better results when we outline from linkonceodr functions. 707 F->setLinkage(GlobalValue::InternalLinkage); 708 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 709 710 // Set optsize/minsize, so we don't insert padding between outlined 711 // functions. 712 F->addFnAttr(Attribute::OptimizeForSize); 713 F->addFnAttr(Attribute::MinSize); 714 715 Candidate &FirstCand = OF.Candidates.front(); 716 const TargetInstrInfo &TII = 717 *FirstCand.getMF()->getSubtarget().getInstrInfo(); 718 719 TII.mergeOutliningCandidateAttributes(*F, OF.Candidates); 720 721 // Set uwtable, so we generate eh_frame. 722 UWTableKind UW = std::accumulate( 723 OF.Candidates.cbegin(), OF.Candidates.cend(), UWTableKind::None, 724 [](UWTableKind K, const outliner::Candidate &C) { 725 return std::max(K, C.getMF()->getFunction().getUWTableKind()); 726 }); 727 F->setUWTableKind(UW); 728 729 BasicBlock *EntryBB = BasicBlock::Create(C, "entry", F); 730 IRBuilder<> Builder(EntryBB); 731 Builder.CreateRetVoid(); 732 733 MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); 734 MachineFunction &MF = MMI.getOrCreateMachineFunction(*F); 735 MF.setIsOutlined(true); 736 MachineBasicBlock &MBB = *MF.CreateMachineBasicBlock(); 737 738 // Insert the new function into the module. 739 MF.insert(MF.begin(), &MBB); 740 741 MachineFunction *OriginalMF = FirstCand.front().getMF(); 742 const std::vector<MCCFIInstruction> &Instrs = 743 OriginalMF->getFrameInstructions(); 744 for (auto &MI : FirstCand) { 745 if (MI.isDebugInstr()) 746 continue; 747 748 // Don't keep debug information for outlined instructions. 749 auto DL = DebugLoc(); 750 if (MI.isCFIInstruction()) { 751 unsigned CFIIndex = MI.getOperand(0).getCFIIndex(); 752 MCCFIInstruction CFI = Instrs[CFIIndex]; 753 BuildMI(MBB, MBB.end(), DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) 754 .addCFIIndex(MF.addFrameInst(CFI)); 755 } else { 756 MachineInstr *NewMI = MF.CloneMachineInstr(&MI); 757 NewMI->dropMemRefs(MF); 758 NewMI->setDebugLoc(DL); 759 MBB.insert(MBB.end(), NewMI); 760 } 761 } 762 763 // Set normal properties for a late MachineFunction. 764 MF.getProperties().reset(MachineFunctionProperties::Property::IsSSA); 765 MF.getProperties().set(MachineFunctionProperties::Property::NoPHIs); 766 MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs); 767 MF.getProperties().set(MachineFunctionProperties::Property::TracksLiveness); 768 MF.getRegInfo().freezeReservedRegs(); 769 770 // Compute live-in set for outlined fn 771 const MachineRegisterInfo &MRI = MF.getRegInfo(); 772 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 773 LivePhysRegs LiveIns(TRI); 774 for (auto &Cand : OF.Candidates) { 775 // Figure out live-ins at the first instruction. 776 MachineBasicBlock &OutlineBB = *Cand.front().getParent(); 777 LivePhysRegs CandLiveIns(TRI); 778 CandLiveIns.addLiveOuts(OutlineBB); 779 for (const MachineInstr &MI : 780 reverse(make_range(Cand.begin(), OutlineBB.end()))) 781 CandLiveIns.stepBackward(MI); 782 783 // The live-in set for the outlined function is the union of the live-ins 784 // from all the outlining points. 785 for (MCPhysReg Reg : CandLiveIns) 786 LiveIns.addReg(Reg); 787 } 788 addLiveIns(MBB, LiveIns); 789 790 TII.buildOutlinedFrame(MBB, MF, OF); 791 792 // If there's a DISubprogram associated with this outlined function, then 793 // emit debug info for the outlined function. 794 if (DISubprogram *SP = getSubprogramOrNull(OF)) { 795 // We have a DISubprogram. Get its DICompileUnit. 796 DICompileUnit *CU = SP->getUnit(); 797 DIBuilder DB(M, true, CU); 798 DIFile *Unit = SP->getFile(); 799 Mangler Mg; 800 // Get the mangled name of the function for the linkage name. 801 std::string Dummy; 802 raw_string_ostream MangledNameStream(Dummy); 803 Mg.getNameWithPrefix(MangledNameStream, F, false); 804 805 DISubprogram *OutlinedSP = DB.createFunction( 806 Unit /* Context */, F->getName(), StringRef(MangledNameStream.str()), 807 Unit /* File */, 808 0 /* Line 0 is reserved for compiler-generated code. */, 809 DB.createSubroutineType( 810 DB.getOrCreateTypeArray(std::nullopt)), /* void type */ 811 0, /* Line 0 is reserved for compiler-generated code. */ 812 DINode::DIFlags::FlagArtificial /* Compiler-generated code. */, 813 /* Outlined code is optimized code by definition. */ 814 DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized); 815 816 // Don't add any new variables to the subprogram. 817 DB.finalizeSubprogram(OutlinedSP); 818 819 // Attach subprogram to the function. 820 F->setSubprogram(OutlinedSP); 821 // We're done with the DIBuilder. 822 DB.finalize(); 823 } 824 825 return &MF; 826 } 827 828 bool MachineOutliner::outline(Module &M, 829 std::vector<OutlinedFunction> &FunctionList, 830 InstructionMapper &Mapper, 831 unsigned &OutlinedFunctionNum) { 832 LLVM_DEBUG(dbgs() << "*** Outlining ***\n"); 833 LLVM_DEBUG(dbgs() << "NUMBER OF POTENTIAL FUNCTIONS: " << FunctionList.size() 834 << "\n"); 835 bool OutlinedSomething = false; 836 837 // Sort by priority where priority := getNotOutlinedCost / getOutliningCost. 838 // The function with highest priority should be outlined first. 839 stable_sort(FunctionList, 840 [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) { 841 return LHS.getNotOutlinedCost() * RHS.getOutliningCost() > 842 RHS.getNotOutlinedCost() * LHS.getOutliningCost(); 843 }); 844 845 // Walk over each function, outlining them as we go along. Functions are 846 // outlined greedily, based off the sort above. 847 auto *UnsignedVecBegin = Mapper.UnsignedVec.begin(); 848 LLVM_DEBUG(dbgs() << "WALKING FUNCTION LIST\n"); 849 for (OutlinedFunction &OF : FunctionList) { 850 #ifndef NDEBUG 851 auto NumCandidatesBefore = OF.Candidates.size(); 852 #endif 853 // If we outlined something that overlapped with a candidate in a previous 854 // step, then we can't outline from it. 855 erase_if(OF.Candidates, [&UnsignedVecBegin](Candidate &C) { 856 return std::any_of(UnsignedVecBegin + C.getStartIdx(), 857 UnsignedVecBegin + C.getEndIdx() + 1, [](unsigned I) { 858 return I == static_cast<unsigned>(-1); 859 }); 860 }); 861 862 #ifndef NDEBUG 863 auto NumCandidatesAfter = OF.Candidates.size(); 864 LLVM_DEBUG(dbgs() << "PRUNED: " << NumCandidatesBefore - NumCandidatesAfter 865 << "/" << NumCandidatesBefore << " candidates\n"); 866 #endif 867 868 // If we made it unbeneficial to outline this function, skip it. 869 if (OF.getBenefit() < OutlinerBenefitThreshold) { 870 LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF.getBenefit() 871 << " B) < threshold (" << OutlinerBenefitThreshold 872 << " B)\n"); 873 continue; 874 } 875 876 LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF.getBenefit() 877 << " B) > threshold (" << OutlinerBenefitThreshold 878 << " B)\n"); 879 880 // It's beneficial. Create the function and outline its sequence's 881 // occurrences. 882 OF.MF = createOutlinedFunction(M, OF, Mapper, OutlinedFunctionNum); 883 emitOutlinedFunctionRemark(OF); 884 FunctionsCreated++; 885 OutlinedFunctionNum++; // Created a function, move to the next name. 886 MachineFunction *MF = OF.MF; 887 const TargetSubtargetInfo &STI = MF->getSubtarget(); 888 const TargetInstrInfo &TII = *STI.getInstrInfo(); 889 890 // Replace occurrences of the sequence with calls to the new function. 891 LLVM_DEBUG(dbgs() << "CREATE OUTLINED CALLS\n"); 892 for (Candidate &C : OF.Candidates) { 893 MachineBasicBlock &MBB = *C.getMBB(); 894 MachineBasicBlock::iterator StartIt = C.begin(); 895 MachineBasicBlock::iterator EndIt = std::prev(C.end()); 896 897 // Insert the call. 898 auto CallInst = TII.insertOutlinedCall(M, MBB, StartIt, *MF, C); 899 // Insert the call. 900 #ifndef NDEBUG 901 auto MBBBeingOutlinedFromName = 902 MBB.getName().empty() ? "<unknown>" : MBB.getName().str(); 903 auto MFBeingOutlinedFromName = MBB.getParent()->getName().empty() 904 ? "<unknown>" 905 : MBB.getParent()->getName().str(); 906 LLVM_DEBUG(dbgs() << " CALL: " << MF->getName() << " in " 907 << MFBeingOutlinedFromName << ":" 908 << MBBBeingOutlinedFromName << "\n"); 909 LLVM_DEBUG(dbgs() << " .. " << *CallInst); 910 #endif 911 912 // If the caller tracks liveness, then we need to make sure that 913 // anything we outline doesn't break liveness assumptions. The outlined 914 // functions themselves currently don't track liveness, but we should 915 // make sure that the ranges we yank things out of aren't wrong. 916 if (MBB.getParent()->getProperties().hasProperty( 917 MachineFunctionProperties::Property::TracksLiveness)) { 918 // The following code is to add implicit def operands to the call 919 // instruction. It also updates call site information for moved 920 // code. 921 SmallSet<Register, 2> UseRegs, DefRegs; 922 // Copy over the defs in the outlined range. 923 // First inst in outlined range <-- Anything that's defined in this 924 // ... .. range has to be added as an 925 // implicit Last inst in outlined range <-- def to the call 926 // instruction. Also remove call site information for outlined block 927 // of code. The exposed uses need to be copied in the outlined range. 928 for (MachineBasicBlock::reverse_iterator 929 Iter = EndIt.getReverse(), 930 Last = std::next(CallInst.getReverse()); 931 Iter != Last; Iter++) { 932 MachineInstr *MI = &*Iter; 933 SmallSet<Register, 2> InstrUseRegs; 934 for (MachineOperand &MOP : MI->operands()) { 935 // Skip over anything that isn't a register. 936 if (!MOP.isReg()) 937 continue; 938 939 if (MOP.isDef()) { 940 // Introduce DefRegs set to skip the redundant register. 941 DefRegs.insert(MOP.getReg()); 942 if (UseRegs.count(MOP.getReg()) && 943 !InstrUseRegs.count(MOP.getReg())) 944 // Since the regiester is modeled as defined, 945 // it is not necessary to be put in use register set. 946 UseRegs.erase(MOP.getReg()); 947 } else if (!MOP.isUndef()) { 948 // Any register which is not undefined should 949 // be put in the use register set. 950 UseRegs.insert(MOP.getReg()); 951 InstrUseRegs.insert(MOP.getReg()); 952 } 953 } 954 if (MI->isCandidateForCallSiteEntry()) 955 MI->getMF()->eraseCallSiteInfo(MI); 956 } 957 958 for (const Register &I : DefRegs) 959 // If it's a def, add it to the call instruction. 960 CallInst->addOperand( 961 MachineOperand::CreateReg(I, true, /* isDef = true */ 962 true /* isImp = true */)); 963 964 for (const Register &I : UseRegs) 965 // If it's a exposed use, add it to the call instruction. 966 CallInst->addOperand( 967 MachineOperand::CreateReg(I, false, /* isDef = false */ 968 true /* isImp = true */)); 969 } 970 971 // Erase from the point after where the call was inserted up to, and 972 // including, the final instruction in the sequence. 973 // Erase needs one past the end, so we need std::next there too. 974 MBB.erase(std::next(StartIt), std::next(EndIt)); 975 976 // Keep track of what we removed by marking them all as -1. 977 for (unsigned &I : make_range(UnsignedVecBegin + C.getStartIdx(), 978 UnsignedVecBegin + C.getEndIdx() + 1)) 979 I = static_cast<unsigned>(-1); 980 OutlinedSomething = true; 981 982 // Statistics. 983 NumOutlined++; 984 } 985 } 986 987 LLVM_DEBUG(dbgs() << "OutlinedSomething = " << OutlinedSomething << "\n";); 988 return OutlinedSomething; 989 } 990 991 void MachineOutliner::populateMapper(InstructionMapper &Mapper, Module &M, 992 MachineModuleInfo &MMI) { 993 // Build instruction mappings for each function in the module. Start by 994 // iterating over each Function in M. 995 LLVM_DEBUG(dbgs() << "*** Populating mapper ***\n"); 996 for (Function &F : M) { 997 LLVM_DEBUG(dbgs() << "MAPPING FUNCTION: " << F.getName() << "\n"); 998 999 if (F.hasFnAttribute("nooutline")) { 1000 LLVM_DEBUG(dbgs() << "SKIP: Function has nooutline attribute\n"); 1001 continue; 1002 } 1003 1004 // There's something in F. Check if it has a MachineFunction associated with 1005 // it. 1006 MachineFunction *MF = MMI.getMachineFunction(F); 1007 1008 // If it doesn't, then there's nothing to outline from. Move to the next 1009 // Function. 1010 if (!MF) { 1011 LLVM_DEBUG(dbgs() << "SKIP: Function does not have a MachineFunction\n"); 1012 continue; 1013 } 1014 1015 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); 1016 if (!RunOnAllFunctions && !TII->shouldOutlineFromFunctionByDefault(*MF)) { 1017 LLVM_DEBUG(dbgs() << "SKIP: Target does not want to outline from " 1018 "function by default\n"); 1019 continue; 1020 } 1021 1022 // We have a MachineFunction. Ask the target if it's suitable for outlining. 1023 // If it isn't, then move on to the next Function in the module. 1024 if (!TII->isFunctionSafeToOutlineFrom(*MF, OutlineFromLinkOnceODRs)) { 1025 LLVM_DEBUG(dbgs() << "SKIP: " << MF->getName() 1026 << ": unsafe to outline from\n"); 1027 continue; 1028 } 1029 1030 // We have a function suitable for outlining. Iterate over every 1031 // MachineBasicBlock in MF and try to map its instructions to a list of 1032 // unsigned integers. 1033 const unsigned MinMBBSize = 2; 1034 1035 for (MachineBasicBlock &MBB : *MF) { 1036 LLVM_DEBUG(dbgs() << " MAPPING MBB: '" << MBB.getName() << "'\n"); 1037 // If there isn't anything in MBB, then there's no point in outlining from 1038 // it. 1039 // If there are fewer than 2 instructions in the MBB, then it can't ever 1040 // contain something worth outlining. 1041 // FIXME: This should be based off of the maximum size in B of an outlined 1042 // call versus the size in B of the MBB. 1043 if (MBB.size() < MinMBBSize) { 1044 LLVM_DEBUG(dbgs() << " SKIP: MBB size less than minimum size of " 1045 << MinMBBSize << "\n"); 1046 continue; 1047 } 1048 1049 // Check if MBB could be the target of an indirect branch. If it is, then 1050 // we don't want to outline from it. 1051 if (MBB.hasAddressTaken()) { 1052 LLVM_DEBUG(dbgs() << " SKIP: MBB's address is taken\n"); 1053 continue; 1054 } 1055 1056 // MBB is suitable for outlining. Map it to a list of unsigneds. 1057 Mapper.convertToUnsignedVec(MBB, *TII); 1058 } 1059 } 1060 // Statistics. 1061 UnsignedVecSize = Mapper.UnsignedVec.size(); 1062 } 1063 1064 void MachineOutliner::initSizeRemarkInfo( 1065 const Module &M, const MachineModuleInfo &MMI, 1066 StringMap<unsigned> &FunctionToInstrCount) { 1067 // Collect instruction counts for every function. We'll use this to emit 1068 // per-function size remarks later. 1069 for (const Function &F : M) { 1070 MachineFunction *MF = MMI.getMachineFunction(F); 1071 1072 // We only care about MI counts here. If there's no MachineFunction at this 1073 // point, then there won't be after the outliner runs, so let's move on. 1074 if (!MF) 1075 continue; 1076 FunctionToInstrCount[F.getName().str()] = MF->getInstructionCount(); 1077 } 1078 } 1079 1080 void MachineOutliner::emitInstrCountChangedRemark( 1081 const Module &M, const MachineModuleInfo &MMI, 1082 const StringMap<unsigned> &FunctionToInstrCount) { 1083 // Iterate over each function in the module and emit remarks. 1084 // Note that we won't miss anything by doing this, because the outliner never 1085 // deletes functions. 1086 for (const Function &F : M) { 1087 MachineFunction *MF = MMI.getMachineFunction(F); 1088 1089 // The outliner never deletes functions. If we don't have a MF here, then we 1090 // didn't have one prior to outlining either. 1091 if (!MF) 1092 continue; 1093 1094 std::string Fname = std::string(F.getName()); 1095 unsigned FnCountAfter = MF->getInstructionCount(); 1096 unsigned FnCountBefore = 0; 1097 1098 // Check if the function was recorded before. 1099 auto It = FunctionToInstrCount.find(Fname); 1100 1101 // Did we have a previously-recorded size? If yes, then set FnCountBefore 1102 // to that. 1103 if (It != FunctionToInstrCount.end()) 1104 FnCountBefore = It->second; 1105 1106 // Compute the delta and emit a remark if there was a change. 1107 int64_t FnDelta = static_cast<int64_t>(FnCountAfter) - 1108 static_cast<int64_t>(FnCountBefore); 1109 if (FnDelta == 0) 1110 continue; 1111 1112 MachineOptimizationRemarkEmitter MORE(*MF, nullptr); 1113 MORE.emit([&]() { 1114 MachineOptimizationRemarkAnalysis R("size-info", "FunctionMISizeChange", 1115 DiagnosticLocation(), &MF->front()); 1116 R << DiagnosticInfoOptimizationBase::Argument("Pass", "Machine Outliner") 1117 << ": Function: " 1118 << DiagnosticInfoOptimizationBase::Argument("Function", F.getName()) 1119 << ": MI instruction count changed from " 1120 << DiagnosticInfoOptimizationBase::Argument("MIInstrsBefore", 1121 FnCountBefore) 1122 << " to " 1123 << DiagnosticInfoOptimizationBase::Argument("MIInstrsAfter", 1124 FnCountAfter) 1125 << "; Delta: " 1126 << DiagnosticInfoOptimizationBase::Argument("Delta", FnDelta); 1127 return R; 1128 }); 1129 } 1130 } 1131 1132 bool MachineOutliner::runOnModule(Module &M) { 1133 // Check if there's anything in the module. If it's empty, then there's 1134 // nothing to outline. 1135 if (M.empty()) 1136 return false; 1137 1138 // Number to append to the current outlined function. 1139 unsigned OutlinedFunctionNum = 0; 1140 1141 OutlineRepeatedNum = 0; 1142 if (!doOutline(M, OutlinedFunctionNum)) 1143 return false; 1144 1145 for (unsigned I = 0; I < OutlinerReruns; ++I) { 1146 OutlinedFunctionNum = 0; 1147 OutlineRepeatedNum++; 1148 if (!doOutline(M, OutlinedFunctionNum)) { 1149 LLVM_DEBUG({ 1150 dbgs() << "Did not outline on iteration " << I + 2 << " out of " 1151 << OutlinerReruns + 1 << "\n"; 1152 }); 1153 break; 1154 } 1155 } 1156 1157 return true; 1158 } 1159 1160 bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) { 1161 MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); 1162 1163 // If the user passed -enable-machine-outliner=always or 1164 // -enable-machine-outliner, the pass will run on all functions in the module. 1165 // Otherwise, if the target supports default outlining, it will run on all 1166 // functions deemed by the target to be worth outlining from by default. Tell 1167 // the user how the outliner is running. 1168 LLVM_DEBUG({ 1169 dbgs() << "Machine Outliner: Running on "; 1170 if (RunOnAllFunctions) 1171 dbgs() << "all functions"; 1172 else 1173 dbgs() << "target-default functions"; 1174 dbgs() << "\n"; 1175 }); 1176 1177 // If the user specifies that they want to outline from linkonceodrs, set 1178 // it here. 1179 OutlineFromLinkOnceODRs = EnableLinkOnceODROutlining; 1180 InstructionMapper Mapper; 1181 1182 // Prepare instruction mappings for the suffix tree. 1183 populateMapper(Mapper, M, MMI); 1184 std::vector<OutlinedFunction> FunctionList; 1185 1186 // Find all of the outlining candidates. 1187 findCandidates(Mapper, FunctionList); 1188 1189 // If we've requested size remarks, then collect the MI counts of every 1190 // function before outlining, and the MI counts after outlining. 1191 // FIXME: This shouldn't be in the outliner at all; it should ultimately be 1192 // the pass manager's responsibility. 1193 // This could pretty easily be placed in outline instead, but because we 1194 // really ultimately *don't* want this here, it's done like this for now 1195 // instead. 1196 1197 // Check if we want size remarks. 1198 bool ShouldEmitSizeRemarks = M.shouldEmitInstrCountChangedRemark(); 1199 StringMap<unsigned> FunctionToInstrCount; 1200 if (ShouldEmitSizeRemarks) 1201 initSizeRemarkInfo(M, MMI, FunctionToInstrCount); 1202 1203 // Outline each of the candidates and return true if something was outlined. 1204 bool OutlinedSomething = 1205 outline(M, FunctionList, Mapper, OutlinedFunctionNum); 1206 1207 // If we outlined something, we definitely changed the MI count of the 1208 // module. If we've asked for size remarks, then output them. 1209 // FIXME: This should be in the pass manager. 1210 if (ShouldEmitSizeRemarks && OutlinedSomething) 1211 emitInstrCountChangedRemark(M, MMI, FunctionToInstrCount); 1212 1213 LLVM_DEBUG({ 1214 if (!OutlinedSomething) 1215 dbgs() << "Stopped outlining at iteration " << OutlineRepeatedNum 1216 << " because no changes were found.\n"; 1217 }); 1218 1219 return OutlinedSomething; 1220 } 1221