1 //===- TargetPassConfig.cpp - Target independent code generation passes ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines interfaces to access the target independent code 10 // generation passes provided by the LLVM backend. 11 // 12 //===---------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/TargetPassConfig.h" 15 #include "llvm/ADT/DenseMap.h" 16 #include "llvm/ADT/SmallVector.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/Analysis/BasicAliasAnalysis.h" 19 #include "llvm/Analysis/CallGraphSCCPass.h" 20 #include "llvm/Analysis/ScopedNoAliasAA.h" 21 #include "llvm/Analysis/TargetTransformInfo.h" 22 #include "llvm/Analysis/TypeBasedAliasAnalysis.h" 23 #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" 24 #include "llvm/CodeGen/CSEConfigBase.h" 25 #include "llvm/CodeGen/MachineFunctionPass.h" 26 #include "llvm/CodeGen/MachinePassRegistry.h" 27 #include "llvm/CodeGen/Passes.h" 28 #include "llvm/CodeGen/RegAllocRegistry.h" 29 #include "llvm/IR/IRPrintingPasses.h" 30 #include "llvm/IR/LegacyPassManager.h" 31 #include "llvm/IR/PassInstrumentation.h" 32 #include "llvm/IR/Verifier.h" 33 #include "llvm/InitializePasses.h" 34 #include "llvm/MC/MCAsmInfo.h" 35 #include "llvm/MC/MCTargetOptions.h" 36 #include "llvm/Pass.h" 37 #include "llvm/Support/CodeGen.h" 38 #include "llvm/Support/CommandLine.h" 39 #include "llvm/Support/Compiler.h" 40 #include "llvm/Support/Debug.h" 41 #include "llvm/Support/Discriminator.h" 42 #include "llvm/Support/ErrorHandling.h" 43 #include "llvm/Support/SaveAndRestore.h" 44 #include "llvm/Support/Threading.h" 45 #include "llvm/Support/VirtualFileSystem.h" 46 #include "llvm/Support/WithColor.h" 47 #include "llvm/Target/CGPassBuilderOption.h" 48 #include "llvm/Target/TargetMachine.h" 49 #include "llvm/Transforms/ObjCARC.h" 50 #include "llvm/Transforms/Scalar.h" 51 #include "llvm/Transforms/Utils.h" 52 #include <cassert> 53 #include <optional> 54 #include <string> 55 56 using namespace llvm; 57 58 static cl::opt<bool> 59 EnableIPRA("enable-ipra", cl::init(false), cl::Hidden, 60 cl::desc("Enable interprocedural register allocation " 61 "to reduce load/store at procedure calls.")); 62 static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden, 63 cl::desc("Disable Post Regalloc Scheduler")); 64 static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden, 65 cl::desc("Disable branch folding")); 66 static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden, 67 cl::desc("Disable tail duplication")); 68 static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden, 69 cl::desc("Disable pre-register allocation tail duplication")); 70 static cl::opt<bool> DisableBlockPlacement("disable-block-placement", 71 cl::Hidden, cl::desc("Disable probability-driven block placement")); 72 static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats", 73 cl::Hidden, cl::desc("Collect probability-driven block placement stats")); 74 static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden, 75 cl::desc("Disable Stack Slot Coloring")); 76 static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden, 77 cl::desc("Disable Machine Dead Code Elimination")); 78 static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden, 79 cl::desc("Disable Early If-conversion")); 80 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden, 81 cl::desc("Disable Machine LICM")); 82 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden, 83 cl::desc("Disable Machine Common Subexpression Elimination")); 84 static cl::opt<cl::boolOrDefault> OptimizeRegAlloc( 85 "optimize-regalloc", cl::Hidden, 86 cl::desc("Enable optimized register allocation compilation path.")); 87 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm", 88 cl::Hidden, 89 cl::desc("Disable Machine LICM")); 90 static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden, 91 cl::desc("Disable Machine Sinking")); 92 static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink", 93 cl::Hidden, 94 cl::desc("Disable PostRA Machine Sinking")); 95 static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden, 96 cl::desc("Disable Loop Strength Reduction Pass")); 97 static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting", 98 cl::Hidden, cl::desc("Disable ConstantHoisting")); 99 static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden, 100 cl::desc("Disable Codegen Prepare")); 101 static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden, 102 cl::desc("Disable Copy Propagation pass")); 103 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining", 104 cl::Hidden, cl::desc("Disable Partial Libcall Inlining")); 105 static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering( 106 "disable-atexit-based-global-dtor-lowering", cl::Hidden, 107 cl::desc("For MachO, disable atexit()-based global destructor lowering")); 108 static cl::opt<bool> EnableImplicitNullChecks( 109 "enable-implicit-null-checks", 110 cl::desc("Fold null checks into faulting memory operations"), 111 cl::init(false), cl::Hidden); 112 static cl::opt<bool> DisableMergeICmps("disable-mergeicmps", 113 cl::desc("Disable MergeICmps Pass"), 114 cl::init(false), cl::Hidden); 115 static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden, 116 cl::desc("Print LLVM IR produced by the loop-reduce pass")); 117 static cl::opt<bool> 118 PrintISelInput("print-isel-input", cl::Hidden, 119 cl::desc("Print LLVM IR input to isel pass")); 120 static cl::opt<cl::boolOrDefault> 121 VerifyMachineCode("verify-machineinstrs", cl::Hidden, 122 cl::desc("Verify generated machine code")); 123 static cl::opt<cl::boolOrDefault> 124 DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden, 125 cl::desc("Debugify MIR before and Strip debug after " 126 "each pass except those known to be unsafe " 127 "when debug info is present")); 128 static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll( 129 "debugify-check-and-strip-all-safe", cl::Hidden, 130 cl::desc( 131 "Debugify MIR before, by checking and stripping the debug info after, " 132 "each pass except those known to be unsafe when debug info is " 133 "present")); 134 // Enable or disable the MachineOutliner. 135 static cl::opt<RunOutliner> EnableMachineOutliner( 136 "enable-machine-outliner", cl::desc("Enable the machine outliner"), 137 cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault), 138 cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always", 139 "Run on all functions guaranteed to be beneficial"), 140 clEnumValN(RunOutliner::NeverOutline, "never", 141 "Disable all outlining"), 142 // Sentinel value for unspecified option. 143 clEnumValN(RunOutliner::AlwaysOutline, "", ""))); 144 static cl::opt<bool> EnableGlobalMergeFunc( 145 "enable-global-merge-func", cl::Hidden, 146 cl::desc("Enable global merge functions that are based on hash function")); 147 // Disable the pass to fix unwind information. Whether the pass is included in 148 // the pipeline is controlled via the target options, this option serves as 149 // manual override. 150 static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden, 151 cl::desc("Disable the CFI fixup pass")); 152 // Enable or disable FastISel. Both options are needed, because 153 // FastISel is enabled by default with -fast, and we wish to be 154 // able to enable or disable fast-isel independently from -O0. 155 static cl::opt<cl::boolOrDefault> 156 EnableFastISelOption("fast-isel", cl::Hidden, 157 cl::desc("Enable the \"fast\" instruction selector")); 158 159 static cl::opt<cl::boolOrDefault> EnableGlobalISelOption( 160 "global-isel", cl::Hidden, 161 cl::desc("Enable the \"global\" instruction selector")); 162 163 // FIXME: remove this after switching to NPM or GlobalISel, whichever gets there 164 // first... 165 static cl::opt<bool> 166 PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden, 167 cl::desc("Print machine instrs after ISel")); 168 169 static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort( 170 "global-isel-abort", cl::Hidden, 171 cl::desc("Enable abort calls when \"global\" instruction selection " 172 "fails to lower/select an instruction"), 173 cl::values( 174 clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"), 175 clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"), 176 clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2", 177 "Disable the abort but emit a diagnostic on failure"))); 178 179 // Disable MIRProfileLoader before RegAlloc. This is for for debugging and 180 // tuning purpose. 181 static cl::opt<bool> DisableRAFSProfileLoader( 182 "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden, 183 cl::desc("Disable MIRProfileLoader before RegAlloc")); 184 // Disable MIRProfileLoader before BloackPlacement. This is for for debugging 185 // and tuning purpose. 186 static cl::opt<bool> DisableLayoutFSProfileLoader( 187 "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden, 188 cl::desc("Disable MIRProfileLoader before BlockPlacement")); 189 // Specify FSProfile file name. 190 static cl::opt<std::string> 191 FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"), 192 cl::desc("Flow Sensitive profile file name."), cl::Hidden); 193 // Specify Remapping file for FSProfile. 194 static cl::opt<std::string> FSRemappingFile( 195 "fs-remapping-file", cl::init(""), cl::value_desc("filename"), 196 cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden); 197 198 // Temporary option to allow experimenting with MachineScheduler as a post-RA 199 // scheduler. Targets can "properly" enable this with 200 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID). 201 // Targets can return true in targetSchedulesPostRAScheduling() and 202 // insert a PostRA scheduling pass wherever it wants. 203 static cl::opt<bool> MISchedPostRA( 204 "misched-postra", cl::Hidden, 205 cl::desc( 206 "Run MachineScheduler post regalloc (independent of preRA sched)")); 207 208 // Experimental option to run live interval analysis early. 209 static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden, 210 cl::desc("Run live interval analysis earlier in the pipeline")); 211 212 static cl::opt<bool> DisableReplaceWithVecLib( 213 "disable-replace-with-vec-lib", cl::Hidden, 214 cl::desc("Disable replace with vector math call pass")); 215 216 /// Option names for limiting the codegen pipeline. 217 /// Those are used in error reporting and we didn't want 218 /// to duplicate their names all over the place. 219 static const char StartAfterOptName[] = "start-after"; 220 static const char StartBeforeOptName[] = "start-before"; 221 static const char StopAfterOptName[] = "stop-after"; 222 static const char StopBeforeOptName[] = "stop-before"; 223 224 static cl::opt<std::string> 225 StartAfterOpt(StringRef(StartAfterOptName), 226 cl::desc("Resume compilation after a specific pass"), 227 cl::value_desc("pass-name"), cl::init(""), cl::Hidden); 228 229 static cl::opt<std::string> 230 StartBeforeOpt(StringRef(StartBeforeOptName), 231 cl::desc("Resume compilation before a specific pass"), 232 cl::value_desc("pass-name"), cl::init(""), cl::Hidden); 233 234 static cl::opt<std::string> 235 StopAfterOpt(StringRef(StopAfterOptName), 236 cl::desc("Stop compilation after a specific pass"), 237 cl::value_desc("pass-name"), cl::init(""), cl::Hidden); 238 239 static cl::opt<std::string> 240 StopBeforeOpt(StringRef(StopBeforeOptName), 241 cl::desc("Stop compilation before a specific pass"), 242 cl::value_desc("pass-name"), cl::init(""), cl::Hidden); 243 244 /// Enable the machine function splitter pass. 245 static cl::opt<bool> EnableMachineFunctionSplitter( 246 "enable-split-machine-functions", cl::Hidden, 247 cl::desc("Split out cold blocks from machine functions based on profile " 248 "information.")); 249 250 /// Disable the expand reductions pass for testing. 251 static cl::opt<bool> DisableExpandReductions( 252 "disable-expand-reductions", cl::init(false), cl::Hidden, 253 cl::desc("Disable the expand reduction intrinsics pass from running")); 254 255 /// Disable the select optimization pass. 256 static cl::opt<bool> DisableSelectOptimize( 257 "disable-select-optimize", cl::init(true), cl::Hidden, 258 cl::desc("Disable the select-optimization pass from running")); 259 260 /// Enable garbage-collecting empty basic blocks. 261 static cl::opt<bool> 262 GCEmptyBlocks("gc-empty-basic-blocks", cl::init(false), cl::Hidden, 263 cl::desc("Enable garbage-collecting empty basic blocks")); 264 265 /// Allow standard passes to be disabled by command line options. This supports 266 /// simple binary flags that either suppress the pass or do nothing. 267 /// i.e. -disable-mypass=false has no effect. 268 /// These should be converted to boolOrDefault in order to use applyOverride. 269 static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID, 270 bool Override) { 271 if (Override) 272 return IdentifyingPassPtr(); 273 return PassID; 274 } 275 276 /// Allow standard passes to be disabled by the command line, regardless of who 277 /// is adding the pass. 278 /// 279 /// StandardID is the pass identified in the standard pass pipeline and provided 280 /// to addPass(). It may be a target-specific ID in the case that the target 281 /// directly adds its own pass, but in that case we harmlessly fall through. 282 /// 283 /// TargetID is the pass that the target has configured to override StandardID. 284 /// 285 /// StandardID may be a pseudo ID. In that case TargetID is the name of the real 286 /// pass to run. This allows multiple options to control a single pass depending 287 /// on where in the pipeline that pass is added. 288 static IdentifyingPassPtr overridePass(AnalysisID StandardID, 289 IdentifyingPassPtr TargetID) { 290 if (StandardID == &PostRASchedulerID) 291 return applyDisable(TargetID, DisablePostRASched); 292 293 if (StandardID == &BranchFolderPassID) 294 return applyDisable(TargetID, DisableBranchFold); 295 296 if (StandardID == &TailDuplicateLegacyID) 297 return applyDisable(TargetID, DisableTailDuplicate); 298 299 if (StandardID == &EarlyTailDuplicateLegacyID) 300 return applyDisable(TargetID, DisableEarlyTailDup); 301 302 if (StandardID == &MachineBlockPlacementID) 303 return applyDisable(TargetID, DisableBlockPlacement); 304 305 if (StandardID == &StackSlotColoringID) 306 return applyDisable(TargetID, DisableSSC); 307 308 if (StandardID == &DeadMachineInstructionElimID) 309 return applyDisable(TargetID, DisableMachineDCE); 310 311 if (StandardID == &EarlyIfConverterLegacyID) 312 return applyDisable(TargetID, DisableEarlyIfConversion); 313 314 if (StandardID == &EarlyMachineLICMID) 315 return applyDisable(TargetID, DisableMachineLICM); 316 317 if (StandardID == &MachineCSELegacyID) 318 return applyDisable(TargetID, DisableMachineCSE); 319 320 if (StandardID == &MachineLICMID) 321 return applyDisable(TargetID, DisablePostRAMachineLICM); 322 323 if (StandardID == &MachineSinkingID) 324 return applyDisable(TargetID, DisableMachineSink); 325 326 if (StandardID == &PostRAMachineSinkingID) 327 return applyDisable(TargetID, DisablePostRAMachineSink); 328 329 if (StandardID == &MachineCopyPropagationID) 330 return applyDisable(TargetID, DisableCopyProp); 331 332 return TargetID; 333 } 334 335 // Find the FSProfile file name. The internal option takes the precedence 336 // before getting from TargetMachine. 337 static std::string getFSProfileFile(const TargetMachine *TM) { 338 if (!FSProfileFile.empty()) 339 return FSProfileFile.getValue(); 340 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); 341 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) 342 return std::string(); 343 return PGOOpt->ProfileFile; 344 } 345 346 // Find the Profile remapping file name. The internal option takes the 347 // precedence before getting from TargetMachine. 348 static std::string getFSRemappingFile(const TargetMachine *TM) { 349 if (!FSRemappingFile.empty()) 350 return FSRemappingFile.getValue(); 351 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); 352 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) 353 return std::string(); 354 return PGOOpt->ProfileRemappingFile; 355 } 356 357 //===---------------------------------------------------------------------===// 358 /// TargetPassConfig 359 //===---------------------------------------------------------------------===// 360 361 INITIALIZE_PASS(TargetPassConfig, "targetpassconfig", 362 "Target Pass Configuration", false, false) 363 char TargetPassConfig::ID = 0; 364 365 namespace { 366 367 struct InsertedPass { 368 AnalysisID TargetPassID; 369 IdentifyingPassPtr InsertedPassID; 370 371 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID) 372 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {} 373 374 Pass *getInsertedPass() const { 375 assert(InsertedPassID.isValid() && "Illegal Pass ID!"); 376 if (InsertedPassID.isInstance()) 377 return InsertedPassID.getInstance(); 378 Pass *NP = Pass::createPass(InsertedPassID.getID()); 379 assert(NP && "Pass ID not registered"); 380 return NP; 381 } 382 }; 383 384 } // end anonymous namespace 385 386 namespace llvm { 387 388 extern cl::opt<bool> EnableFSDiscriminator; 389 390 class PassConfigImpl { 391 public: 392 // List of passes explicitly substituted by this target. Normally this is 393 // empty, but it is a convenient way to suppress or replace specific passes 394 // that are part of a standard pass pipeline without overridding the entire 395 // pipeline. This mechanism allows target options to inherit a standard pass's 396 // user interface. For example, a target may disable a standard pass by 397 // default by substituting a pass ID of zero, and the user may still enable 398 // that standard pass with an explicit command line option. 399 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses; 400 401 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass 402 /// is inserted after each instance of the first one. 403 SmallVector<InsertedPass, 4> InsertedPasses; 404 }; 405 406 } // end namespace llvm 407 408 // Out of line virtual method. 409 TargetPassConfig::~TargetPassConfig() { 410 delete Impl; 411 } 412 413 static const PassInfo *getPassInfo(StringRef PassName) { 414 if (PassName.empty()) 415 return nullptr; 416 417 const PassRegistry &PR = *PassRegistry::getPassRegistry(); 418 const PassInfo *PI = PR.getPassInfo(PassName); 419 if (!PI) 420 report_fatal_error(Twine('\"') + Twine(PassName) + 421 Twine("\" pass is not registered.")); 422 return PI; 423 } 424 425 static AnalysisID getPassIDFromName(StringRef PassName) { 426 const PassInfo *PI = getPassInfo(PassName); 427 return PI ? PI->getTypeInfo() : nullptr; 428 } 429 430 static std::pair<StringRef, unsigned> 431 getPassNameAndInstanceNum(StringRef PassName) { 432 StringRef Name, InstanceNumStr; 433 std::tie(Name, InstanceNumStr) = PassName.split(','); 434 435 unsigned InstanceNum = 0; 436 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum)) 437 report_fatal_error("invalid pass instance specifier " + PassName); 438 439 return std::make_pair(Name, InstanceNum); 440 } 441 442 void TargetPassConfig::setStartStopPasses() { 443 StringRef StartBeforeName; 444 std::tie(StartBeforeName, StartBeforeInstanceNum) = 445 getPassNameAndInstanceNum(StartBeforeOpt); 446 447 StringRef StartAfterName; 448 std::tie(StartAfterName, StartAfterInstanceNum) = 449 getPassNameAndInstanceNum(StartAfterOpt); 450 451 StringRef StopBeforeName; 452 std::tie(StopBeforeName, StopBeforeInstanceNum) 453 = getPassNameAndInstanceNum(StopBeforeOpt); 454 455 StringRef StopAfterName; 456 std::tie(StopAfterName, StopAfterInstanceNum) 457 = getPassNameAndInstanceNum(StopAfterOpt); 458 459 StartBefore = getPassIDFromName(StartBeforeName); 460 StartAfter = getPassIDFromName(StartAfterName); 461 StopBefore = getPassIDFromName(StopBeforeName); 462 StopAfter = getPassIDFromName(StopAfterName); 463 if (StartBefore && StartAfter) 464 report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") + 465 Twine(StartAfterOptName) + Twine(" specified!")); 466 if (StopBefore && StopAfter) 467 report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") + 468 Twine(StopAfterOptName) + Twine(" specified!")); 469 Started = (StartAfter == nullptr) && (StartBefore == nullptr); 470 } 471 472 CGPassBuilderOption llvm::getCGPassBuilderOption() { 473 CGPassBuilderOption Opt; 474 475 #define SET_OPTION(Option) \ 476 if (Option.getNumOccurrences()) \ 477 Opt.Option = Option; 478 479 SET_OPTION(EnableFastISelOption) 480 SET_OPTION(EnableGlobalISelAbort) 481 SET_OPTION(EnableGlobalISelOption) 482 SET_OPTION(EnableIPRA) 483 SET_OPTION(OptimizeRegAlloc) 484 SET_OPTION(VerifyMachineCode) 485 SET_OPTION(DisableAtExitBasedGlobalDtorLowering) 486 SET_OPTION(DisableExpandReductions) 487 SET_OPTION(PrintAfterISel) 488 SET_OPTION(FSProfileFile) 489 SET_OPTION(GCEmptyBlocks) 490 491 #define SET_BOOLEAN_OPTION(Option) Opt.Option = Option; 492 493 SET_BOOLEAN_OPTION(EarlyLiveIntervals) 494 SET_BOOLEAN_OPTION(EnableBlockPlacementStats) 495 SET_BOOLEAN_OPTION(EnableGlobalMergeFunc) 496 SET_BOOLEAN_OPTION(EnableImplicitNullChecks) 497 SET_BOOLEAN_OPTION(EnableMachineOutliner) 498 SET_BOOLEAN_OPTION(MISchedPostRA) 499 SET_BOOLEAN_OPTION(DisableMergeICmps) 500 SET_BOOLEAN_OPTION(DisableLSR) 501 SET_BOOLEAN_OPTION(DisableConstantHoisting) 502 SET_BOOLEAN_OPTION(DisableCGP) 503 SET_BOOLEAN_OPTION(DisablePartialLibcallInlining) 504 SET_BOOLEAN_OPTION(DisableSelectOptimize) 505 SET_BOOLEAN_OPTION(PrintLSR) 506 SET_BOOLEAN_OPTION(PrintISelInput) 507 SET_BOOLEAN_OPTION(DebugifyAndStripAll) 508 SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll) 509 SET_BOOLEAN_OPTION(DisableRAFSProfileLoader) 510 SET_BOOLEAN_OPTION(DisableCFIFixup) 511 SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter) 512 513 return Opt; 514 } 515 516 void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC, 517 LLVMTargetMachine &LLVMTM) { 518 519 // Register a callback for disabling passes. 520 PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) { 521 522 #define DISABLE_PASS(Option, Name) \ 523 if (Option && P.contains(#Name)) \ 524 return false; 525 DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass) 526 DISABLE_PASS(DisableBranchFold, BranchFolderPass) 527 DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass) 528 DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterLegacyPass) 529 DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass) 530 DISABLE_PASS(DisableMachineCSE, MachineCSELegacyPass) 531 DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass) 532 DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass) 533 DISABLE_PASS(DisableMachineSink, MachineSinkingPass) 534 DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass) 535 DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass) 536 DISABLE_PASS(DisablePostRASched, PostRASchedulerPass) 537 DISABLE_PASS(DisableSSC, StackSlotColoringPass) 538 DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass) 539 540 return true; 541 }); 542 } 543 544 Expected<TargetPassConfig::StartStopInfo> 545 TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) { 546 auto [StartBefore, StartBeforeInstanceNum] = 547 getPassNameAndInstanceNum(StartBeforeOpt); 548 auto [StartAfter, StartAfterInstanceNum] = 549 getPassNameAndInstanceNum(StartAfterOpt); 550 auto [StopBefore, StopBeforeInstanceNum] = 551 getPassNameAndInstanceNum(StopBeforeOpt); 552 auto [StopAfter, StopAfterInstanceNum] = 553 getPassNameAndInstanceNum(StopAfterOpt); 554 555 if (!StartBefore.empty() && !StartAfter.empty()) 556 return make_error<StringError>( 557 Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!", 558 std::make_error_code(std::errc::invalid_argument)); 559 if (!StopBefore.empty() && !StopAfter.empty()) 560 return make_error<StringError>( 561 Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!", 562 std::make_error_code(std::errc::invalid_argument)); 563 564 StartStopInfo Result; 565 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore; 566 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore; 567 Result.StartInstanceNum = 568 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum; 569 Result.StopInstanceNum = 570 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum; 571 Result.StartAfter = !StartAfter.empty(); 572 Result.StopAfter = !StopAfter.empty(); 573 Result.StartInstanceNum += Result.StartInstanceNum == 0; 574 Result.StopInstanceNum += Result.StopInstanceNum == 0; 575 return Result; 576 } 577 578 // Out of line constructor provides default values for pass options and 579 // registers all common codegen passes. 580 TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm) 581 : ImmutablePass(ID), PM(&pm), TM(&TM) { 582 Impl = new PassConfigImpl(); 583 584 // Register all target independent codegen passes to activate their PassIDs, 585 // including this pass itself. 586 initializeCodeGen(*PassRegistry::getPassRegistry()); 587 588 // Also register alias analysis passes required by codegen passes. 589 initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry()); 590 initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry()); 591 592 if (EnableIPRA.getNumOccurrences()) 593 TM.Options.EnableIPRA = EnableIPRA; 594 else { 595 // If not explicitly specified, use target default. 596 TM.Options.EnableIPRA |= TM.useIPRA(); 597 } 598 599 if (TM.Options.EnableIPRA) 600 setRequiresCodeGenSCCOrder(); 601 602 if (EnableGlobalISelAbort.getNumOccurrences()) 603 TM.Options.GlobalISelAbort = EnableGlobalISelAbort; 604 605 setStartStopPasses(); 606 } 607 608 CodeGenOptLevel TargetPassConfig::getOptLevel() const { 609 return TM->getOptLevel(); 610 } 611 612 /// Insert InsertedPassID pass after TargetPassID. 613 void TargetPassConfig::insertPass(AnalysisID TargetPassID, 614 IdentifyingPassPtr InsertedPassID) { 615 assert(((!InsertedPassID.isInstance() && 616 TargetPassID != InsertedPassID.getID()) || 617 (InsertedPassID.isInstance() && 618 TargetPassID != InsertedPassID.getInstance()->getPassID())) && 619 "Insert a pass after itself!"); 620 Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID); 621 } 622 623 /// createPassConfig - Create a pass configuration object to be used by 624 /// addPassToEmitX methods for generating a pipeline of CodeGen passes. 625 /// 626 /// Targets may override this to extend TargetPassConfig. 627 TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) { 628 return new TargetPassConfig(*this, PM); 629 } 630 631 TargetPassConfig::TargetPassConfig() 632 : ImmutablePass(ID) { 633 report_fatal_error("Trying to construct TargetPassConfig without a target " 634 "machine. Scheduling a CodeGen pass without a target " 635 "triple set?"); 636 } 637 638 bool TargetPassConfig::willCompleteCodeGenPipeline() { 639 return StopBeforeOpt.empty() && StopAfterOpt.empty(); 640 } 641 642 bool TargetPassConfig::hasLimitedCodeGenPipeline() { 643 return !StartBeforeOpt.empty() || !StartAfterOpt.empty() || 644 !willCompleteCodeGenPipeline(); 645 } 646 647 std::string TargetPassConfig::getLimitedCodeGenPipelineReason() { 648 if (!hasLimitedCodeGenPipeline()) 649 return std::string(); 650 std::string Res; 651 static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt, 652 &StopAfterOpt, &StopBeforeOpt}; 653 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName, 654 StopAfterOptName, StopBeforeOptName}; 655 bool IsFirst = true; 656 for (int Idx = 0; Idx < 4; ++Idx) 657 if (!PassNames[Idx]->empty()) { 658 if (!IsFirst) 659 Res += " and "; 660 IsFirst = false; 661 Res += OptNames[Idx]; 662 } 663 return Res; 664 } 665 666 // Helper to verify the analysis is really immutable. 667 void TargetPassConfig::setOpt(bool &Opt, bool Val) { 668 assert(!Initialized && "PassConfig is immutable"); 669 Opt = Val; 670 } 671 672 void TargetPassConfig::substitutePass(AnalysisID StandardID, 673 IdentifyingPassPtr TargetID) { 674 Impl->TargetPasses[StandardID] = TargetID; 675 } 676 677 IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const { 678 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator 679 I = Impl->TargetPasses.find(ID); 680 if (I == Impl->TargetPasses.end()) 681 return ID; 682 return I->second; 683 } 684 685 bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const { 686 IdentifyingPassPtr TargetID = getPassSubstitution(ID); 687 IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID); 688 return !FinalPtr.isValid() || FinalPtr.isInstance() || 689 FinalPtr.getID() != ID; 690 } 691 692 /// Add a pass to the PassManager if that pass is supposed to be run. If the 693 /// Started/Stopped flags indicate either that the compilation should start at 694 /// a later pass or that it should stop after an earlier pass, then do not add 695 /// the pass. Finally, compare the current pass against the StartAfter 696 /// and StopAfter options and change the Started/Stopped flags accordingly. 697 void TargetPassConfig::addPass(Pass *P) { 698 assert(!Initialized && "PassConfig is immutable"); 699 700 // Cache the Pass ID here in case the pass manager finds this pass is 701 // redundant with ones already scheduled / available, and deletes it. 702 // Fundamentally, once we add the pass to the manager, we no longer own it 703 // and shouldn't reference it. 704 AnalysisID PassID = P->getPassID(); 705 706 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum) 707 Started = true; 708 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum) 709 Stopped = true; 710 if (Started && !Stopped) { 711 if (AddingMachinePasses) { 712 // Construct banner message before PM->add() as that may delete the pass. 713 std::string Banner = 714 std::string("After ") + std::string(P->getPassName()); 715 addMachinePrePasses(); 716 PM->add(P); 717 addMachinePostPasses(Banner); 718 } else { 719 PM->add(P); 720 } 721 722 // Add the passes after the pass P if there is any. 723 for (const auto &IP : Impl->InsertedPasses) 724 if (IP.TargetPassID == PassID) 725 addPass(IP.getInsertedPass()); 726 } else { 727 delete P; 728 } 729 730 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum) 731 Stopped = true; 732 733 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum) 734 Started = true; 735 if (Stopped && !Started) 736 report_fatal_error("Cannot stop compilation after pass that is not run"); 737 } 738 739 /// Add a CodeGen pass at this point in the pipeline after checking for target 740 /// and command line overrides. 741 /// 742 /// addPass cannot return a pointer to the pass instance because is internal the 743 /// PassManager and the instance we create here may already be freed. 744 AnalysisID TargetPassConfig::addPass(AnalysisID PassID) { 745 IdentifyingPassPtr TargetID = getPassSubstitution(PassID); 746 IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID); 747 if (!FinalPtr.isValid()) 748 return nullptr; 749 750 Pass *P; 751 if (FinalPtr.isInstance()) 752 P = FinalPtr.getInstance(); 753 else { 754 P = Pass::createPass(FinalPtr.getID()); 755 if (!P) 756 llvm_unreachable("Pass ID not registered"); 757 } 758 AnalysisID FinalID = P->getPassID(); 759 addPass(P); // Ends the lifetime of P. 760 761 return FinalID; 762 } 763 764 void TargetPassConfig::printAndVerify(const std::string &Banner) { 765 addPrintPass(Banner); 766 addVerifyPass(Banner); 767 } 768 769 void TargetPassConfig::addPrintPass(const std::string &Banner) { 770 if (PrintAfterISel) 771 PM->add(createMachineFunctionPrinterPass(dbgs(), Banner)); 772 } 773 774 void TargetPassConfig::addVerifyPass(const std::string &Banner) { 775 bool Verify = VerifyMachineCode == cl::BOU_TRUE; 776 #ifdef EXPENSIVE_CHECKS 777 if (VerifyMachineCode == cl::BOU_UNSET) 778 Verify = TM->isMachineVerifierClean(); 779 #endif 780 if (Verify) 781 PM->add(createMachineVerifierPass(Banner)); 782 } 783 784 void TargetPassConfig::addDebugifyPass() { 785 PM->add(createDebugifyMachineModulePass()); 786 } 787 788 void TargetPassConfig::addStripDebugPass() { 789 PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true)); 790 } 791 792 void TargetPassConfig::addCheckDebugPass() { 793 PM->add(createCheckDebugMachineModulePass()); 794 } 795 796 void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) { 797 if (AllowDebugify && DebugifyIsSafe && 798 (DebugifyAndStripAll == cl::BOU_TRUE || 799 DebugifyCheckAndStripAll == cl::BOU_TRUE)) 800 addDebugifyPass(); 801 } 802 803 void TargetPassConfig::addMachinePostPasses(const std::string &Banner) { 804 if (DebugifyIsSafe) { 805 if (DebugifyCheckAndStripAll == cl::BOU_TRUE) { 806 addCheckDebugPass(); 807 addStripDebugPass(); 808 } else if (DebugifyAndStripAll == cl::BOU_TRUE) 809 addStripDebugPass(); 810 } 811 addVerifyPass(Banner); 812 } 813 814 /// Add common target configurable passes that perform LLVM IR to IR transforms 815 /// following machine independent optimization. 816 void TargetPassConfig::addIRPasses() { 817 // Before running any passes, run the verifier to determine if the input 818 // coming from the front-end and/or optimizer is valid. 819 if (!DisableVerify) 820 addPass(createVerifierPass()); 821 822 if (getOptLevel() != CodeGenOptLevel::None) { 823 // Basic AliasAnalysis support. 824 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that 825 // BasicAliasAnalysis wins if they disagree. This is intended to help 826 // support "obvious" type-punning idioms. 827 addPass(createTypeBasedAAWrapperPass()); 828 addPass(createScopedNoAliasAAWrapperPass()); 829 addPass(createBasicAAWrapperPass()); 830 831 // Run loop strength reduction before anything else. 832 if (!DisableLSR) { 833 addPass(createCanonicalizeFreezeInLoopsPass()); 834 addPass(createLoopStrengthReducePass()); 835 if (EnableLoopTermFold) 836 addPass(createLoopTermFoldPass()); 837 if (PrintLSR) 838 addPass(createPrintFunctionPass(dbgs(), 839 "\n\n*** Code after LSR ***\n")); 840 } 841 842 // The MergeICmpsPass tries to create memcmp calls by grouping sequences of 843 // loads and compares. ExpandMemCmpPass then tries to expand those calls 844 // into optimally-sized loads and compares. The transforms are enabled by a 845 // target lowering hook. 846 if (!DisableMergeICmps) 847 addPass(createMergeICmpsLegacyPass()); 848 addPass(createExpandMemCmpLegacyPass()); 849 } 850 851 // Run GC lowering passes for builtin collectors 852 // TODO: add a pass insertion point here 853 addPass(&GCLoweringID); 854 addPass(&ShadowStackGCLoweringID); 855 856 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with 857 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. 858 if (TM->getTargetTriple().isOSBinFormatMachO() && 859 !DisableAtExitBasedGlobalDtorLowering) 860 addPass(createLowerGlobalDtorsLegacyPass()); 861 862 // Make sure that no unreachable blocks are instruction selected. 863 addPass(createUnreachableBlockEliminationPass()); 864 865 // Prepare expensive constants for SelectionDAG. 866 if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting) 867 addPass(createConstantHoistingPass()); 868 869 if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib) 870 addPass(createReplaceWithVeclibLegacyPass()); 871 872 if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining) 873 addPass(createPartiallyInlineLibCallsPass()); 874 875 // Instrument function entry after all inlining. 876 addPass(createPostInlineEntryExitInstrumenterPass()); 877 878 // Add scalarization of target's unsupported masked memory intrinsics pass. 879 // the unsupported intrinsic will be replaced with a chain of basic blocks, 880 // that stores/loads element one-by-one if the appropriate mask bit is set. 881 addPass(createScalarizeMaskedMemIntrinLegacyPass()); 882 883 // Expand reduction intrinsics into shuffle sequences if the target wants to. 884 // Allow disabling it for testing purposes. 885 if (!DisableExpandReductions) 886 addPass(createExpandReductionsPass()); 887 888 // Convert conditional moves to conditional jumps when profitable. 889 if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize) 890 addPass(createSelectOptimizePass()); 891 892 if (EnableGlobalMergeFunc) 893 addPass(createGlobalMergeFuncPass()); 894 } 895 896 /// Turn exception handling constructs into something the code generators can 897 /// handle. 898 void TargetPassConfig::addPassesToHandleExceptions() { 899 const MCAsmInfo *MCAI = TM->getMCAsmInfo(); 900 assert(MCAI && "No MCAsmInfo"); 901 switch (MCAI->getExceptionHandlingType()) { 902 case ExceptionHandling::SjLj: 903 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both 904 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, 905 // catch info can get misplaced when a selector ends up more than one block 906 // removed from the parent invoke(s). This could happen when a landing 907 // pad is shared by multiple invokes and is also a target of a normal 908 // edge from elsewhere. 909 addPass(createSjLjEHPreparePass(TM)); 910 [[fallthrough]]; 911 case ExceptionHandling::DwarfCFI: 912 case ExceptionHandling::ARM: 913 case ExceptionHandling::AIX: 914 case ExceptionHandling::ZOS: 915 addPass(createDwarfEHPass(getOptLevel())); 916 break; 917 case ExceptionHandling::WinEH: 918 // We support using both GCC-style and MSVC-style exceptions on Windows, so 919 // add both preparation passes. Each pass will only actually run if it 920 // recognizes the personality function. 921 addPass(createWinEHPass()); 922 addPass(createDwarfEHPass(getOptLevel())); 923 break; 924 case ExceptionHandling::Wasm: 925 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs 926 // on catchpads and cleanuppads because it does not outline them into 927 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we 928 // should remove PHIs there. 929 addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true)); 930 addPass(createWasmEHPass()); 931 break; 932 case ExceptionHandling::None: 933 addPass(createLowerInvokePass()); 934 935 // The lower invoke pass may create unreachable code. Remove it. 936 addPass(createUnreachableBlockEliminationPass()); 937 break; 938 } 939 } 940 941 /// Add pass to prepare the LLVM IR for code generation. This should be done 942 /// before exception handling preparation passes. 943 void TargetPassConfig::addCodeGenPrepare() { 944 if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP) 945 addPass(createCodeGenPrepareLegacyPass()); 946 } 947 948 /// Add common passes that perform LLVM IR to IR transforms in preparation for 949 /// instruction selection. 950 void TargetPassConfig::addISelPrepare() { 951 addPreISel(); 952 953 // Force codegen to run according to the callgraph. 954 if (requiresCodeGenSCCOrder()) 955 addPass(new DummyCGSCCPass); 956 957 if (getOptLevel() != CodeGenOptLevel::None) 958 addPass(createObjCARCContractPass()); 959 960 addPass(createCallBrPass()); 961 962 // Add both the safe stack and the stack protection passes: each of them will 963 // only protect functions that have corresponding attributes. 964 addPass(createSafeStackPass()); 965 addPass(createStackProtectorPass()); 966 967 if (PrintISelInput) 968 addPass(createPrintFunctionPass( 969 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n")); 970 971 // All passes which modify the LLVM IR are now complete; run the verifier 972 // to ensure that the IR is valid. 973 if (!DisableVerify) 974 addPass(createVerifierPass()); 975 } 976 977 bool TargetPassConfig::addCoreISelPasses() { 978 // Enable FastISel with -fast-isel, but allow that to be overridden. 979 TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE); 980 981 // Determine an instruction selector. 982 enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; 983 SelectorType Selector; 984 985 if (EnableFastISelOption == cl::BOU_TRUE) 986 Selector = SelectorType::FastISel; 987 else if (EnableGlobalISelOption == cl::BOU_TRUE || 988 (TM->Options.EnableGlobalISel && 989 EnableGlobalISelOption != cl::BOU_FALSE)) 990 Selector = SelectorType::GlobalISel; 991 else if (TM->getOptLevel() == CodeGenOptLevel::None && 992 TM->getO0WantsFastISel()) 993 Selector = SelectorType::FastISel; 994 else 995 Selector = SelectorType::SelectionDAG; 996 997 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel. 998 if (Selector == SelectorType::FastISel) { 999 TM->setFastISel(true); 1000 TM->setGlobalISel(false); 1001 } else if (Selector == SelectorType::GlobalISel) { 1002 TM->setFastISel(false); 1003 TM->setGlobalISel(true); 1004 } 1005 1006 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with 1007 // analyses needing to be re-run. This can result in being unable to 1008 // schedule passes (particularly with 'Function Alias Analysis 1009 // Results'). It's not entirely clear why but AFAICT this seems to be 1010 // due to one FunctionPassManager not being able to use analyses from a 1011 // previous one. As we're injecting a ModulePass we break the usual 1012 // pass manager into two. GlobalISel with the fallback path disabled 1013 // and -run-pass seem to be unaffected. The majority of GlobalISel 1014 // testing uses -run-pass so this probably isn't too bad. 1015 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe); 1016 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled()) 1017 DebugifyIsSafe = false; 1018 1019 // Add instruction selector passes. 1020 if (Selector == SelectorType::GlobalISel) { 1021 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true); 1022 if (addIRTranslator()) 1023 return true; 1024 1025 addPreLegalizeMachineIR(); 1026 1027 if (addLegalizeMachineIR()) 1028 return true; 1029 1030 // Before running the register bank selector, ask the target if it 1031 // wants to run some passes. 1032 addPreRegBankSelect(); 1033 1034 if (addRegBankSelect()) 1035 return true; 1036 1037 addPreGlobalInstructionSelect(); 1038 1039 if (addGlobalInstructionSelect()) 1040 return true; 1041 1042 // Pass to reset the MachineFunction if the ISel failed. 1043 addPass(createResetMachineFunctionPass( 1044 reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled())); 1045 1046 // Provide a fallback path when we do not want to abort on 1047 // not-yet-supported input. 1048 if (!isGlobalISelAbortEnabled() && addInstSelector()) 1049 return true; 1050 1051 } else if (addInstSelector()) 1052 return true; 1053 1054 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before 1055 // FinalizeISel. 1056 addPass(&FinalizeISelID); 1057 1058 // Print the instruction selected machine code... 1059 printAndVerify("After Instruction Selection"); 1060 1061 return false; 1062 } 1063 1064 bool TargetPassConfig::addISelPasses() { 1065 if (TM->useEmulatedTLS()) 1066 addPass(createLowerEmuTLSPass()); 1067 1068 PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); 1069 addPass(createPreISelIntrinsicLoweringPass()); 1070 addPass(createExpandLargeDivRemPass()); 1071 addPass(createExpandLargeFpConvertPass()); 1072 addIRPasses(); 1073 addCodeGenPrepare(); 1074 addPassesToHandleExceptions(); 1075 addISelPrepare(); 1076 1077 return addCoreISelPasses(); 1078 } 1079 1080 /// -regalloc=... command line option. 1081 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } 1082 static cl::opt<RegisterRegAlloc::FunctionPassCtor, false, 1083 RegisterPassParser<RegisterRegAlloc>> 1084 RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator), 1085 cl::desc("Register allocator to use")); 1086 1087 /// Add the complete set of target-independent postISel code generator passes. 1088 /// 1089 /// This can be read as the standard order of major LLVM CodeGen stages. Stages 1090 /// with nontrivial configuration or multiple passes are broken out below in 1091 /// add%Stage routines. 1092 /// 1093 /// Any TargetPassConfig::addXX routine may be overriden by the Target. The 1094 /// addPre/Post methods with empty header implementations allow injecting 1095 /// target-specific fixups just before or after major stages. Additionally, 1096 /// targets have the flexibility to change pass order within a stage by 1097 /// overriding default implementation of add%Stage routines below. Each 1098 /// technique has maintainability tradeoffs because alternate pass orders are 1099 /// not well supported. addPre/Post works better if the target pass is easily 1100 /// tied to a common pass. But if it has subtle dependencies on multiple passes, 1101 /// the target should override the stage instead. 1102 /// 1103 /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection 1104 /// before/after any target-independent pass. But it's currently overkill. 1105 void TargetPassConfig::addMachinePasses() { 1106 AddingMachinePasses = true; 1107 1108 // Add passes that optimize machine instructions in SSA form. 1109 if (getOptLevel() != CodeGenOptLevel::None) { 1110 addMachineSSAOptimization(); 1111 } else { 1112 // If the target requests it, assign local variables to stack slots relative 1113 // to one another and simplify frame index references where possible. 1114 addPass(&LocalStackSlotAllocationID); 1115 } 1116 1117 if (TM->Options.EnableIPRA) 1118 addPass(createRegUsageInfoPropPass()); 1119 1120 // Run pre-ra passes. 1121 addPreRegAlloc(); 1122 1123 // Debugifying the register allocator passes seems to provoke some 1124 // non-determinism that affects CodeGen and there doesn't seem to be a point 1125 // where it becomes safe again so stop debugifying here. 1126 DebugifyIsSafe = false; 1127 1128 // Add a FSDiscriminator pass right before RA, so that we could get 1129 // more precise SampleFDO profile for RA. 1130 if (EnableFSDiscriminator) { 1131 addPass(createMIRAddFSDiscriminatorsPass( 1132 sampleprof::FSDiscriminatorPass::Pass1)); 1133 const std::string ProfileFile = getFSProfileFile(TM); 1134 if (!ProfileFile.empty() && !DisableRAFSProfileLoader) 1135 addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM), 1136 sampleprof::FSDiscriminatorPass::Pass1, 1137 nullptr)); 1138 } 1139 1140 // Run register allocation and passes that are tightly coupled with it, 1141 // including phi elimination and scheduling. 1142 if (getOptimizeRegAlloc()) 1143 addOptimizedRegAlloc(); 1144 else 1145 addFastRegAlloc(); 1146 1147 // Run post-ra passes. 1148 addPostRegAlloc(); 1149 1150 addPass(&RemoveRedundantDebugValuesID); 1151 1152 addPass(&FixupStatepointCallerSavedID); 1153 1154 // Insert prolog/epilog code. Eliminate abstract frame index references... 1155 if (getOptLevel() != CodeGenOptLevel::None) { 1156 addPass(&PostRAMachineSinkingID); 1157 addPass(&ShrinkWrapID); 1158 } 1159 1160 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only 1161 // do so if it hasn't been disabled, substituted, or overridden. 1162 if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID)) 1163 addPass(createPrologEpilogInserterPass()); 1164 1165 /// Add passes that optimize machine instructions after register allocation. 1166 if (getOptLevel() != CodeGenOptLevel::None) 1167 addMachineLateOptimization(); 1168 1169 // Expand pseudo instructions before second scheduling pass. 1170 addPass(&ExpandPostRAPseudosID); 1171 1172 // Run pre-sched2 passes. 1173 addPreSched2(); 1174 1175 if (EnableImplicitNullChecks) 1176 addPass(&ImplicitNullChecksID); 1177 1178 // Second pass scheduler. 1179 // Let Target optionally insert this pass by itself at some other 1180 // point. 1181 if (getOptLevel() != CodeGenOptLevel::None && 1182 !TM->targetSchedulesPostRAScheduling()) { 1183 if (MISchedPostRA) 1184 addPass(&PostMachineSchedulerID); 1185 else 1186 addPass(&PostRASchedulerID); 1187 } 1188 1189 // GC 1190 addGCPasses(); 1191 1192 // Basic block placement. 1193 if (getOptLevel() != CodeGenOptLevel::None) 1194 addBlockPlacement(); 1195 1196 // Insert before XRay Instrumentation. 1197 addPass(&FEntryInserterID); 1198 1199 addPass(&XRayInstrumentationID); 1200 addPass(&PatchableFunctionID); 1201 1202 addPreEmitPass(); 1203 1204 if (TM->Options.EnableIPRA) 1205 // Collect register usage information and produce a register mask of 1206 // clobbered registers, to be used to optimize call sites. 1207 addPass(createRegUsageInfoCollector()); 1208 1209 // FIXME: Some backends are incompatible with running the verifier after 1210 // addPreEmitPass. Maybe only pass "false" here for those targets? 1211 addPass(&FuncletLayoutID); 1212 1213 addPass(&RemoveLoadsIntoFakeUsesID); 1214 addPass(&StackMapLivenessID); 1215 addPass(&LiveDebugValuesID); 1216 addPass(&MachineSanitizerBinaryMetadataID); 1217 1218 if (TM->Options.EnableMachineOutliner && 1219 getOptLevel() != CodeGenOptLevel::None && 1220 EnableMachineOutliner != RunOutliner::NeverOutline) { 1221 bool RunOnAllFunctions = 1222 (EnableMachineOutliner == RunOutliner::AlwaysOutline); 1223 bool AddOutliner = 1224 RunOnAllFunctions || TM->Options.SupportsDefaultOutlining; 1225 if (AddOutliner) 1226 addPass(createMachineOutlinerPass(RunOnAllFunctions)); 1227 } 1228 1229 if (GCEmptyBlocks) 1230 addPass(llvm::createGCEmptyBasicBlocksPass()); 1231 1232 if (EnableFSDiscriminator) 1233 addPass(createMIRAddFSDiscriminatorsPass( 1234 sampleprof::FSDiscriminatorPass::PassLast)); 1235 1236 bool NeedsBBSections = 1237 TM->getBBSectionsType() != llvm::BasicBlockSection::None; 1238 // Machine function splitter uses the basic block sections feature. Both 1239 // cannot be enabled at the same time. We do not apply machine function 1240 // splitter if -basic-block-sections is requested. 1241 if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter || 1242 EnableMachineFunctionSplitter)) { 1243 const std::string ProfileFile = getFSProfileFile(TM); 1244 if (!ProfileFile.empty()) { 1245 if (EnableFSDiscriminator) { 1246 addPass(createMIRProfileLoaderPass( 1247 ProfileFile, getFSRemappingFile(TM), 1248 sampleprof::FSDiscriminatorPass::PassLast, nullptr)); 1249 } else { 1250 // Sample profile is given, but FSDiscriminator is not 1251 // enabled, this may result in performance regression. 1252 WithColor::warning() 1253 << "Using AutoFDO without FSDiscriminator for MFS may regress " 1254 "performance.\n"; 1255 } 1256 } 1257 addPass(createMachineFunctionSplitterPass()); 1258 } 1259 // We run the BasicBlockSections pass if either we need BB sections or BB 1260 // address map (or both). 1261 if (NeedsBBSections || TM->Options.BBAddrMap) { 1262 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { 1263 addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass( 1264 TM->getBBSectionsFuncListBuf())); 1265 addPass(llvm::createBasicBlockPathCloningPass()); 1266 } 1267 addPass(llvm::createBasicBlockSectionsPass()); 1268 } 1269 1270 addPostBBSections(); 1271 1272 if (!DisableCFIFixup && TM->Options.EnableCFIFixup) 1273 addPass(createCFIFixup()); 1274 1275 PM->add(createStackFrameLayoutAnalysisPass()); 1276 1277 // Add passes that directly emit MI after all other MI passes. 1278 addPreEmitPass2(); 1279 1280 AddingMachinePasses = false; 1281 } 1282 1283 /// Add passes that optimize machine instructions in SSA form. 1284 void TargetPassConfig::addMachineSSAOptimization() { 1285 // Pre-ra tail duplication. 1286 addPass(&EarlyTailDuplicateLegacyID); 1287 1288 // Optimize PHIs before DCE: removing dead PHI cycles may make more 1289 // instructions dead. 1290 addPass(&OptimizePHIsLegacyID); 1291 1292 // This pass merges large allocas. StackSlotColoring is a different pass 1293 // which merges spill slots. 1294 addPass(&StackColoringLegacyID); 1295 1296 // If the target requests it, assign local variables to stack slots relative 1297 // to one another and simplify frame index references where possible. 1298 addPass(&LocalStackSlotAllocationID); 1299 1300 // With optimization, dead code should already be eliminated. However 1301 // there is one known exception: lowered code for arguments that are only 1302 // used by tail calls, where the tail calls reuse the incoming stack 1303 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). 1304 addPass(&DeadMachineInstructionElimID); 1305 1306 // Allow targets to insert passes that improve instruction level parallelism, 1307 // like if-conversion. Such passes will typically need dominator trees and 1308 // loop info, just like LICM and CSE below. 1309 addILPOpts(); 1310 1311 addPass(&EarlyMachineLICMID); 1312 addPass(&MachineCSELegacyID); 1313 1314 addPass(&MachineSinkingID); 1315 1316 addPass(&PeepholeOptimizerID); 1317 // Clean-up the dead code that may have been generated by peephole 1318 // rewriting. 1319 addPass(&DeadMachineInstructionElimID); 1320 } 1321 1322 //===---------------------------------------------------------------------===// 1323 /// Register Allocation Pass Configuration 1324 //===---------------------------------------------------------------------===// 1325 1326 bool TargetPassConfig::getOptimizeRegAlloc() const { 1327 switch (OptimizeRegAlloc) { 1328 case cl::BOU_UNSET: 1329 return getOptLevel() != CodeGenOptLevel::None; 1330 case cl::BOU_TRUE: return true; 1331 case cl::BOU_FALSE: return false; 1332 } 1333 llvm_unreachable("Invalid optimize-regalloc state"); 1334 } 1335 1336 /// A dummy default pass factory indicates whether the register allocator is 1337 /// overridden on the command line. 1338 static llvm::once_flag InitializeDefaultRegisterAllocatorFlag; 1339 1340 static RegisterRegAlloc 1341 defaultRegAlloc("default", 1342 "pick register allocator based on -O option", 1343 useDefaultRegisterAllocator); 1344 1345 static void initializeDefaultRegisterAllocatorOnce() { 1346 if (!RegisterRegAlloc::getDefault()) 1347 RegisterRegAlloc::setDefault(RegAlloc); 1348 } 1349 1350 /// Instantiate the default register allocator pass for this target for either 1351 /// the optimized or unoptimized allocation path. This will be added to the pass 1352 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc 1353 /// in the optimized case. 1354 /// 1355 /// A target that uses the standard regalloc pass order for fast or optimized 1356 /// allocation may still override this for per-target regalloc 1357 /// selection. But -regalloc=... always takes precedence. 1358 FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) { 1359 if (Optimized) 1360 return createGreedyRegisterAllocator(); 1361 else 1362 return createFastRegisterAllocator(); 1363 } 1364 1365 /// Find and instantiate the register allocation pass requested by this target 1366 /// at the current optimization level. Different register allocators are 1367 /// defined as separate passes because they may require different analysis. 1368 /// 1369 /// This helper ensures that the regalloc= option is always available, 1370 /// even for targets that override the default allocator. 1371 /// 1372 /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs, 1373 /// this can be folded into addPass. 1374 FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) { 1375 // Initialize the global default. 1376 llvm::call_once(InitializeDefaultRegisterAllocatorFlag, 1377 initializeDefaultRegisterAllocatorOnce); 1378 1379 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault(); 1380 if (Ctor != useDefaultRegisterAllocator) 1381 return Ctor(); 1382 1383 // With no -regalloc= override, ask the target for a regalloc pass. 1384 return createTargetRegisterAllocator(Optimized); 1385 } 1386 1387 bool TargetPassConfig::isCustomizedRegAlloc() { 1388 return RegAlloc != 1389 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator; 1390 } 1391 1392 bool TargetPassConfig::addRegAssignAndRewriteFast() { 1393 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator && 1394 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator) 1395 report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc."); 1396 1397 addPass(createRegAllocPass(false)); 1398 1399 // Allow targets to change the register assignments after 1400 // fast register allocation. 1401 addPostFastRegAllocRewrite(); 1402 return true; 1403 } 1404 1405 bool TargetPassConfig::addRegAssignAndRewriteOptimized() { 1406 // Add the selected register allocation pass. 1407 addPass(createRegAllocPass(true)); 1408 1409 // Allow targets to change the register assignments before rewriting. 1410 addPreRewrite(); 1411 1412 // Finally rewrite virtual registers. 1413 addPass(&VirtRegRewriterID); 1414 1415 // Regalloc scoring for ML-driven eviction - noop except when learning a new 1416 // eviction policy. 1417 addPass(createRegAllocScoringPass()); 1418 return true; 1419 } 1420 1421 /// Return true if the default global register allocator is in use and 1422 /// has not be overriden on the command line with '-regalloc=...' 1423 bool TargetPassConfig::usingDefaultRegAlloc() const { 1424 return RegAlloc.getNumOccurrences() == 0; 1425 } 1426 1427 /// Add the minimum set of target-independent passes that are required for 1428 /// register allocation. No coalescing or scheduling. 1429 void TargetPassConfig::addFastRegAlloc() { 1430 addPass(&PHIEliminationID); 1431 addPass(&TwoAddressInstructionPassID); 1432 1433 addRegAssignAndRewriteFast(); 1434 } 1435 1436 /// Add standard target-independent passes that are tightly coupled with 1437 /// optimized register allocation, including coalescing, machine instruction 1438 /// scheduling, and register allocation itself. 1439 void TargetPassConfig::addOptimizedRegAlloc() { 1440 addPass(&DetectDeadLanesID); 1441 1442 addPass(&InitUndefID); 1443 1444 addPass(&ProcessImplicitDefsID); 1445 1446 // LiveVariables currently requires pure SSA form. 1447 // 1448 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags, 1449 // LiveVariables can be removed completely, and LiveIntervals can be directly 1450 // computed. (We still either need to regenerate kill flags after regalloc, or 1451 // preferably fix the scavenger to not depend on them). 1452 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables. 1453 // When LiveVariables is removed this has to be removed/moved either. 1454 // Explicit addition of UnreachableMachineBlockElim allows stopping before or 1455 // after it with -stop-before/-stop-after. 1456 addPass(&UnreachableMachineBlockElimID); 1457 addPass(&LiveVariablesID); 1458 1459 // Edge splitting is smarter with machine loop info. 1460 addPass(&MachineLoopInfoID); 1461 addPass(&PHIEliminationID); 1462 1463 // Eventually, we want to run LiveIntervals before PHI elimination. 1464 if (EarlyLiveIntervals) 1465 addPass(&LiveIntervalsID); 1466 1467 addPass(&TwoAddressInstructionPassID); 1468 addPass(&RegisterCoalescerID); 1469 1470 // The machine scheduler may accidentally create disconnected components 1471 // when moving subregister definitions around, avoid this by splitting them to 1472 // separate vregs before. Splitting can also improve reg. allocation quality. 1473 addPass(&RenameIndependentSubregsID); 1474 1475 // PreRA instruction scheduling. 1476 addPass(&MachineSchedulerID); 1477 1478 if (addRegAssignAndRewriteOptimized()) { 1479 // Perform stack slot coloring and post-ra machine LICM. 1480 addPass(&StackSlotColoringID); 1481 1482 // Allow targets to expand pseudo instructions depending on the choice of 1483 // registers before MachineCopyPropagation. 1484 addPostRewrite(); 1485 1486 // Copy propagate to forward register uses and try to eliminate COPYs that 1487 // were not coalesced. 1488 addPass(&MachineCopyPropagationID); 1489 1490 // Run post-ra machine LICM to hoist reloads / remats. 1491 // 1492 // FIXME: can this move into MachineLateOptimization? 1493 addPass(&MachineLICMID); 1494 } 1495 } 1496 1497 //===---------------------------------------------------------------------===// 1498 /// Post RegAlloc Pass Configuration 1499 //===---------------------------------------------------------------------===// 1500 1501 /// Add passes that optimize machine instructions after register allocation. 1502 void TargetPassConfig::addMachineLateOptimization() { 1503 // Cleanup of redundant immediate/address loads. 1504 addPass(&MachineLateInstrsCleanupID); 1505 1506 // Branch folding must be run after regalloc and prolog/epilog insertion. 1507 addPass(&BranchFolderPassID); 1508 1509 // Tail duplication. 1510 // Note that duplicating tail just increases code size and degrades 1511 // performance for targets that require Structured Control Flow. 1512 // In addition it can also make CFG irreducible. Thus we disable it. 1513 if (!TM->requiresStructuredCFG()) 1514 addPass(&TailDuplicateLegacyID); 1515 1516 // Copy propagation. 1517 addPass(&MachineCopyPropagationID); 1518 } 1519 1520 /// Add standard GC passes. 1521 bool TargetPassConfig::addGCPasses() { 1522 addPass(&GCMachineCodeAnalysisID); 1523 return true; 1524 } 1525 1526 /// Add standard basic block placement passes. 1527 void TargetPassConfig::addBlockPlacement() { 1528 if (EnableFSDiscriminator) { 1529 addPass(createMIRAddFSDiscriminatorsPass( 1530 sampleprof::FSDiscriminatorPass::Pass2)); 1531 const std::string ProfileFile = getFSProfileFile(TM); 1532 if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader) 1533 addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM), 1534 sampleprof::FSDiscriminatorPass::Pass2, 1535 nullptr)); 1536 } 1537 if (addPass(&MachineBlockPlacementID)) { 1538 // Run a separate pass to collect block placement statistics. 1539 if (EnableBlockPlacementStats) 1540 addPass(&MachineBlockPlacementStatsID); 1541 } 1542 } 1543 1544 //===---------------------------------------------------------------------===// 1545 /// GlobalISel Configuration 1546 //===---------------------------------------------------------------------===// 1547 bool TargetPassConfig::isGlobalISelAbortEnabled() const { 1548 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable; 1549 } 1550 1551 bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const { 1552 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; 1553 } 1554 1555 bool TargetPassConfig::isGISelCSEEnabled() const { 1556 return true; 1557 } 1558 1559 std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const { 1560 return std::make_unique<CSEConfigBase>(); 1561 } 1562