xref: /llvm-project/llvm/lib/CodeGen/TargetPassConfig.cpp (revision d23c5c2d6566fce4380cfa31d438422db19fbce9)
1 //===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines interfaces to access the target independent code
10 // generation passes provided by the LLVM backend.
11 //
12 //===---------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/TargetPassConfig.h"
15 #include "llvm/ADT/DenseMap.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/Analysis/BasicAliasAnalysis.h"
19 #include "llvm/Analysis/CallGraphSCCPass.h"
20 #include "llvm/Analysis/ScopedNoAliasAA.h"
21 #include "llvm/Analysis/TargetTransformInfo.h"
22 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
23 #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
24 #include "llvm/CodeGen/CSEConfigBase.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachinePassRegistry.h"
27 #include "llvm/CodeGen/Passes.h"
28 #include "llvm/CodeGen/RegAllocRegistry.h"
29 #include "llvm/IR/IRPrintingPasses.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/IR/PassInstrumentation.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/MC/MCAsmInfo.h"
35 #include "llvm/MC/MCTargetOptions.h"
36 #include "llvm/Pass.h"
37 #include "llvm/Support/CodeGen.h"
38 #include "llvm/Support/CommandLine.h"
39 #include "llvm/Support/Compiler.h"
40 #include "llvm/Support/Debug.h"
41 #include "llvm/Support/Discriminator.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/SaveAndRestore.h"
44 #include "llvm/Support/Threading.h"
45 #include "llvm/Support/VirtualFileSystem.h"
46 #include "llvm/Support/WithColor.h"
47 #include "llvm/Target/CGPassBuilderOption.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/ObjCARC.h"
50 #include "llvm/Transforms/Scalar.h"
51 #include "llvm/Transforms/Utils.h"
52 #include <cassert>
53 #include <optional>
54 #include <string>
55 
56 using namespace llvm;
57 
58 static cl::opt<bool>
59     EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
60                cl::desc("Enable interprocedural register allocation "
61                         "to reduce load/store at procedure calls."));
62 static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
63     cl::desc("Disable Post Regalloc Scheduler"));
64 static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
65     cl::desc("Disable branch folding"));
66 static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
67     cl::desc("Disable tail duplication"));
68 static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
69     cl::desc("Disable pre-register allocation tail duplication"));
70 static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
71     cl::Hidden, cl::desc("Disable probability-driven block placement"));
72 static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
73     cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
74 static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
75     cl::desc("Disable Stack Slot Coloring"));
76 static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
77     cl::desc("Disable Machine Dead Code Elimination"));
78 static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
79     cl::desc("Disable Early If-conversion"));
80 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
81     cl::desc("Disable Machine LICM"));
82 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
83     cl::desc("Disable Machine Common Subexpression Elimination"));
84 static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
85     "optimize-regalloc", cl::Hidden,
86     cl::desc("Enable optimized register allocation compilation path."));
87 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
88     cl::Hidden,
89     cl::desc("Disable Machine LICM"));
90 static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
91     cl::desc("Disable Machine Sinking"));
92 static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
93     cl::Hidden,
94     cl::desc("Disable PostRA Machine Sinking"));
95 static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
96     cl::desc("Disable Loop Strength Reduction Pass"));
97 static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
98     cl::Hidden, cl::desc("Disable ConstantHoisting"));
99 static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
100     cl::desc("Disable Codegen Prepare"));
101 static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
102     cl::desc("Disable Copy Propagation pass"));
103 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
104     cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
105 static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
106     "disable-atexit-based-global-dtor-lowering", cl::Hidden,
107     cl::desc("For MachO, disable atexit()-based global destructor lowering"));
108 static cl::opt<bool> EnableImplicitNullChecks(
109     "enable-implicit-null-checks",
110     cl::desc("Fold null checks into faulting memory operations"),
111     cl::init(false), cl::Hidden);
112 static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
113     cl::desc("Disable MergeICmps Pass"),
114     cl::init(false), cl::Hidden);
115 static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
116     cl::desc("Print LLVM IR produced by the loop-reduce pass"));
117 static cl::opt<bool>
118     PrintISelInput("print-isel-input", cl::Hidden,
119                    cl::desc("Print LLVM IR input to isel pass"));
120 static cl::opt<cl::boolOrDefault>
121     VerifyMachineCode("verify-machineinstrs", cl::Hidden,
122                       cl::desc("Verify generated machine code"));
123 static cl::opt<cl::boolOrDefault>
124     DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
125                         cl::desc("Debugify MIR before and Strip debug after "
126                                  "each pass except those known to be unsafe "
127                                  "when debug info is present"));
128 static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
129     "debugify-check-and-strip-all-safe", cl::Hidden,
130     cl::desc(
131         "Debugify MIR before, by checking and stripping the debug info after, "
132         "each pass except those known to be unsafe when debug info is "
133         "present"));
134 // Enable or disable the MachineOutliner.
135 static cl::opt<RunOutliner> EnableMachineOutliner(
136     "enable-machine-outliner", cl::desc("Enable the machine outliner"),
137     cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),
138     cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always",
139                           "Run on all functions guaranteed to be beneficial"),
140                clEnumValN(RunOutliner::NeverOutline, "never",
141                           "Disable all outlining"),
142                // Sentinel value for unspecified option.
143                clEnumValN(RunOutliner::AlwaysOutline, "", "")));
144 static cl::opt<bool> EnableGlobalMergeFunc(
145     "enable-global-merge-func", cl::Hidden,
146     cl::desc("Enable global merge functions that are based on hash function"));
147 // Disable the pass to fix unwind information. Whether the pass is included in
148 // the pipeline is controlled via the target options, this option serves as
149 // manual override.
150 static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
151                                      cl::desc("Disable the CFI fixup pass"));
152 // Enable or disable FastISel. Both options are needed, because
153 // FastISel is enabled by default with -fast, and we wish to be
154 // able to enable or disable fast-isel independently from -O0.
155 static cl::opt<cl::boolOrDefault>
156 EnableFastISelOption("fast-isel", cl::Hidden,
157   cl::desc("Enable the \"fast\" instruction selector"));
158 
159 static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
160     "global-isel", cl::Hidden,
161     cl::desc("Enable the \"global\" instruction selector"));
162 
163 // FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
164 //        first...
165 static cl::opt<bool>
166     PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,
167                    cl::desc("Print machine instrs after ISel"));
168 
169 static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
170     "global-isel-abort", cl::Hidden,
171     cl::desc("Enable abort calls when \"global\" instruction selection "
172              "fails to lower/select an instruction"),
173     cl::values(
174         clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
175         clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
176         clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
177                    "Disable the abort but emit a diagnostic on failure")));
178 
179 // Disable MIRProfileLoader before RegAlloc. This is for for debugging and
180 // tuning purpose.
181 static cl::opt<bool> DisableRAFSProfileLoader(
182     "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden,
183     cl::desc("Disable MIRProfileLoader before RegAlloc"));
184 // Disable MIRProfileLoader before BloackPlacement. This is for for debugging
185 // and tuning purpose.
186 static cl::opt<bool> DisableLayoutFSProfileLoader(
187     "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
188     cl::desc("Disable MIRProfileLoader before BlockPlacement"));
189 // Specify FSProfile file name.
190 static cl::opt<std::string>
191     FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
192                   cl::desc("Flow Sensitive profile file name."), cl::Hidden);
193 // Specify Remapping file for FSProfile.
194 static cl::opt<std::string> FSRemappingFile(
195     "fs-remapping-file", cl::init(""), cl::value_desc("filename"),
196     cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
197 
198 // Temporary option to allow experimenting with MachineScheduler as a post-RA
199 // scheduler. Targets can "properly" enable this with
200 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
201 // Targets can return true in targetSchedulesPostRAScheduling() and
202 // insert a PostRA scheduling pass wherever it wants.
203 static cl::opt<bool> MISchedPostRA(
204     "misched-postra", cl::Hidden,
205     cl::desc(
206         "Run MachineScheduler post regalloc (independent of preRA sched)"));
207 
208 // Experimental option to run live interval analysis early.
209 static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
210     cl::desc("Run live interval analysis earlier in the pipeline"));
211 
212 static cl::opt<bool> DisableReplaceWithVecLib(
213     "disable-replace-with-vec-lib", cl::Hidden,
214     cl::desc("Disable replace with vector math call pass"));
215 
216 /// Option names for limiting the codegen pipeline.
217 /// Those are used in error reporting and we didn't want
218 /// to duplicate their names all over the place.
219 static const char StartAfterOptName[] = "start-after";
220 static const char StartBeforeOptName[] = "start-before";
221 static const char StopAfterOptName[] = "stop-after";
222 static const char StopBeforeOptName[] = "stop-before";
223 
224 static cl::opt<std::string>
225     StartAfterOpt(StringRef(StartAfterOptName),
226                   cl::desc("Resume compilation after a specific pass"),
227                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
228 
229 static cl::opt<std::string>
230     StartBeforeOpt(StringRef(StartBeforeOptName),
231                    cl::desc("Resume compilation before a specific pass"),
232                    cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
233 
234 static cl::opt<std::string>
235     StopAfterOpt(StringRef(StopAfterOptName),
236                  cl::desc("Stop compilation after a specific pass"),
237                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
238 
239 static cl::opt<std::string>
240     StopBeforeOpt(StringRef(StopBeforeOptName),
241                   cl::desc("Stop compilation before a specific pass"),
242                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
243 
244 /// Enable the machine function splitter pass.
245 static cl::opt<bool> EnableMachineFunctionSplitter(
246     "enable-split-machine-functions", cl::Hidden,
247     cl::desc("Split out cold blocks from machine functions based on profile "
248              "information."));
249 
250 /// Disable the expand reductions pass for testing.
251 static cl::opt<bool> DisableExpandReductions(
252     "disable-expand-reductions", cl::init(false), cl::Hidden,
253     cl::desc("Disable the expand reduction intrinsics pass from running"));
254 
255 /// Disable the select optimization pass.
256 static cl::opt<bool> DisableSelectOptimize(
257     "disable-select-optimize", cl::init(true), cl::Hidden,
258     cl::desc("Disable the select-optimization pass from running"));
259 
260 /// Enable garbage-collecting empty basic blocks.
261 static cl::opt<bool>
262     GCEmptyBlocks("gc-empty-basic-blocks", cl::init(false), cl::Hidden,
263                   cl::desc("Enable garbage-collecting empty basic blocks"));
264 
265 /// Allow standard passes to be disabled by command line options. This supports
266 /// simple binary flags that either suppress the pass or do nothing.
267 /// i.e. -disable-mypass=false has no effect.
268 /// These should be converted to boolOrDefault in order to use applyOverride.
269 static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
270                                        bool Override) {
271   if (Override)
272     return IdentifyingPassPtr();
273   return PassID;
274 }
275 
276 /// Allow standard passes to be disabled by the command line, regardless of who
277 /// is adding the pass.
278 ///
279 /// StandardID is the pass identified in the standard pass pipeline and provided
280 /// to addPass(). It may be a target-specific ID in the case that the target
281 /// directly adds its own pass, but in that case we harmlessly fall through.
282 ///
283 /// TargetID is the pass that the target has configured to override StandardID.
284 ///
285 /// StandardID may be a pseudo ID. In that case TargetID is the name of the real
286 /// pass to run. This allows multiple options to control a single pass depending
287 /// on where in the pipeline that pass is added.
288 static IdentifyingPassPtr overridePass(AnalysisID StandardID,
289                                        IdentifyingPassPtr TargetID) {
290   if (StandardID == &PostRASchedulerID)
291     return applyDisable(TargetID, DisablePostRASched);
292 
293   if (StandardID == &BranchFolderPassID)
294     return applyDisable(TargetID, DisableBranchFold);
295 
296   if (StandardID == &TailDuplicateLegacyID)
297     return applyDisable(TargetID, DisableTailDuplicate);
298 
299   if (StandardID == &EarlyTailDuplicateLegacyID)
300     return applyDisable(TargetID, DisableEarlyTailDup);
301 
302   if (StandardID == &MachineBlockPlacementID)
303     return applyDisable(TargetID, DisableBlockPlacement);
304 
305   if (StandardID == &StackSlotColoringID)
306     return applyDisable(TargetID, DisableSSC);
307 
308   if (StandardID == &DeadMachineInstructionElimID)
309     return applyDisable(TargetID, DisableMachineDCE);
310 
311   if (StandardID == &EarlyIfConverterLegacyID)
312     return applyDisable(TargetID, DisableEarlyIfConversion);
313 
314   if (StandardID == &EarlyMachineLICMID)
315     return applyDisable(TargetID, DisableMachineLICM);
316 
317   if (StandardID == &MachineCSELegacyID)
318     return applyDisable(TargetID, DisableMachineCSE);
319 
320   if (StandardID == &MachineLICMID)
321     return applyDisable(TargetID, DisablePostRAMachineLICM);
322 
323   if (StandardID == &MachineSinkingID)
324     return applyDisable(TargetID, DisableMachineSink);
325 
326   if (StandardID == &PostRAMachineSinkingID)
327     return applyDisable(TargetID, DisablePostRAMachineSink);
328 
329   if (StandardID == &MachineCopyPropagationID)
330     return applyDisable(TargetID, DisableCopyProp);
331 
332   return TargetID;
333 }
334 
335 // Find the FSProfile file name. The internal option takes the precedence
336 // before getting from TargetMachine.
337 static std::string getFSProfileFile(const TargetMachine *TM) {
338   if (!FSProfileFile.empty())
339     return FSProfileFile.getValue();
340   const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
341   if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
342     return std::string();
343   return PGOOpt->ProfileFile;
344 }
345 
346 // Find the Profile remapping file name. The internal option takes the
347 // precedence before getting from TargetMachine.
348 static std::string getFSRemappingFile(const TargetMachine *TM) {
349   if (!FSRemappingFile.empty())
350     return FSRemappingFile.getValue();
351   const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
352   if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
353     return std::string();
354   return PGOOpt->ProfileRemappingFile;
355 }
356 
357 //===---------------------------------------------------------------------===//
358 /// TargetPassConfig
359 //===---------------------------------------------------------------------===//
360 
361 INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
362                 "Target Pass Configuration", false, false)
363 char TargetPassConfig::ID = 0;
364 
365 namespace {
366 
367 struct InsertedPass {
368   AnalysisID TargetPassID;
369   IdentifyingPassPtr InsertedPassID;
370 
371   InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
372       : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
373 
374   Pass *getInsertedPass() const {
375     assert(InsertedPassID.isValid() && "Illegal Pass ID!");
376     if (InsertedPassID.isInstance())
377       return InsertedPassID.getInstance();
378     Pass *NP = Pass::createPass(InsertedPassID.getID());
379     assert(NP && "Pass ID not registered");
380     return NP;
381   }
382 };
383 
384 } // end anonymous namespace
385 
386 namespace llvm {
387 
388 extern cl::opt<bool> EnableFSDiscriminator;
389 
390 class PassConfigImpl {
391 public:
392   // List of passes explicitly substituted by this target. Normally this is
393   // empty, but it is a convenient way to suppress or replace specific passes
394   // that are part of a standard pass pipeline without overridding the entire
395   // pipeline. This mechanism allows target options to inherit a standard pass's
396   // user interface. For example, a target may disable a standard pass by
397   // default by substituting a pass ID of zero, and the user may still enable
398   // that standard pass with an explicit command line option.
399   DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
400 
401   /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
402   /// is inserted after each instance of the first one.
403   SmallVector<InsertedPass, 4> InsertedPasses;
404 };
405 
406 } // end namespace llvm
407 
408 // Out of line virtual method.
409 TargetPassConfig::~TargetPassConfig() {
410   delete Impl;
411 }
412 
413 static const PassInfo *getPassInfo(StringRef PassName) {
414   if (PassName.empty())
415     return nullptr;
416 
417   const PassRegistry &PR = *PassRegistry::getPassRegistry();
418   const PassInfo *PI = PR.getPassInfo(PassName);
419   if (!PI)
420     report_fatal_error(Twine('\"') + Twine(PassName) +
421                        Twine("\" pass is not registered."));
422   return PI;
423 }
424 
425 static AnalysisID getPassIDFromName(StringRef PassName) {
426   const PassInfo *PI = getPassInfo(PassName);
427   return PI ? PI->getTypeInfo() : nullptr;
428 }
429 
430 static std::pair<StringRef, unsigned>
431 getPassNameAndInstanceNum(StringRef PassName) {
432   StringRef Name, InstanceNumStr;
433   std::tie(Name, InstanceNumStr) = PassName.split(',');
434 
435   unsigned InstanceNum = 0;
436   if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))
437     report_fatal_error("invalid pass instance specifier " + PassName);
438 
439   return std::make_pair(Name, InstanceNum);
440 }
441 
442 void TargetPassConfig::setStartStopPasses() {
443   StringRef StartBeforeName;
444   std::tie(StartBeforeName, StartBeforeInstanceNum) =
445     getPassNameAndInstanceNum(StartBeforeOpt);
446 
447   StringRef StartAfterName;
448   std::tie(StartAfterName, StartAfterInstanceNum) =
449     getPassNameAndInstanceNum(StartAfterOpt);
450 
451   StringRef StopBeforeName;
452   std::tie(StopBeforeName, StopBeforeInstanceNum)
453     = getPassNameAndInstanceNum(StopBeforeOpt);
454 
455   StringRef StopAfterName;
456   std::tie(StopAfterName, StopAfterInstanceNum)
457     = getPassNameAndInstanceNum(StopAfterOpt);
458 
459   StartBefore = getPassIDFromName(StartBeforeName);
460   StartAfter = getPassIDFromName(StartAfterName);
461   StopBefore = getPassIDFromName(StopBeforeName);
462   StopAfter = getPassIDFromName(StopAfterName);
463   if (StartBefore && StartAfter)
464     report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
465                        Twine(StartAfterOptName) + Twine(" specified!"));
466   if (StopBefore && StopAfter)
467     report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
468                        Twine(StopAfterOptName) + Twine(" specified!"));
469   Started = (StartAfter == nullptr) && (StartBefore == nullptr);
470 }
471 
472 CGPassBuilderOption llvm::getCGPassBuilderOption() {
473   CGPassBuilderOption Opt;
474 
475 #define SET_OPTION(Option)                                                     \
476   if (Option.getNumOccurrences())                                              \
477     Opt.Option = Option;
478 
479   SET_OPTION(EnableFastISelOption)
480   SET_OPTION(EnableGlobalISelAbort)
481   SET_OPTION(EnableGlobalISelOption)
482   SET_OPTION(EnableIPRA)
483   SET_OPTION(OptimizeRegAlloc)
484   SET_OPTION(VerifyMachineCode)
485   SET_OPTION(DisableAtExitBasedGlobalDtorLowering)
486   SET_OPTION(DisableExpandReductions)
487   SET_OPTION(PrintAfterISel)
488   SET_OPTION(FSProfileFile)
489   SET_OPTION(GCEmptyBlocks)
490 
491 #define SET_BOOLEAN_OPTION(Option) Opt.Option = Option;
492 
493   SET_BOOLEAN_OPTION(EarlyLiveIntervals)
494   SET_BOOLEAN_OPTION(EnableBlockPlacementStats)
495   SET_BOOLEAN_OPTION(EnableGlobalMergeFunc)
496   SET_BOOLEAN_OPTION(EnableImplicitNullChecks)
497   SET_BOOLEAN_OPTION(EnableMachineOutliner)
498   SET_BOOLEAN_OPTION(MISchedPostRA)
499   SET_BOOLEAN_OPTION(DisableMergeICmps)
500   SET_BOOLEAN_OPTION(DisableLSR)
501   SET_BOOLEAN_OPTION(DisableConstantHoisting)
502   SET_BOOLEAN_OPTION(DisableCGP)
503   SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
504   SET_BOOLEAN_OPTION(DisableSelectOptimize)
505   SET_BOOLEAN_OPTION(PrintLSR)
506   SET_BOOLEAN_OPTION(PrintISelInput)
507   SET_BOOLEAN_OPTION(DebugifyAndStripAll)
508   SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)
509   SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)
510   SET_BOOLEAN_OPTION(DisableCFIFixup)
511   SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)
512 
513   return Opt;
514 }
515 
516 void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
517                                    LLVMTargetMachine &LLVMTM) {
518 
519   // Register a callback for disabling passes.
520   PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) {
521 
522 #define DISABLE_PASS(Option, Name)                                             \
523   if (Option && P.contains(#Name))                                             \
524     return false;
525     DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
526     DISABLE_PASS(DisableBranchFold, BranchFolderPass)
527     DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
528     DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterLegacyPass)
529     DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
530     DISABLE_PASS(DisableMachineCSE, MachineCSELegacyPass)
531     DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
532     DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
533     DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
534     DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
535     DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
536     DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
537     DISABLE_PASS(DisableSSC, StackSlotColoringPass)
538     DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
539 
540     return true;
541   });
542 }
543 
544 Expected<TargetPassConfig::StartStopInfo>
545 TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
546   auto [StartBefore, StartBeforeInstanceNum] =
547       getPassNameAndInstanceNum(StartBeforeOpt);
548   auto [StartAfter, StartAfterInstanceNum] =
549       getPassNameAndInstanceNum(StartAfterOpt);
550   auto [StopBefore, StopBeforeInstanceNum] =
551       getPassNameAndInstanceNum(StopBeforeOpt);
552   auto [StopAfter, StopAfterInstanceNum] =
553       getPassNameAndInstanceNum(StopAfterOpt);
554 
555   if (!StartBefore.empty() && !StartAfter.empty())
556     return make_error<StringError>(
557         Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
558         std::make_error_code(std::errc::invalid_argument));
559   if (!StopBefore.empty() && !StopAfter.empty())
560     return make_error<StringError>(
561         Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
562         std::make_error_code(std::errc::invalid_argument));
563 
564   StartStopInfo Result;
565   Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
566   Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
567   Result.StartInstanceNum =
568       StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
569   Result.StopInstanceNum =
570       StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
571   Result.StartAfter = !StartAfter.empty();
572   Result.StopAfter = !StopAfter.empty();
573   Result.StartInstanceNum += Result.StartInstanceNum == 0;
574   Result.StopInstanceNum += Result.StopInstanceNum == 0;
575   return Result;
576 }
577 
578 // Out of line constructor provides default values for pass options and
579 // registers all common codegen passes.
580 TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
581     : ImmutablePass(ID), PM(&pm), TM(&TM) {
582   Impl = new PassConfigImpl();
583 
584   // Register all target independent codegen passes to activate their PassIDs,
585   // including this pass itself.
586   initializeCodeGen(*PassRegistry::getPassRegistry());
587 
588   // Also register alias analysis passes required by codegen passes.
589   initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
590   initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
591 
592   if (EnableIPRA.getNumOccurrences())
593     TM.Options.EnableIPRA = EnableIPRA;
594   else {
595     // If not explicitly specified, use target default.
596     TM.Options.EnableIPRA |= TM.useIPRA();
597   }
598 
599   if (TM.Options.EnableIPRA)
600     setRequiresCodeGenSCCOrder();
601 
602   if (EnableGlobalISelAbort.getNumOccurrences())
603     TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
604 
605   setStartStopPasses();
606 }
607 
608 CodeGenOptLevel TargetPassConfig::getOptLevel() const {
609   return TM->getOptLevel();
610 }
611 
612 /// Insert InsertedPassID pass after TargetPassID.
613 void TargetPassConfig::insertPass(AnalysisID TargetPassID,
614                                   IdentifyingPassPtr InsertedPassID) {
615   assert(((!InsertedPassID.isInstance() &&
616            TargetPassID != InsertedPassID.getID()) ||
617           (InsertedPassID.isInstance() &&
618            TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
619          "Insert a pass after itself!");
620   Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID);
621 }
622 
623 /// createPassConfig - Create a pass configuration object to be used by
624 /// addPassToEmitX methods for generating a pipeline of CodeGen passes.
625 ///
626 /// Targets may override this to extend TargetPassConfig.
627 TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
628   return new TargetPassConfig(*this, PM);
629 }
630 
631 TargetPassConfig::TargetPassConfig()
632   : ImmutablePass(ID) {
633   report_fatal_error("Trying to construct TargetPassConfig without a target "
634                      "machine. Scheduling a CodeGen pass without a target "
635                      "triple set?");
636 }
637 
638 bool TargetPassConfig::willCompleteCodeGenPipeline() {
639   return StopBeforeOpt.empty() && StopAfterOpt.empty();
640 }
641 
642 bool TargetPassConfig::hasLimitedCodeGenPipeline() {
643   return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
644          !willCompleteCodeGenPipeline();
645 }
646 
647 std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {
648   if (!hasLimitedCodeGenPipeline())
649     return std::string();
650   std::string Res;
651   static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
652                                               &StopAfterOpt, &StopBeforeOpt};
653   static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
654                                    StopAfterOptName, StopBeforeOptName};
655   bool IsFirst = true;
656   for (int Idx = 0; Idx < 4; ++Idx)
657     if (!PassNames[Idx]->empty()) {
658       if (!IsFirst)
659         Res += " and ";
660       IsFirst = false;
661       Res += OptNames[Idx];
662     }
663   return Res;
664 }
665 
666 // Helper to verify the analysis is really immutable.
667 void TargetPassConfig::setOpt(bool &Opt, bool Val) {
668   assert(!Initialized && "PassConfig is immutable");
669   Opt = Val;
670 }
671 
672 void TargetPassConfig::substitutePass(AnalysisID StandardID,
673                                       IdentifyingPassPtr TargetID) {
674   Impl->TargetPasses[StandardID] = TargetID;
675 }
676 
677 IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
678   DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
679     I = Impl->TargetPasses.find(ID);
680   if (I == Impl->TargetPasses.end())
681     return ID;
682   return I->second;
683 }
684 
685 bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
686   IdentifyingPassPtr TargetID = getPassSubstitution(ID);
687   IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
688   return !FinalPtr.isValid() || FinalPtr.isInstance() ||
689       FinalPtr.getID() != ID;
690 }
691 
692 /// Add a pass to the PassManager if that pass is supposed to be run.  If the
693 /// Started/Stopped flags indicate either that the compilation should start at
694 /// a later pass or that it should stop after an earlier pass, then do not add
695 /// the pass.  Finally, compare the current pass against the StartAfter
696 /// and StopAfter options and change the Started/Stopped flags accordingly.
697 void TargetPassConfig::addPass(Pass *P) {
698   assert(!Initialized && "PassConfig is immutable");
699 
700   // Cache the Pass ID here in case the pass manager finds this pass is
701   // redundant with ones already scheduled / available, and deletes it.
702   // Fundamentally, once we add the pass to the manager, we no longer own it
703   // and shouldn't reference it.
704   AnalysisID PassID = P->getPassID();
705 
706   if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
707     Started = true;
708   if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
709     Stopped = true;
710   if (Started && !Stopped) {
711     if (AddingMachinePasses) {
712       // Construct banner message before PM->add() as that may delete the pass.
713       std::string Banner =
714           std::string("After ") + std::string(P->getPassName());
715       addMachinePrePasses();
716       PM->add(P);
717       addMachinePostPasses(Banner);
718     } else {
719       PM->add(P);
720     }
721 
722     // Add the passes after the pass P if there is any.
723     for (const auto &IP : Impl->InsertedPasses)
724       if (IP.TargetPassID == PassID)
725         addPass(IP.getInsertedPass());
726   } else {
727     delete P;
728   }
729 
730   if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
731     Stopped = true;
732 
733   if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
734     Started = true;
735   if (Stopped && !Started)
736     report_fatal_error("Cannot stop compilation after pass that is not run");
737 }
738 
739 /// Add a CodeGen pass at this point in the pipeline after checking for target
740 /// and command line overrides.
741 ///
742 /// addPass cannot return a pointer to the pass instance because is internal the
743 /// PassManager and the instance we create here may already be freed.
744 AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
745   IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
746   IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
747   if (!FinalPtr.isValid())
748     return nullptr;
749 
750   Pass *P;
751   if (FinalPtr.isInstance())
752     P = FinalPtr.getInstance();
753   else {
754     P = Pass::createPass(FinalPtr.getID());
755     if (!P)
756       llvm_unreachable("Pass ID not registered");
757   }
758   AnalysisID FinalID = P->getPassID();
759   addPass(P); // Ends the lifetime of P.
760 
761   return FinalID;
762 }
763 
764 void TargetPassConfig::printAndVerify(const std::string &Banner) {
765   addPrintPass(Banner);
766   addVerifyPass(Banner);
767 }
768 
769 void TargetPassConfig::addPrintPass(const std::string &Banner) {
770   if (PrintAfterISel)
771     PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
772 }
773 
774 void TargetPassConfig::addVerifyPass(const std::string &Banner) {
775   bool Verify = VerifyMachineCode == cl::BOU_TRUE;
776 #ifdef EXPENSIVE_CHECKS
777   if (VerifyMachineCode == cl::BOU_UNSET)
778     Verify = TM->isMachineVerifierClean();
779 #endif
780   if (Verify)
781     PM->add(createMachineVerifierPass(Banner));
782 }
783 
784 void TargetPassConfig::addDebugifyPass() {
785   PM->add(createDebugifyMachineModulePass());
786 }
787 
788 void TargetPassConfig::addStripDebugPass() {
789   PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true));
790 }
791 
792 void TargetPassConfig::addCheckDebugPass() {
793   PM->add(createCheckDebugMachineModulePass());
794 }
795 
796 void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
797   if (AllowDebugify && DebugifyIsSafe &&
798       (DebugifyAndStripAll == cl::BOU_TRUE ||
799        DebugifyCheckAndStripAll == cl::BOU_TRUE))
800     addDebugifyPass();
801 }
802 
803 void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
804   if (DebugifyIsSafe) {
805     if (DebugifyCheckAndStripAll == cl::BOU_TRUE) {
806       addCheckDebugPass();
807       addStripDebugPass();
808     } else if (DebugifyAndStripAll == cl::BOU_TRUE)
809       addStripDebugPass();
810   }
811   addVerifyPass(Banner);
812 }
813 
814 /// Add common target configurable passes that perform LLVM IR to IR transforms
815 /// following machine independent optimization.
816 void TargetPassConfig::addIRPasses() {
817   // Before running any passes, run the verifier to determine if the input
818   // coming from the front-end and/or optimizer is valid.
819   if (!DisableVerify)
820     addPass(createVerifierPass());
821 
822   if (getOptLevel() != CodeGenOptLevel::None) {
823     // Basic AliasAnalysis support.
824     // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
825     // BasicAliasAnalysis wins if they disagree. This is intended to help
826     // support "obvious" type-punning idioms.
827     addPass(createTypeBasedAAWrapperPass());
828     addPass(createScopedNoAliasAAWrapperPass());
829     addPass(createBasicAAWrapperPass());
830 
831     // Run loop strength reduction before anything else.
832     if (!DisableLSR) {
833       addPass(createCanonicalizeFreezeInLoopsPass());
834       addPass(createLoopStrengthReducePass());
835       if (EnableLoopTermFold)
836         addPass(createLoopTermFoldPass());
837       if (PrintLSR)
838         addPass(createPrintFunctionPass(dbgs(),
839                                         "\n\n*** Code after LSR ***\n"));
840     }
841 
842     // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
843     // loads and compares. ExpandMemCmpPass then tries to expand those calls
844     // into optimally-sized loads and compares. The transforms are enabled by a
845     // target lowering hook.
846     if (!DisableMergeICmps)
847       addPass(createMergeICmpsLegacyPass());
848     addPass(createExpandMemCmpLegacyPass());
849   }
850 
851   // Run GC lowering passes for builtin collectors
852   // TODO: add a pass insertion point here
853   addPass(&GCLoweringID);
854   addPass(&ShadowStackGCLoweringID);
855 
856   // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
857   // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
858   if (TM->getTargetTriple().isOSBinFormatMachO() &&
859       !DisableAtExitBasedGlobalDtorLowering)
860     addPass(createLowerGlobalDtorsLegacyPass());
861 
862   // Make sure that no unreachable blocks are instruction selected.
863   addPass(createUnreachableBlockEliminationPass());
864 
865   // Prepare expensive constants for SelectionDAG.
866   if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)
867     addPass(createConstantHoistingPass());
868 
869   if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib)
870     addPass(createReplaceWithVeclibLegacyPass());
871 
872   if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
873     addPass(createPartiallyInlineLibCallsPass());
874 
875   // Instrument function entry after all inlining.
876   addPass(createPostInlineEntryExitInstrumenterPass());
877 
878   // Add scalarization of target's unsupported masked memory intrinsics pass.
879   // the unsupported intrinsic will be replaced with a chain of basic blocks,
880   // that stores/loads element one-by-one if the appropriate mask bit is set.
881   addPass(createScalarizeMaskedMemIntrinLegacyPass());
882 
883   // Expand reduction intrinsics into shuffle sequences if the target wants to.
884   // Allow disabling it for testing purposes.
885   if (!DisableExpandReductions)
886     addPass(createExpandReductionsPass());
887 
888   // Convert conditional moves to conditional jumps when profitable.
889   if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)
890     addPass(createSelectOptimizePass());
891 
892   if (EnableGlobalMergeFunc)
893     addPass(createGlobalMergeFuncPass());
894 }
895 
896 /// Turn exception handling constructs into something the code generators can
897 /// handle.
898 void TargetPassConfig::addPassesToHandleExceptions() {
899   const MCAsmInfo *MCAI = TM->getMCAsmInfo();
900   assert(MCAI && "No MCAsmInfo");
901   switch (MCAI->getExceptionHandlingType()) {
902   case ExceptionHandling::SjLj:
903     // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
904     // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
905     // catch info can get misplaced when a selector ends up more than one block
906     // removed from the parent invoke(s). This could happen when a landing
907     // pad is shared by multiple invokes and is also a target of a normal
908     // edge from elsewhere.
909     addPass(createSjLjEHPreparePass(TM));
910     [[fallthrough]];
911   case ExceptionHandling::DwarfCFI:
912   case ExceptionHandling::ARM:
913   case ExceptionHandling::AIX:
914   case ExceptionHandling::ZOS:
915     addPass(createDwarfEHPass(getOptLevel()));
916     break;
917   case ExceptionHandling::WinEH:
918     // We support using both GCC-style and MSVC-style exceptions on Windows, so
919     // add both preparation passes. Each pass will only actually run if it
920     // recognizes the personality function.
921     addPass(createWinEHPass());
922     addPass(createDwarfEHPass(getOptLevel()));
923     break;
924   case ExceptionHandling::Wasm:
925     // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
926     // on catchpads and cleanuppads because it does not outline them into
927     // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
928     // should remove PHIs there.
929     addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true));
930     addPass(createWasmEHPass());
931     break;
932   case ExceptionHandling::None:
933     addPass(createLowerInvokePass());
934 
935     // The lower invoke pass may create unreachable code. Remove it.
936     addPass(createUnreachableBlockEliminationPass());
937     break;
938   }
939 }
940 
941 /// Add pass to prepare the LLVM IR for code generation. This should be done
942 /// before exception handling preparation passes.
943 void TargetPassConfig::addCodeGenPrepare() {
944   if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)
945     addPass(createCodeGenPrepareLegacyPass());
946 }
947 
948 /// Add common passes that perform LLVM IR to IR transforms in preparation for
949 /// instruction selection.
950 void TargetPassConfig::addISelPrepare() {
951   addPreISel();
952 
953   // Force codegen to run according to the callgraph.
954   if (requiresCodeGenSCCOrder())
955     addPass(new DummyCGSCCPass);
956 
957   if (getOptLevel() != CodeGenOptLevel::None)
958     addPass(createObjCARCContractPass());
959 
960   addPass(createCallBrPass());
961 
962   // Add both the safe stack and the stack protection passes: each of them will
963   // only protect functions that have corresponding attributes.
964   addPass(createSafeStackPass());
965   addPass(createStackProtectorPass());
966 
967   if (PrintISelInput)
968     addPass(createPrintFunctionPass(
969         dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
970 
971   // All passes which modify the LLVM IR are now complete; run the verifier
972   // to ensure that the IR is valid.
973   if (!DisableVerify)
974     addPass(createVerifierPass());
975 }
976 
977 bool TargetPassConfig::addCoreISelPasses() {
978   // Enable FastISel with -fast-isel, but allow that to be overridden.
979   TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
980 
981   // Determine an instruction selector.
982   enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
983   SelectorType Selector;
984 
985   if (EnableFastISelOption == cl::BOU_TRUE)
986     Selector = SelectorType::FastISel;
987   else if (EnableGlobalISelOption == cl::BOU_TRUE ||
988            (TM->Options.EnableGlobalISel &&
989             EnableGlobalISelOption != cl::BOU_FALSE))
990     Selector = SelectorType::GlobalISel;
991   else if (TM->getOptLevel() == CodeGenOptLevel::None &&
992            TM->getO0WantsFastISel())
993     Selector = SelectorType::FastISel;
994   else
995     Selector = SelectorType::SelectionDAG;
996 
997   // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
998   if (Selector == SelectorType::FastISel) {
999     TM->setFastISel(true);
1000     TM->setGlobalISel(false);
1001   } else if (Selector == SelectorType::GlobalISel) {
1002     TM->setFastISel(false);
1003     TM->setGlobalISel(true);
1004   }
1005 
1006   // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
1007   //        analyses needing to be re-run. This can result in being unable to
1008   //        schedule passes (particularly with 'Function Alias Analysis
1009   //        Results'). It's not entirely clear why but AFAICT this seems to be
1010   //        due to one FunctionPassManager not being able to use analyses from a
1011   //        previous one. As we're injecting a ModulePass we break the usual
1012   //        pass manager into two. GlobalISel with the fallback path disabled
1013   //        and -run-pass seem to be unaffected. The majority of GlobalISel
1014   //        testing uses -run-pass so this probably isn't too bad.
1015   SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
1016   if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1017     DebugifyIsSafe = false;
1018 
1019   // Add instruction selector passes.
1020   if (Selector == SelectorType::GlobalISel) {
1021     SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
1022     if (addIRTranslator())
1023       return true;
1024 
1025     addPreLegalizeMachineIR();
1026 
1027     if (addLegalizeMachineIR())
1028       return true;
1029 
1030     // Before running the register bank selector, ask the target if it
1031     // wants to run some passes.
1032     addPreRegBankSelect();
1033 
1034     if (addRegBankSelect())
1035       return true;
1036 
1037     addPreGlobalInstructionSelect();
1038 
1039     if (addGlobalInstructionSelect())
1040       return true;
1041 
1042     // Pass to reset the MachineFunction if the ISel failed.
1043     addPass(createResetMachineFunctionPass(
1044         reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
1045 
1046     // Provide a fallback path when we do not want to abort on
1047     // not-yet-supported input.
1048     if (!isGlobalISelAbortEnabled() && addInstSelector())
1049       return true;
1050 
1051   } else if (addInstSelector())
1052     return true;
1053 
1054   // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
1055   // FinalizeISel.
1056   addPass(&FinalizeISelID);
1057 
1058   // Print the instruction selected machine code...
1059   printAndVerify("After Instruction Selection");
1060 
1061   return false;
1062 }
1063 
1064 bool TargetPassConfig::addISelPasses() {
1065   if (TM->useEmulatedTLS())
1066     addPass(createLowerEmuTLSPass());
1067 
1068   PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
1069   addPass(createPreISelIntrinsicLoweringPass());
1070   addPass(createExpandLargeDivRemPass());
1071   addPass(createExpandLargeFpConvertPass());
1072   addIRPasses();
1073   addCodeGenPrepare();
1074   addPassesToHandleExceptions();
1075   addISelPrepare();
1076 
1077   return addCoreISelPasses();
1078 }
1079 
1080 /// -regalloc=... command line option.
1081 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
1082 static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
1083                RegisterPassParser<RegisterRegAlloc>>
1084     RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
1085              cl::desc("Register allocator to use"));
1086 
1087 /// Add the complete set of target-independent postISel code generator passes.
1088 ///
1089 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
1090 /// with nontrivial configuration or multiple passes are broken out below in
1091 /// add%Stage routines.
1092 ///
1093 /// Any TargetPassConfig::addXX routine may be overriden by the Target. The
1094 /// addPre/Post methods with empty header implementations allow injecting
1095 /// target-specific fixups just before or after major stages. Additionally,
1096 /// targets have the flexibility to change pass order within a stage by
1097 /// overriding default implementation of add%Stage routines below. Each
1098 /// technique has maintainability tradeoffs because alternate pass orders are
1099 /// not well supported. addPre/Post works better if the target pass is easily
1100 /// tied to a common pass. But if it has subtle dependencies on multiple passes,
1101 /// the target should override the stage instead.
1102 ///
1103 /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
1104 /// before/after any target-independent pass. But it's currently overkill.
1105 void TargetPassConfig::addMachinePasses() {
1106   AddingMachinePasses = true;
1107 
1108   // Add passes that optimize machine instructions in SSA form.
1109   if (getOptLevel() != CodeGenOptLevel::None) {
1110     addMachineSSAOptimization();
1111   } else {
1112     // If the target requests it, assign local variables to stack slots relative
1113     // to one another and simplify frame index references where possible.
1114     addPass(&LocalStackSlotAllocationID);
1115   }
1116 
1117   if (TM->Options.EnableIPRA)
1118     addPass(createRegUsageInfoPropPass());
1119 
1120   // Run pre-ra passes.
1121   addPreRegAlloc();
1122 
1123   // Debugifying the register allocator passes seems to provoke some
1124   // non-determinism that affects CodeGen and there doesn't seem to be a point
1125   // where it becomes safe again so stop debugifying here.
1126   DebugifyIsSafe = false;
1127 
1128   // Add a FSDiscriminator pass right before RA, so that we could get
1129   // more precise SampleFDO profile for RA.
1130   if (EnableFSDiscriminator) {
1131     addPass(createMIRAddFSDiscriminatorsPass(
1132         sampleprof::FSDiscriminatorPass::Pass1));
1133     const std::string ProfileFile = getFSProfileFile(TM);
1134     if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
1135       addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1136                                          sampleprof::FSDiscriminatorPass::Pass1,
1137                                          nullptr));
1138   }
1139 
1140   // Run register allocation and passes that are tightly coupled with it,
1141   // including phi elimination and scheduling.
1142   if (getOptimizeRegAlloc())
1143     addOptimizedRegAlloc();
1144   else
1145     addFastRegAlloc();
1146 
1147   // Run post-ra passes.
1148   addPostRegAlloc();
1149 
1150   addPass(&RemoveRedundantDebugValuesID);
1151 
1152   addPass(&FixupStatepointCallerSavedID);
1153 
1154   // Insert prolog/epilog code.  Eliminate abstract frame index references...
1155   if (getOptLevel() != CodeGenOptLevel::None) {
1156     addPass(&PostRAMachineSinkingID);
1157     addPass(&ShrinkWrapID);
1158   }
1159 
1160   // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1161   // do so if it hasn't been disabled, substituted, or overridden.
1162   if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
1163       addPass(createPrologEpilogInserterPass());
1164 
1165   /// Add passes that optimize machine instructions after register allocation.
1166   if (getOptLevel() != CodeGenOptLevel::None)
1167       addMachineLateOptimization();
1168 
1169   // Expand pseudo instructions before second scheduling pass.
1170   addPass(&ExpandPostRAPseudosID);
1171 
1172   // Run pre-sched2 passes.
1173   addPreSched2();
1174 
1175   if (EnableImplicitNullChecks)
1176     addPass(&ImplicitNullChecksID);
1177 
1178   // Second pass scheduler.
1179   // Let Target optionally insert this pass by itself at some other
1180   // point.
1181   if (getOptLevel() != CodeGenOptLevel::None &&
1182       !TM->targetSchedulesPostRAScheduling()) {
1183     if (MISchedPostRA)
1184       addPass(&PostMachineSchedulerID);
1185     else
1186       addPass(&PostRASchedulerID);
1187   }
1188 
1189   // GC
1190   addGCPasses();
1191 
1192   // Basic block placement.
1193   if (getOptLevel() != CodeGenOptLevel::None)
1194     addBlockPlacement();
1195 
1196   // Insert before XRay Instrumentation.
1197   addPass(&FEntryInserterID);
1198 
1199   addPass(&XRayInstrumentationID);
1200   addPass(&PatchableFunctionID);
1201 
1202   addPreEmitPass();
1203 
1204   if (TM->Options.EnableIPRA)
1205     // Collect register usage information and produce a register mask of
1206     // clobbered registers, to be used to optimize call sites.
1207     addPass(createRegUsageInfoCollector());
1208 
1209   // FIXME: Some backends are incompatible with running the verifier after
1210   // addPreEmitPass.  Maybe only pass "false" here for those targets?
1211   addPass(&FuncletLayoutID);
1212 
1213   addPass(&RemoveLoadsIntoFakeUsesID);
1214   addPass(&StackMapLivenessID);
1215   addPass(&LiveDebugValuesID);
1216   addPass(&MachineSanitizerBinaryMetadataID);
1217 
1218   if (TM->Options.EnableMachineOutliner &&
1219       getOptLevel() != CodeGenOptLevel::None &&
1220       EnableMachineOutliner != RunOutliner::NeverOutline) {
1221     bool RunOnAllFunctions =
1222         (EnableMachineOutliner == RunOutliner::AlwaysOutline);
1223     bool AddOutliner =
1224         RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
1225     if (AddOutliner)
1226       addPass(createMachineOutlinerPass(RunOnAllFunctions));
1227   }
1228 
1229   if (GCEmptyBlocks)
1230     addPass(llvm::createGCEmptyBasicBlocksPass());
1231 
1232   if (EnableFSDiscriminator)
1233     addPass(createMIRAddFSDiscriminatorsPass(
1234         sampleprof::FSDiscriminatorPass::PassLast));
1235 
1236   bool NeedsBBSections =
1237       TM->getBBSectionsType() != llvm::BasicBlockSection::None;
1238   // Machine function splitter uses the basic block sections feature. Both
1239   // cannot be enabled at the same time. We do not apply machine function
1240   // splitter if -basic-block-sections is requested.
1241   if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter ||
1242                            EnableMachineFunctionSplitter)) {
1243     const std::string ProfileFile = getFSProfileFile(TM);
1244     if (!ProfileFile.empty()) {
1245       if (EnableFSDiscriminator) {
1246         addPass(createMIRProfileLoaderPass(
1247             ProfileFile, getFSRemappingFile(TM),
1248             sampleprof::FSDiscriminatorPass::PassLast, nullptr));
1249       } else {
1250         // Sample profile is given, but FSDiscriminator is not
1251         // enabled, this may result in performance regression.
1252         WithColor::warning()
1253             << "Using AutoFDO without FSDiscriminator for MFS may regress "
1254                "performance.\n";
1255       }
1256     }
1257     addPass(createMachineFunctionSplitterPass());
1258   }
1259   // We run the BasicBlockSections pass if either we need BB sections or BB
1260   // address map (or both).
1261   if (NeedsBBSections || TM->Options.BBAddrMap) {
1262     if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1263       addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1264           TM->getBBSectionsFuncListBuf()));
1265       addPass(llvm::createBasicBlockPathCloningPass());
1266     }
1267     addPass(llvm::createBasicBlockSectionsPass());
1268   }
1269 
1270   addPostBBSections();
1271 
1272   if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
1273     addPass(createCFIFixup());
1274 
1275   PM->add(createStackFrameLayoutAnalysisPass());
1276 
1277   // Add passes that directly emit MI after all other MI passes.
1278   addPreEmitPass2();
1279 
1280   AddingMachinePasses = false;
1281 }
1282 
1283 /// Add passes that optimize machine instructions in SSA form.
1284 void TargetPassConfig::addMachineSSAOptimization() {
1285   // Pre-ra tail duplication.
1286   addPass(&EarlyTailDuplicateLegacyID);
1287 
1288   // Optimize PHIs before DCE: removing dead PHI cycles may make more
1289   // instructions dead.
1290   addPass(&OptimizePHIsLegacyID);
1291 
1292   // This pass merges large allocas. StackSlotColoring is a different pass
1293   // which merges spill slots.
1294   addPass(&StackColoringLegacyID);
1295 
1296   // If the target requests it, assign local variables to stack slots relative
1297   // to one another and simplify frame index references where possible.
1298   addPass(&LocalStackSlotAllocationID);
1299 
1300   // With optimization, dead code should already be eliminated. However
1301   // there is one known exception: lowered code for arguments that are only
1302   // used by tail calls, where the tail calls reuse the incoming stack
1303   // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1304   addPass(&DeadMachineInstructionElimID);
1305 
1306   // Allow targets to insert passes that improve instruction level parallelism,
1307   // like if-conversion. Such passes will typically need dominator trees and
1308   // loop info, just like LICM and CSE below.
1309   addILPOpts();
1310 
1311   addPass(&EarlyMachineLICMID);
1312   addPass(&MachineCSELegacyID);
1313 
1314   addPass(&MachineSinkingID);
1315 
1316   addPass(&PeepholeOptimizerID);
1317   // Clean-up the dead code that may have been generated by peephole
1318   // rewriting.
1319   addPass(&DeadMachineInstructionElimID);
1320 }
1321 
1322 //===---------------------------------------------------------------------===//
1323 /// Register Allocation Pass Configuration
1324 //===---------------------------------------------------------------------===//
1325 
1326 bool TargetPassConfig::getOptimizeRegAlloc() const {
1327   switch (OptimizeRegAlloc) {
1328   case cl::BOU_UNSET:
1329     return getOptLevel() != CodeGenOptLevel::None;
1330   case cl::BOU_TRUE:  return true;
1331   case cl::BOU_FALSE: return false;
1332   }
1333   llvm_unreachable("Invalid optimize-regalloc state");
1334 }
1335 
1336 /// A dummy default pass factory indicates whether the register allocator is
1337 /// overridden on the command line.
1338 static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1339 
1340 static RegisterRegAlloc
1341 defaultRegAlloc("default",
1342                 "pick register allocator based on -O option",
1343                 useDefaultRegisterAllocator);
1344 
1345 static void initializeDefaultRegisterAllocatorOnce() {
1346   if (!RegisterRegAlloc::getDefault())
1347     RegisterRegAlloc::setDefault(RegAlloc);
1348 }
1349 
1350 /// Instantiate the default register allocator pass for this target for either
1351 /// the optimized or unoptimized allocation path. This will be added to the pass
1352 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1353 /// in the optimized case.
1354 ///
1355 /// A target that uses the standard regalloc pass order for fast or optimized
1356 /// allocation may still override this for per-target regalloc
1357 /// selection. But -regalloc=... always takes precedence.
1358 FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1359   if (Optimized)
1360     return createGreedyRegisterAllocator();
1361   else
1362     return createFastRegisterAllocator();
1363 }
1364 
1365 /// Find and instantiate the register allocation pass requested by this target
1366 /// at the current optimization level.  Different register allocators are
1367 /// defined as separate passes because they may require different analysis.
1368 ///
1369 /// This helper ensures that the regalloc= option is always available,
1370 /// even for targets that override the default allocator.
1371 ///
1372 /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1373 /// this can be folded into addPass.
1374 FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1375   // Initialize the global default.
1376   llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
1377                   initializeDefaultRegisterAllocatorOnce);
1378 
1379   RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1380   if (Ctor != useDefaultRegisterAllocator)
1381     return Ctor();
1382 
1383   // With no -regalloc= override, ask the target for a regalloc pass.
1384   return createTargetRegisterAllocator(Optimized);
1385 }
1386 
1387 bool TargetPassConfig::isCustomizedRegAlloc() {
1388   return RegAlloc !=
1389          (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1390 }
1391 
1392 bool TargetPassConfig::addRegAssignAndRewriteFast() {
1393   if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1394       RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1395     report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
1396 
1397   addPass(createRegAllocPass(false));
1398 
1399   // Allow targets to change the register assignments after
1400   // fast register allocation.
1401   addPostFastRegAllocRewrite();
1402   return true;
1403 }
1404 
1405 bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1406   // Add the selected register allocation pass.
1407   addPass(createRegAllocPass(true));
1408 
1409   // Allow targets to change the register assignments before rewriting.
1410   addPreRewrite();
1411 
1412   // Finally rewrite virtual registers.
1413   addPass(&VirtRegRewriterID);
1414 
1415   // Regalloc scoring for ML-driven eviction - noop except when learning a new
1416   // eviction policy.
1417   addPass(createRegAllocScoringPass());
1418   return true;
1419 }
1420 
1421 /// Return true if the default global register allocator is in use and
1422 /// has not be overriden on the command line with '-regalloc=...'
1423 bool TargetPassConfig::usingDefaultRegAlloc() const {
1424   return RegAlloc.getNumOccurrences() == 0;
1425 }
1426 
1427 /// Add the minimum set of target-independent passes that are required for
1428 /// register allocation. No coalescing or scheduling.
1429 void TargetPassConfig::addFastRegAlloc() {
1430   addPass(&PHIEliminationID);
1431   addPass(&TwoAddressInstructionPassID);
1432 
1433   addRegAssignAndRewriteFast();
1434 }
1435 
1436 /// Add standard target-independent passes that are tightly coupled with
1437 /// optimized register allocation, including coalescing, machine instruction
1438 /// scheduling, and register allocation itself.
1439 void TargetPassConfig::addOptimizedRegAlloc() {
1440   addPass(&DetectDeadLanesID);
1441 
1442   addPass(&InitUndefID);
1443 
1444   addPass(&ProcessImplicitDefsID);
1445 
1446   // LiveVariables currently requires pure SSA form.
1447   //
1448   // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1449   // LiveVariables can be removed completely, and LiveIntervals can be directly
1450   // computed. (We still either need to regenerate kill flags after regalloc, or
1451   // preferably fix the scavenger to not depend on them).
1452   // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1453   // When LiveVariables is removed this has to be removed/moved either.
1454   // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1455   // after it with -stop-before/-stop-after.
1456   addPass(&UnreachableMachineBlockElimID);
1457   addPass(&LiveVariablesID);
1458 
1459   // Edge splitting is smarter with machine loop info.
1460   addPass(&MachineLoopInfoID);
1461   addPass(&PHIEliminationID);
1462 
1463   // Eventually, we want to run LiveIntervals before PHI elimination.
1464   if (EarlyLiveIntervals)
1465     addPass(&LiveIntervalsID);
1466 
1467   addPass(&TwoAddressInstructionPassID);
1468   addPass(&RegisterCoalescerID);
1469 
1470   // The machine scheduler may accidentally create disconnected components
1471   // when moving subregister definitions around, avoid this by splitting them to
1472   // separate vregs before. Splitting can also improve reg. allocation quality.
1473   addPass(&RenameIndependentSubregsID);
1474 
1475   // PreRA instruction scheduling.
1476   addPass(&MachineSchedulerID);
1477 
1478   if (addRegAssignAndRewriteOptimized()) {
1479     // Perform stack slot coloring and post-ra machine LICM.
1480     addPass(&StackSlotColoringID);
1481 
1482     // Allow targets to expand pseudo instructions depending on the choice of
1483     // registers before MachineCopyPropagation.
1484     addPostRewrite();
1485 
1486     // Copy propagate to forward register uses and try to eliminate COPYs that
1487     // were not coalesced.
1488     addPass(&MachineCopyPropagationID);
1489 
1490     // Run post-ra machine LICM to hoist reloads / remats.
1491     //
1492     // FIXME: can this move into MachineLateOptimization?
1493     addPass(&MachineLICMID);
1494   }
1495 }
1496 
1497 //===---------------------------------------------------------------------===//
1498 /// Post RegAlloc Pass Configuration
1499 //===---------------------------------------------------------------------===//
1500 
1501 /// Add passes that optimize machine instructions after register allocation.
1502 void TargetPassConfig::addMachineLateOptimization() {
1503   // Cleanup of redundant immediate/address loads.
1504   addPass(&MachineLateInstrsCleanupID);
1505 
1506   // Branch folding must be run after regalloc and prolog/epilog insertion.
1507   addPass(&BranchFolderPassID);
1508 
1509   // Tail duplication.
1510   // Note that duplicating tail just increases code size and degrades
1511   // performance for targets that require Structured Control Flow.
1512   // In addition it can also make CFG irreducible. Thus we disable it.
1513   if (!TM->requiresStructuredCFG())
1514     addPass(&TailDuplicateLegacyID);
1515 
1516   // Copy propagation.
1517   addPass(&MachineCopyPropagationID);
1518 }
1519 
1520 /// Add standard GC passes.
1521 bool TargetPassConfig::addGCPasses() {
1522   addPass(&GCMachineCodeAnalysisID);
1523   return true;
1524 }
1525 
1526 /// Add standard basic block placement passes.
1527 void TargetPassConfig::addBlockPlacement() {
1528   if (EnableFSDiscriminator) {
1529     addPass(createMIRAddFSDiscriminatorsPass(
1530         sampleprof::FSDiscriminatorPass::Pass2));
1531     const std::string ProfileFile = getFSProfileFile(TM);
1532     if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
1533       addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1534                                          sampleprof::FSDiscriminatorPass::Pass2,
1535                                          nullptr));
1536   }
1537   if (addPass(&MachineBlockPlacementID)) {
1538     // Run a separate pass to collect block placement statistics.
1539     if (EnableBlockPlacementStats)
1540       addPass(&MachineBlockPlacementStatsID);
1541   }
1542 }
1543 
1544 //===---------------------------------------------------------------------===//
1545 /// GlobalISel Configuration
1546 //===---------------------------------------------------------------------===//
1547 bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1548   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1549 }
1550 
1551 bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1552   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1553 }
1554 
1555 bool TargetPassConfig::isGISelCSEEnabled() const {
1556   return true;
1557 }
1558 
1559 std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1560   return std::make_unique<CSEConfigBase>();
1561 }
1562