xref: /llvm-project/llvm/include/llvm/Passes/CodeGenPassBuilder.h (revision 9efa7d7af3decfc5223963b19e1b85cacac48084)
1 //===- Construction of codegen pass pipelines ------------------*- C++ -*--===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// Interfaces for producing common pass manager configurations.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_PASSES_CODEGENPASSBUILDER_H
15 #define LLVM_PASSES_CODEGENPASSBUILDER_H
16 
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/BasicAliasAnalysis.h"
21 #include "llvm/Analysis/ProfileSummaryInfo.h"
22 #include "llvm/Analysis/ScopedNoAliasAA.h"
23 #include "llvm/Analysis/TargetTransformInfo.h"
24 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
25 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
26 #include "llvm/CodeGen/CallBrPrepare.h"
27 #include "llvm/CodeGen/CodeGenPrepare.h"
28 #include "llvm/CodeGen/DeadMachineInstructionElim.h"
29 #include "llvm/CodeGen/DwarfEHPrepare.h"
30 #include "llvm/CodeGen/EarlyIfConversion.h"
31 #include "llvm/CodeGen/ExpandLargeDivRem.h"
32 #include "llvm/CodeGen/ExpandLargeFpConvert.h"
33 #include "llvm/CodeGen/ExpandMemCmp.h"
34 #include "llvm/CodeGen/ExpandReductions.h"
35 #include "llvm/CodeGen/FinalizeISel.h"
36 #include "llvm/CodeGen/GCMetadata.h"
37 #include "llvm/CodeGen/GlobalMerge.h"
38 #include "llvm/CodeGen/GlobalMergeFunctions.h"
39 #include "llvm/CodeGen/IndirectBrExpand.h"
40 #include "llvm/CodeGen/InterleavedAccess.h"
41 #include "llvm/CodeGen/InterleavedLoadCombine.h"
42 #include "llvm/CodeGen/JMCInstrumenter.h"
43 #include "llvm/CodeGen/LiveIntervals.h"
44 #include "llvm/CodeGen/LocalStackSlotAllocation.h"
45 #include "llvm/CodeGen/LowerEmuTLS.h"
46 #include "llvm/CodeGen/MIRPrinter.h"
47 #include "llvm/CodeGen/MachineCSE.h"
48 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
49 #include "llvm/CodeGen/MachineLICM.h"
50 #include "llvm/CodeGen/MachineModuleInfo.h"
51 #include "llvm/CodeGen/MachinePassManager.h"
52 #include "llvm/CodeGen/MachineVerifier.h"
53 #include "llvm/CodeGen/OptimizePHIs.h"
54 #include "llvm/CodeGen/PHIElimination.h"
55 #include "llvm/CodeGen/PeepholeOptimizer.h"
56 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
57 #include "llvm/CodeGen/RegAllocFast.h"
58 #include "llvm/CodeGen/RegUsageInfoCollector.h"
59 #include "llvm/CodeGen/RegUsageInfoPropagate.h"
60 #include "llvm/CodeGen/RegisterUsageInfo.h"
61 #include "llvm/CodeGen/ReplaceWithVeclib.h"
62 #include "llvm/CodeGen/SafeStack.h"
63 #include "llvm/CodeGen/SelectOptimize.h"
64 #include "llvm/CodeGen/ShadowStackGCLowering.h"
65 #include "llvm/CodeGen/SjLjEHPrepare.h"
66 #include "llvm/CodeGen/StackColoring.h"
67 #include "llvm/CodeGen/StackProtector.h"
68 #include "llvm/CodeGen/TailDuplication.h"
69 #include "llvm/CodeGen/TargetPassConfig.h"
70 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
71 #include "llvm/CodeGen/UnreachableBlockElim.h"
72 #include "llvm/CodeGen/WasmEHPrepare.h"
73 #include "llvm/CodeGen/WinEHPrepare.h"
74 #include "llvm/IR/PassManager.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/IRPrinter/IRPrintingPasses.h"
77 #include "llvm/MC/MCAsmInfo.h"
78 #include "llvm/MC/MCTargetOptions.h"
79 #include "llvm/Support/CodeGen.h"
80 #include "llvm/Support/Debug.h"
81 #include "llvm/Support/Error.h"
82 #include "llvm/Support/ErrorHandling.h"
83 #include "llvm/Target/CGPassBuilderOption.h"
84 #include "llvm/Target/TargetMachine.h"
85 #include "llvm/Transforms/CFGuard.h"
86 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
87 #include "llvm/Transforms/Scalar/LoopPassManager.h"
88 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
89 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
90 #include "llvm/Transforms/Scalar/MergeICmps.h"
91 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
92 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
93 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
94 #include "llvm/Transforms/Utils/LowerInvoke.h"
95 #include <cassert>
96 #include <type_traits>
97 #include <utility>
98 
99 namespace llvm {
100 
101 // FIXME: Dummy target independent passes definitions that have not yet been
102 // ported to new pass manager. Once they do, remove these.
103 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME)                                   \
104   struct PASS_NAME : public PassInfoMixin<PASS_NAME> {                         \
105     template <typename... Ts> PASS_NAME(Ts &&...) {}                           \
106     PreservedAnalyses run(Function &, FunctionAnalysisManager &) {             \
107       return PreservedAnalyses::all();                                         \
108     }                                                                          \
109   };
110 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME)                             \
111   struct PASS_NAME : public PassInfoMixin<PASS_NAME> {                         \
112     template <typename... Ts> PASS_NAME(Ts &&...) {}                           \
113     PreservedAnalyses run(Module &, ModuleAnalysisManager &) {                 \
114       return PreservedAnalyses::all();                                         \
115     }                                                                          \
116   };
117 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)                           \
118   struct PASS_NAME : public PassInfoMixin<PASS_NAME> {                         \
119     template <typename... Ts> PASS_NAME(Ts &&...) {}                           \
120     PreservedAnalyses run(MachineFunction &,                                   \
121                           MachineFunctionAnalysisManager &) {                  \
122       return PreservedAnalyses::all();                                         \
123     }                                                                          \
124   };
125 #include "llvm/Passes/MachinePassRegistry.def"
126 
127 /// This class provides access to building LLVM's passes.
128 ///
129 /// Its members provide the baseline state available to passes during their
130 /// construction. The \c MachinePassRegistry.def file specifies how to construct
131 /// all of the built-in passes, and those may reference these members during
132 /// construction.
133 template <typename DerivedT, typename TargetMachineT> class CodeGenPassBuilder {
134 public:
135   explicit CodeGenPassBuilder(TargetMachineT &TM,
136                               const CGPassBuilderOption &Opts,
137                               PassInstrumentationCallbacks *PIC)
138       : TM(TM), Opt(Opts), PIC(PIC) {
139     // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
140     //     substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
141 
142     // Target should override TM.Options.EnableIPRA in their target-specific
143     // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
144     if (Opt.EnableIPRA)
145       TM.Options.EnableIPRA = *Opt.EnableIPRA;
146 
147     if (Opt.EnableGlobalISelAbort)
148       TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
149 
150     if (!Opt.OptimizeRegAlloc)
151       Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOptLevel::None;
152   }
153 
154   Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out,
155                       raw_pwrite_stream *DwoOut,
156                       CodeGenFileType FileType) const;
157 
158   PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
159     return PIC;
160   }
161 
162 protected:
163   template <typename PassT>
164   using has_required_t = decltype(std::declval<PassT &>().isRequired());
165 
166   template <typename PassT>
167   using is_module_pass_t = decltype(std::declval<PassT &>().run(
168       std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
169 
170   template <typename PassT>
171   using is_function_pass_t = decltype(std::declval<PassT &>().run(
172       std::declval<Function &>(), std::declval<FunctionAnalysisManager &>()));
173 
174   template <typename PassT>
175   using is_machine_function_pass_t = decltype(std::declval<PassT &>().run(
176       std::declval<MachineFunction &>(),
177       std::declval<MachineFunctionAnalysisManager &>()));
178 
179   // Function object to maintain state while adding codegen IR passes.
180   // TODO: add a Function -> MachineFunction adaptor and merge
181   // AddIRPass/AddMachinePass so we can have a function pipeline that runs both
182   // function passes and machine function passes.
183   class AddIRPass {
184   public:
185     AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {}
186     ~AddIRPass() {
187       if (!FPM.isEmpty())
188         MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
189     }
190 
191     template <typename PassT>
192     void operator()(PassT &&Pass, StringRef Name = PassT::name()) {
193       static_assert((is_detected<is_function_pass_t, PassT>::value ||
194                      is_detected<is_module_pass_t, PassT>::value) &&
195                     "Only module pass and function pass are supported.");
196       bool Required = false;
197       if constexpr (is_detected<has_required_t, PassT>::value)
198         Required = PassT::isRequired();
199       if (!PB.runBeforeAdding(Name) && !Required)
200         return;
201 
202       // Add Function Pass
203       if constexpr (is_detected<is_function_pass_t, PassT>::value) {
204         FPM.addPass(std::forward<PassT>(Pass));
205       } else {
206         // Add Module Pass
207         if (!FPM.isEmpty()) {
208           MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
209           FPM = FunctionPassManager();
210         }
211 
212         MPM.addPass(std::forward<PassT>(Pass));
213       }
214     }
215 
216   private:
217     ModulePassManager &MPM;
218     FunctionPassManager FPM;
219     const DerivedT &PB;
220   };
221 
222   // Function object to maintain state while adding codegen machine passes.
223   class AddMachinePass {
224   public:
225     AddMachinePass(ModulePassManager &MPM, const DerivedT &PB)
226         : MPM(MPM), PB(PB) {}
227     ~AddMachinePass() {
228       if (!MFPM.isEmpty()) {
229         FunctionPassManager FPM;
230         FPM.addPass(
231             createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
232         FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
233         MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
234       }
235     }
236 
237     template <typename PassT>
238     void operator()(PassT &&Pass, bool Force = false,
239                     StringRef Name = PassT::name()) {
240       static_assert((is_detected<is_machine_function_pass_t, PassT>::value ||
241                      is_detected<is_module_pass_t, PassT>::value) &&
242                     "Only module pass and function pass are supported.");
243 
244       if (!Force && !PB.runBeforeAdding(Name))
245         return;
246 
247       // Add Function Pass
248       if constexpr (is_detected<is_machine_function_pass_t, PassT>::value) {
249         MFPM.addPass(std::forward<PassT>(Pass));
250       } else {
251         // Add Module Pass
252         if (!MFPM.isEmpty()) {
253           MPM.addPass(createModuleToFunctionPassAdaptor(
254               createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
255           MFPM = MachineFunctionPassManager();
256         }
257 
258         MPM.addPass(std::forward<PassT>(Pass));
259       }
260 
261       for (auto &C : PB.AfterCallbacks)
262         C(Name, MFPM);
263     }
264 
265   private:
266     ModulePassManager &MPM;
267     MachineFunctionPassManager MFPM;
268     const DerivedT &PB;
269   };
270 
271   TargetMachineT &TM;
272   CGPassBuilderOption Opt;
273   PassInstrumentationCallbacks *PIC;
274 
275   template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
276   CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); }
277 
278   /// Check whether or not GlobalISel should abort on error.
279   /// When this is disabled, GlobalISel will fall back on SDISel instead of
280   /// erroring out.
281   bool isGlobalISelAbortEnabled() const {
282     return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
283   }
284 
285   /// Check whether or not a diagnostic should be emitted when GlobalISel
286   /// uses the fallback path. In other words, it will emit a diagnostic
287   /// when GlobalISel failed and isGlobalISelAbortEnabled is false.
288   bool reportDiagnosticWhenGlobalISelFallback() const {
289     return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
290   }
291 
292   /// addInstSelector - This method should install an instruction selector pass,
293   /// which converts from LLVM code to machine instructions.
294   Error addInstSelector(AddMachinePass &) const {
295     return make_error<StringError>("addInstSelector is not overridden",
296                                    inconvertibleErrorCode());
297   }
298 
299   /// Target can override this to add GlobalMergePass before all IR passes.
300   void addGlobalMergePass(AddIRPass &) const {}
301 
302   /// Add passes that optimize instruction level parallelism for out-of-order
303   /// targets. These passes are run while the machine code is still in SSA
304   /// form, so they can use MachineTraceMetrics to control their heuristics.
305   ///
306   /// All passes added here should preserve the MachineDominatorTree,
307   /// MachineLoopInfo, and MachineTraceMetrics analyses.
308   void addILPOpts(AddMachinePass &) const {}
309 
310   /// This method may be implemented by targets that want to run passes
311   /// immediately before register allocation.
312   void addPreRegAlloc(AddMachinePass &) const {}
313 
314   /// addPreRewrite - Add passes to the optimized register allocation pipeline
315   /// after register allocation is complete, but before virtual registers are
316   /// rewritten to physical registers.
317   ///
318   /// These passes must preserve VirtRegMap and LiveIntervals, and when running
319   /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
320   /// When these passes run, VirtRegMap contains legal physreg assignments for
321   /// all virtual registers.
322   ///
323   /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not
324   /// be honored. This is also not generally used for the fast variant,
325   /// where the allocation and rewriting are done in one pass.
326   void addPreRewrite(AddMachinePass &) const {}
327 
328   /// Add passes to be run immediately after virtual registers are rewritten
329   /// to physical registers.
330   void addPostRewrite(AddMachinePass &) const {}
331 
332   /// This method may be implemented by targets that want to run passes after
333   /// register allocation pass pipeline but before prolog-epilog insertion.
334   void addPostRegAlloc(AddMachinePass &) const {}
335 
336   /// This method may be implemented by targets that want to run passes after
337   /// prolog-epilog insertion and before the second instruction scheduling pass.
338   void addPreSched2(AddMachinePass &) const {}
339 
340   /// This pass may be implemented by targets that want to run passes
341   /// immediately before machine code is emitted.
342   void addPreEmitPass(AddMachinePass &) const {}
343 
344   /// Targets may add passes immediately before machine code is emitted in this
345   /// callback. This is called even later than `addPreEmitPass`.
346   // FIXME: Rename `addPreEmitPass` to something more sensible given its actual
347   // position and remove the `2` suffix here as this callback is what
348   // `addPreEmitPass` *should* be but in reality isn't.
349   void addPreEmitPass2(AddMachinePass &) const {}
350 
351   /// {{@ For GlobalISel
352   ///
353 
354   /// addPreISel - This method should add any "last minute" LLVM->LLVM
355   /// passes (which are run just before instruction selector).
356   void addPreISel(AddIRPass &) const {
357     llvm_unreachable("addPreISel is not overridden");
358   }
359 
360   /// This method should install an IR translator pass, which converts from
361   /// LLVM code to machine instructions with possibly generic opcodes.
362   Error addIRTranslator(AddMachinePass &) const {
363     return make_error<StringError>("addIRTranslator is not overridden",
364                                    inconvertibleErrorCode());
365   }
366 
367   /// This method may be implemented by targets that want to run passes
368   /// immediately before legalization.
369   void addPreLegalizeMachineIR(AddMachinePass &) const {}
370 
371   /// This method should install a legalize pass, which converts the instruction
372   /// sequence into one that can be selected by the target.
373   Error addLegalizeMachineIR(AddMachinePass &) const {
374     return make_error<StringError>("addLegalizeMachineIR is not overridden",
375                                    inconvertibleErrorCode());
376   }
377 
378   /// This method may be implemented by targets that want to run passes
379   /// immediately before the register bank selection.
380   void addPreRegBankSelect(AddMachinePass &) const {}
381 
382   /// This method should install a register bank selector pass, which
383   /// assigns register banks to virtual registers without a register
384   /// class or register banks.
385   Error addRegBankSelect(AddMachinePass &) const {
386     return make_error<StringError>("addRegBankSelect is not overridden",
387                                    inconvertibleErrorCode());
388   }
389 
390   /// This method may be implemented by targets that want to run passes
391   /// immediately before the (global) instruction selection.
392   void addPreGlobalInstructionSelect(AddMachinePass &) const {}
393 
394   /// This method should install a (global) instruction selector pass, which
395   /// converts possibly generic instructions to fully target-specific
396   /// instructions, thereby constraining all generic virtual registers to
397   /// register classes.
398   Error addGlobalInstructionSelect(AddMachinePass &) const {
399     return make_error<StringError>(
400         "addGlobalInstructionSelect is not overridden",
401         inconvertibleErrorCode());
402   }
403   /// @}}
404 
405   /// High level function that adds all passes necessary to go from llvm IR
406   /// representation to the MI representation.
407   /// Adds IR based lowering and target specific optimization passes and finally
408   /// the core instruction selection passes.
409   void addISelPasses(AddIRPass &) const;
410 
411   /// Add the actual instruction selection passes. This does not include
412   /// preparation passes on IR.
413   Error addCoreISelPasses(AddMachinePass &) const;
414 
415   /// Add the complete, standard set of LLVM CodeGen passes.
416   /// Fully developed targets will not generally override this.
417   Error addMachinePasses(AddMachinePass &) const;
418 
419   /// Add passes to lower exception handling for the code generator.
420   void addPassesToHandleExceptions(AddIRPass &) const;
421 
422   /// Add common target configurable passes that perform LLVM IR to IR
423   /// transforms following machine independent optimization.
424   void addIRPasses(AddIRPass &) const;
425 
426   /// Add pass to prepare the LLVM IR for code generation. This should be done
427   /// before exception handling preparation passes.
428   void addCodeGenPrepare(AddIRPass &) const;
429 
430   /// Add common passes that perform LLVM IR to IR transforms in preparation for
431   /// instruction selection.
432   void addISelPrepare(AddIRPass &) const;
433 
434   /// Methods with trivial inline returns are convenient points in the common
435   /// codegen pass pipeline where targets may insert passes. Methods with
436   /// out-of-line standard implementations are major CodeGen stages called by
437   /// addMachinePasses. Some targets may override major stages when inserting
438   /// passes is insufficient, but maintaining overriden stages is more work.
439   ///
440 
441   /// addMachineSSAOptimization - Add standard passes that optimize machine
442   /// instructions in SSA form.
443   void addMachineSSAOptimization(AddMachinePass &) const;
444 
445   /// addFastRegAlloc - Add the minimum set of target-independent passes that
446   /// are required for fast register allocation.
447   Error addFastRegAlloc(AddMachinePass &) const;
448 
449   /// addOptimizedRegAlloc - Add passes related to register allocation.
450   /// CodeGenTargetMachineImpl provides standard regalloc passes for most
451   /// targets.
452   void addOptimizedRegAlloc(AddMachinePass &) const;
453 
454   /// Add passes that optimize machine instructions after register allocation.
455   void addMachineLateOptimization(AddMachinePass &) const;
456 
457   /// addGCPasses - Add late codegen passes that analyze code for garbage
458   /// collection. This should return true if GC info should be printed after
459   /// these passes.
460   void addGCPasses(AddMachinePass &) const {}
461 
462   /// Add standard basic block placement passes.
463   void addBlockPlacement(AddMachinePass &) const;
464 
465   using CreateMCStreamer =
466       std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
467   void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
468     llvm_unreachable("addAsmPrinter is not overridden");
469   }
470 
471   /// Utilities for targets to add passes to the pass manager.
472   ///
473 
474   /// createTargetRegisterAllocator - Create the register allocator pass for
475   /// this target at the current optimization level.
476   void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const;
477 
478   /// addMachinePasses helper to create the target-selected or overriden
479   /// regalloc pass.
480   void addRegAllocPass(AddMachinePass &, bool Optimized) const;
481 
482   /// Add core register alloator passes which do the actual register assignment
483   /// and rewriting. \returns true if any passes were added.
484   Error addRegAssignmentFast(AddMachinePass &) const;
485   Error addRegAssignmentOptimized(AddMachinePass &) const;
486 
487   /// Allow the target to disable a specific pass by default.
488   /// Backend can declare unwanted passes in constructor.
489   template <typename... PassTs> void disablePass() {
490     BeforeCallbacks.emplace_back(
491         [](StringRef Name) { return ((Name != PassTs::name()) && ...); });
492   }
493 
494   /// Insert InsertedPass pass after TargetPass pass.
495   /// Only machine function passes are supported.
496   template <typename TargetPassT, typename InsertedPassT>
497   void insertPass(InsertedPassT &&Pass) {
498     AfterCallbacks.emplace_back(
499         [&](StringRef Name, MachineFunctionPassManager &MFPM) mutable {
500           if (Name == TargetPassT::name())
501             MFPM.addPass(std::forward<InsertedPassT>(Pass));
502         });
503   }
504 
505 private:
506   DerivedT &derived() { return static_cast<DerivedT &>(*this); }
507   const DerivedT &derived() const {
508     return static_cast<const DerivedT &>(*this);
509   }
510 
511   bool runBeforeAdding(StringRef Name) const {
512     bool ShouldAdd = true;
513     for (auto &C : BeforeCallbacks)
514       ShouldAdd &= C(Name);
515     return ShouldAdd;
516   }
517 
518   void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info) const;
519 
520   Error verifyStartStop(const TargetPassConfig::StartStopInfo &Info) const;
521 
522   mutable SmallVector<llvm::unique_function<bool(StringRef)>, 4>
523       BeforeCallbacks;
524   mutable SmallVector<
525       llvm::unique_function<void(StringRef, MachineFunctionPassManager &)>, 4>
526       AfterCallbacks;
527 
528   /// Helper variable for `-start-before/-start-after/-stop-before/-stop-after`
529   mutable bool Started = true;
530   mutable bool Stopped = true;
531 };
532 
533 template <typename Derived, typename TargetMachineT>
534 Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
535     ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
536     CodeGenFileType FileType) const {
537   auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
538   if (!StartStopInfo)
539     return StartStopInfo.takeError();
540   setStartStopPasses(*StartStopInfo);
541 
542   bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline();
543   bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null;
544 
545   {
546     AddIRPass addIRPass(MPM, derived());
547     addIRPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
548     addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
549     addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
550     addISelPasses(addIRPass);
551   }
552 
553   AddMachinePass addPass(MPM, derived());
554 
555   if (PrintMIR)
556     addPass(PrintMIRPreparePass(Out), /*Force=*/true);
557 
558   if (auto Err = addCoreISelPasses(addPass))
559     return std::move(Err);
560 
561   if (auto Err = derived().addMachinePasses(addPass))
562     return std::move(Err);
563 
564   if (!Opt.DisableVerify)
565     addPass(MachineVerifierPass());
566 
567   if (PrintAsm) {
568     derived().addAsmPrinter(
569         addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
570           return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
571         });
572   }
573 
574   if (PrintMIR)
575     addPass(PrintMIRPass(Out), /*Force=*/true);
576 
577   return verifyStartStop(*StartStopInfo);
578 }
579 
580 template <typename Derived, typename TargetMachineT>
581 void CodeGenPassBuilder<Derived, TargetMachineT>::setStartStopPasses(
582     const TargetPassConfig::StartStopInfo &Info) const {
583   if (!Info.StartPass.empty()) {
584     Started = false;
585     BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StartAfter,
586                                   Count = 0u](StringRef ClassName) mutable {
587       if (Count == Info.StartInstanceNum) {
588         if (AfterFlag) {
589           AfterFlag = false;
590           Started = true;
591         }
592         return Started;
593       }
594 
595       auto PassName = PIC->getPassNameForClassName(ClassName);
596       if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum)
597         Started = !Info.StartAfter;
598 
599       return Started;
600     });
601   }
602 
603   if (!Info.StopPass.empty()) {
604     Stopped = false;
605     BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StopAfter,
606                                   Count = 0u](StringRef ClassName) mutable {
607       if (Count == Info.StopInstanceNum) {
608         if (AfterFlag) {
609           AfterFlag = false;
610           Stopped = true;
611         }
612         return !Stopped;
613       }
614 
615       auto PassName = PIC->getPassNameForClassName(ClassName);
616       if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum)
617         Stopped = !Info.StopAfter;
618       return !Stopped;
619     });
620   }
621 }
622 
623 template <typename Derived, typename TargetMachineT>
624 Error CodeGenPassBuilder<Derived, TargetMachineT>::verifyStartStop(
625     const TargetPassConfig::StartStopInfo &Info) const {
626   if (Started && Stopped)
627     return Error::success();
628 
629   if (!Started)
630     return make_error<StringError>(
631         "Can't find start pass \"" + Info.StartPass + "\".",
632         std::make_error_code(std::errc::invalid_argument));
633   if (!Stopped)
634     return make_error<StringError>(
635         "Can't find stop pass \"" + Info.StopPass + "\".",
636         std::make_error_code(std::errc::invalid_argument));
637   return Error::success();
638 }
639 
640 template <typename Derived, typename TargetMachineT>
641 void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses(
642     AddIRPass &addPass) const {
643   derived().addGlobalMergePass(addPass);
644   if (TM.useEmulatedTLS())
645     addPass(LowerEmuTLSPass());
646 
647   addPass(PreISelIntrinsicLoweringPass(&TM));
648   addPass(ExpandLargeDivRemPass(&TM));
649   addPass(ExpandLargeFpConvertPass(&TM));
650 
651   derived().addIRPasses(addPass);
652   derived().addCodeGenPrepare(addPass);
653   addPassesToHandleExceptions(addPass);
654   derived().addISelPrepare(addPass);
655 }
656 
657 /// Add common target configurable passes that perform LLVM IR to IR transforms
658 /// following machine independent optimization.
659 template <typename Derived, typename TargetMachineT>
660 void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses(
661     AddIRPass &addPass) const {
662   // Before running any passes, run the verifier to determine if the input
663   // coming from the front-end and/or optimizer is valid.
664   if (!Opt.DisableVerify)
665     addPass(VerifierPass());
666 
667   // Run loop strength reduction before anything else.
668   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) {
669     addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(),
670                                             /*UseMemorySSA=*/true));
671   }
672 
673   if (getOptLevel() != CodeGenOptLevel::None) {
674     // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
675     // loads and compares. ExpandMemCmpPass then tries to expand those calls
676     // into optimally-sized loads and compares. The transforms are enabled by a
677     // target lowering hook.
678     if (!Opt.DisableMergeICmps)
679       addPass(MergeICmpsPass());
680     addPass(ExpandMemCmpPass(&TM));
681   }
682 
683   // Run GC lowering passes for builtin collectors
684   // TODO: add a pass insertion point here
685   addPass(GCLoweringPass());
686   addPass(ShadowStackGCLoweringPass());
687   addPass(LowerConstantIntrinsicsPass());
688 
689   // Make sure that no unreachable blocks are instruction selected.
690   addPass(UnreachableBlockElimPass());
691 
692   // Prepare expensive constants for SelectionDAG.
693   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting)
694     addPass(ConstantHoistingPass());
695 
696   // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
697   // operands with calls to the corresponding functions in a vector library.
698   if (getOptLevel() != CodeGenOptLevel::None)
699     addPass(ReplaceWithVeclib());
700 
701   if (getOptLevel() != CodeGenOptLevel::None &&
702       !Opt.DisablePartialLibcallInlining)
703     addPass(PartiallyInlineLibCallsPass());
704 
705   // Instrument function entry and exit, e.g. with calls to mcount().
706   addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
707 
708   // Add scalarization of target's unsupported masked memory intrinsics pass.
709   // the unsupported intrinsic will be replaced with a chain of basic blocks,
710   // that stores/loads element one-by-one if the appropriate mask bit is set.
711   addPass(ScalarizeMaskedMemIntrinPass());
712 
713   // Expand reduction intrinsics into shuffle sequences if the target wants to.
714   addPass(ExpandReductionsPass());
715 
716   // Convert conditional moves to conditional jumps when profitable.
717   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize)
718     addPass(SelectOptimizePass(&TM));
719 
720   if (Opt.EnableGlobalMergeFunc)
721     addPass(GlobalMergeFuncPass());
722 }
723 
724 /// Turn exception handling constructs into something the code generators can
725 /// handle.
726 template <typename Derived, typename TargetMachineT>
727 void CodeGenPassBuilder<Derived, TargetMachineT>::addPassesToHandleExceptions(
728     AddIRPass &addPass) const {
729   const MCAsmInfo *MCAI = TM.getMCAsmInfo();
730   assert(MCAI && "No MCAsmInfo");
731   switch (MCAI->getExceptionHandlingType()) {
732   case ExceptionHandling::SjLj:
733     // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
734     // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
735     // catch info can get misplaced when a selector ends up more than one block
736     // removed from the parent invoke(s). This could happen when a landing
737     // pad is shared by multiple invokes and is also a target of a normal
738     // edge from elsewhere.
739     addPass(SjLjEHPreparePass(&TM));
740     [[fallthrough]];
741   case ExceptionHandling::DwarfCFI:
742   case ExceptionHandling::ARM:
743   case ExceptionHandling::AIX:
744   case ExceptionHandling::ZOS:
745     addPass(DwarfEHPreparePass(&TM));
746     break;
747   case ExceptionHandling::WinEH:
748     // We support using both GCC-style and MSVC-style exceptions on Windows, so
749     // add both preparation passes. Each pass will only actually run if it
750     // recognizes the personality function.
751     addPass(WinEHPreparePass());
752     addPass(DwarfEHPreparePass(&TM));
753     break;
754   case ExceptionHandling::Wasm:
755     // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
756     // on catchpads and cleanuppads because it does not outline them into
757     // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
758     // should remove PHIs there.
759     addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false));
760     addPass(WasmEHPreparePass());
761     break;
762   case ExceptionHandling::None:
763     addPass(LowerInvokePass());
764 
765     // The lower invoke pass may create unreachable code. Remove it.
766     addPass(UnreachableBlockElimPass());
767     break;
768   }
769 }
770 
771 /// Add pass to prepare the LLVM IR for code generation. This should be done
772 /// before exception handling preparation passes.
773 template <typename Derived, typename TargetMachineT>
774 void CodeGenPassBuilder<Derived, TargetMachineT>::addCodeGenPrepare(
775     AddIRPass &addPass) const {
776   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
777     addPass(CodeGenPreparePass(&TM));
778   // TODO: Default ctor'd RewriteSymbolPass is no-op.
779   // addPass(RewriteSymbolPass());
780 }
781 
782 /// Add common passes that perform LLVM IR to IR transforms in preparation for
783 /// instruction selection.
784 template <typename Derived, typename TargetMachineT>
785 void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare(
786     AddIRPass &addPass) const {
787   derived().addPreISel(addPass);
788 
789   addPass(CallBrPreparePass());
790   // Add both the safe stack and the stack protection passes: each of them will
791   // only protect functions that have corresponding attributes.
792   addPass(SafeStackPass(&TM));
793   addPass(StackProtectorPass(&TM));
794 
795   if (Opt.PrintISelInput)
796     addPass(PrintFunctionPass(dbgs(),
797                               "\n\n*** Final LLVM Code input to ISel ***\n"));
798 
799   // All passes which modify the LLVM IR are now complete; run the verifier
800   // to ensure that the IR is valid.
801   if (!Opt.DisableVerify)
802     addPass(VerifierPass());
803 }
804 
805 template <typename Derived, typename TargetMachineT>
806 Error CodeGenPassBuilder<Derived, TargetMachineT>::addCoreISelPasses(
807     AddMachinePass &addPass) const {
808   // Enable FastISel with -fast-isel, but allow that to be overridden.
809   TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true));
810 
811   // Determine an instruction selector.
812   enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
813   SelectorType Selector;
814 
815   if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true)
816     Selector = SelectorType::FastISel;
817   else if ((Opt.EnableGlobalISelOption &&
818             *Opt.EnableGlobalISelOption == true) ||
819            (TM.Options.EnableGlobalISel &&
820             (!Opt.EnableGlobalISelOption ||
821              *Opt.EnableGlobalISelOption == false)))
822     Selector = SelectorType::GlobalISel;
823   else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel())
824     Selector = SelectorType::FastISel;
825   else
826     Selector = SelectorType::SelectionDAG;
827 
828   // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
829   if (Selector == SelectorType::FastISel) {
830     TM.setFastISel(true);
831     TM.setGlobalISel(false);
832   } else if (Selector == SelectorType::GlobalISel) {
833     TM.setFastISel(false);
834     TM.setGlobalISel(true);
835   }
836 
837   // Add instruction selector passes.
838   if (Selector == SelectorType::GlobalISel) {
839     if (auto Err = derived().addIRTranslator(addPass))
840       return std::move(Err);
841 
842     derived().addPreLegalizeMachineIR(addPass);
843 
844     if (auto Err = derived().addLegalizeMachineIR(addPass))
845       return std::move(Err);
846 
847     // Before running the register bank selector, ask the target if it
848     // wants to run some passes.
849     derived().addPreRegBankSelect(addPass);
850 
851     if (auto Err = derived().addRegBankSelect(addPass))
852       return std::move(Err);
853 
854     derived().addPreGlobalInstructionSelect(addPass);
855 
856     if (auto Err = derived().addGlobalInstructionSelect(addPass))
857       return std::move(Err);
858 
859     // Pass to reset the MachineFunction if the ISel failed.
860     addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
861                                      isGlobalISelAbortEnabled()));
862 
863     // Provide a fallback path when we do not want to abort on
864     // not-yet-supported input.
865     if (!isGlobalISelAbortEnabled())
866       if (auto Err = derived().addInstSelector(addPass))
867         return std::move(Err);
868 
869   } else if (auto Err = derived().addInstSelector(addPass))
870     return std::move(Err);
871 
872   // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
873   // FinalizeISel.
874   addPass(FinalizeISelPass());
875 
876   // // Print the instruction selected machine code...
877   // printAndVerify("After Instruction Selection");
878 
879   return Error::success();
880 }
881 
882 /// Add the complete set of target-independent postISel code generator passes.
883 ///
884 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
885 /// with nontrivial configuration or multiple passes are broken out below in
886 /// add%Stage routines.
887 ///
888 /// Any CodeGenPassBuilder<Derived, TargetMachine>::addXX routine may be
889 /// overriden by the Target. The addPre/Post methods with empty header
890 /// implementations allow injecting target-specific fixups just before or after
891 /// major stages. Additionally, targets have the flexibility to change pass
892 /// order within a stage by overriding default implementation of add%Stage
893 /// routines below. Each technique has maintainability tradeoffs because
894 /// alternate pass orders are not well supported. addPre/Post works better if
895 /// the target pass is easily tied to a common pass. But if it has subtle
896 /// dependencies on multiple passes, the target should override the stage
897 /// instead.
898 template <typename Derived, typename TargetMachineT>
899 Error CodeGenPassBuilder<Derived, TargetMachineT>::addMachinePasses(
900     AddMachinePass &addPass) const {
901   // Add passes that optimize machine instructions in SSA form.
902   if (getOptLevel() != CodeGenOptLevel::None) {
903     derived().addMachineSSAOptimization(addPass);
904   } else {
905     // If the target requests it, assign local variables to stack slots relative
906     // to one another and simplify frame index references where possible.
907     addPass(LocalStackSlotAllocationPass());
908   }
909 
910   if (TM.Options.EnableIPRA) {
911     addPass(RequireAnalysisPass<PhysicalRegisterUsageAnalysis, Module>());
912     addPass(RegUsageInfoPropagationPass());
913   }
914   // Run pre-ra passes.
915   derived().addPreRegAlloc(addPass);
916 
917   // Run register allocation and passes that are tightly coupled with it,
918   // including phi elimination and scheduling.
919   if (*Opt.OptimizeRegAlloc) {
920     derived().addOptimizedRegAlloc(addPass);
921   } else {
922     if (auto Err = derived().addFastRegAlloc(addPass))
923       return Err;
924   }
925 
926   // Run post-ra passes.
927   derived().addPostRegAlloc(addPass);
928 
929   addPass(RemoveRedundantDebugValuesPass());
930 
931   // Insert prolog/epilog code.  Eliminate abstract frame index references...
932   if (getOptLevel() != CodeGenOptLevel::None) {
933     addPass(PostRAMachineSinkingPass());
934     addPass(ShrinkWrapPass());
935   }
936 
937   addPass(PrologEpilogInserterPass());
938 
939   /// Add passes that optimize machine instructions after register allocation.
940   if (getOptLevel() != CodeGenOptLevel::None)
941     derived().addMachineLateOptimization(addPass);
942 
943   // Expand pseudo instructions before second scheduling pass.
944   addPass(ExpandPostRAPseudosPass());
945 
946   // Run pre-sched2 passes.
947   derived().addPreSched2(addPass);
948 
949   if (Opt.EnableImplicitNullChecks)
950     addPass(ImplicitNullChecksPass());
951 
952   // Second pass scheduler.
953   // Let Target optionally insert this pass by itself at some other
954   // point.
955   if (getOptLevel() != CodeGenOptLevel::None &&
956       !TM.targetSchedulesPostRAScheduling()) {
957     if (Opt.MISchedPostRA)
958       addPass(PostMachineSchedulerPass());
959     else
960       addPass(PostRASchedulerPass());
961   }
962 
963   // GC
964   derived().addGCPasses(addPass);
965 
966   // Basic block placement.
967   if (getOptLevel() != CodeGenOptLevel::None)
968     derived().addBlockPlacement(addPass);
969 
970   // Insert before XRay Instrumentation.
971   addPass(FEntryInserterPass());
972 
973   addPass(XRayInstrumentationPass());
974   addPass(PatchableFunctionPass());
975 
976   derived().addPreEmitPass(addPass);
977 
978   if (TM.Options.EnableIPRA)
979     // Collect register usage information and produce a register mask of
980     // clobbered registers, to be used to optimize call sites.
981     addPass(RegUsageInfoCollectorPass());
982 
983   addPass(FuncletLayoutPass());
984 
985   addPass(StackMapLivenessPass());
986   addPass(LiveDebugValuesPass());
987   addPass(MachineSanitizerBinaryMetadata());
988 
989   if (TM.Options.EnableMachineOutliner &&
990       getOptLevel() != CodeGenOptLevel::None &&
991       Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
992     bool RunOnAllFunctions =
993         (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline);
994     bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining;
995     if (AddOutliner)
996       addPass(MachineOutlinerPass(RunOnAllFunctions));
997   }
998 
999   // Add passes that directly emit MI after all other MI passes.
1000   derived().addPreEmitPass2(addPass);
1001 
1002   return Error::success();
1003 }
1004 
1005 /// Add passes that optimize machine instructions in SSA form.
1006 template <typename Derived, typename TargetMachineT>
1007 void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization(
1008     AddMachinePass &addPass) const {
1009   // Pre-ra tail duplication.
1010   addPass(EarlyTailDuplicatePass());
1011 
1012   // Optimize PHIs before DCE: removing dead PHI cycles may make more
1013   // instructions dead.
1014   addPass(OptimizePHIsPass());
1015 
1016   // This pass merges large allocas. StackSlotColoring is a different pass
1017   // which merges spill slots.
1018   addPass(StackColoringPass());
1019 
1020   // If the target requests it, assign local variables to stack slots relative
1021   // to one another and simplify frame index references where possible.
1022   addPass(LocalStackSlotAllocationPass());
1023 
1024   // With optimization, dead code should already be eliminated. However
1025   // there is one known exception: lowered code for arguments that are only
1026   // used by tail calls, where the tail calls reuse the incoming stack
1027   // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1028   addPass(DeadMachineInstructionElimPass());
1029 
1030   // Allow targets to insert passes that improve instruction level parallelism,
1031   // like if-conversion. Such passes will typically need dominator trees and
1032   // loop info, just like LICM and CSE below.
1033   derived().addILPOpts(addPass);
1034 
1035   addPass(EarlyMachineLICMPass());
1036   addPass(MachineCSEPass());
1037 
1038   addPass(MachineSinkingPass());
1039 
1040   addPass(PeepholeOptimizerPass());
1041   // Clean-up the dead code that may have been generated by peephole
1042   // rewriting.
1043   addPass(DeadMachineInstructionElimPass());
1044 }
1045 
1046 //===---------------------------------------------------------------------===//
1047 /// Register Allocation Pass Configuration
1048 //===---------------------------------------------------------------------===//
1049 
1050 /// Instantiate the default register allocator pass for this target for either
1051 /// the optimized or unoptimized allocation path. This will be added to the pass
1052 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1053 /// in the optimized case.
1054 ///
1055 /// A target that uses the standard regalloc pass order for fast or optimized
1056 /// allocation may still override this for per-target regalloc
1057 /// selection. But -regalloc=... always takes precedence.
1058 template <typename Derived, typename TargetMachineT>
1059 void CodeGenPassBuilder<Derived, TargetMachineT>::addTargetRegisterAllocator(
1060     AddMachinePass &addPass, bool Optimized) const {
1061   if (Optimized)
1062     addPass(RAGreedyPass());
1063   else
1064     addPass(RegAllocFastPass());
1065 }
1066 
1067 /// Find and instantiate the register allocation pass requested by this target
1068 /// at the current optimization level.  Different register allocators are
1069 /// defined as separate passes because they may require different analysis.
1070 template <typename Derived, typename TargetMachineT>
1071 void CodeGenPassBuilder<Derived, TargetMachineT>::addRegAllocPass(
1072     AddMachinePass &addPass, bool Optimized) const {
1073   // TODO: Parse Opt.RegAlloc to add register allocator.
1074 }
1075 
1076 template <typename Derived, typename TargetMachineT>
1077 Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentFast(
1078     AddMachinePass &addPass) const {
1079   // TODO: Ensure allocator is default or fast.
1080   addRegAllocPass(addPass, false);
1081   return Error::success();
1082 }
1083 
1084 template <typename Derived, typename TargetMachineT>
1085 Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentOptimized(
1086     AddMachinePass &addPass) const {
1087   // Add the selected register allocation pass.
1088   addRegAllocPass(addPass, true);
1089 
1090   // Allow targets to change the register assignments before rewriting.
1091   derived().addPreRewrite(addPass);
1092 
1093   // Finally rewrite virtual registers.
1094   addPass(VirtRegRewriterPass());
1095   // Perform stack slot coloring and post-ra machine LICM.
1096   //
1097   // FIXME: Re-enable coloring with register when it's capable of adding
1098   // kill markers.
1099   addPass(StackSlotColoringPass());
1100 
1101   return Error::success();
1102 }
1103 
1104 /// Add the minimum set of target-independent passes that are required for
1105 /// register allocation. No coalescing or scheduling.
1106 template <typename Derived, typename TargetMachineT>
1107 Error CodeGenPassBuilder<Derived, TargetMachineT>::addFastRegAlloc(
1108     AddMachinePass &addPass) const {
1109   addPass(PHIEliminationPass());
1110   addPass(TwoAddressInstructionPass());
1111   return derived().addRegAssignmentFast(addPass);
1112 }
1113 
1114 /// Add standard target-independent passes that are tightly coupled with
1115 /// optimized register allocation, including coalescing, machine instruction
1116 /// scheduling, and register allocation itself.
1117 template <typename Derived, typename TargetMachineT>
1118 void CodeGenPassBuilder<Derived, TargetMachineT>::addOptimizedRegAlloc(
1119     AddMachinePass &addPass) const {
1120   addPass(DetectDeadLanesPass());
1121 
1122   addPass(InitUndefPass());
1123 
1124   addPass(ProcessImplicitDefsPass());
1125 
1126   // Edge splitting is smarter with machine loop info.
1127   addPass(PHIEliminationPass());
1128 
1129   // Eventually, we want to run LiveIntervals before PHI elimination.
1130   if (Opt.EarlyLiveIntervals)
1131     addPass(RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>());
1132 
1133   addPass(TwoAddressInstructionPass());
1134   addPass(RegisterCoalescerPass());
1135 
1136   // The machine scheduler may accidentally create disconnected components
1137   // when moving subregister definitions around, avoid this by splitting them to
1138   // separate vregs before. Splitting can also improve reg. allocation quality.
1139   addPass(RenameIndependentSubregsPass());
1140 
1141   // PreRA instruction scheduling.
1142   addPass(MachineSchedulerPass());
1143 
1144   if (derived().addRegAssignmentOptimized(addPass)) {
1145     // Allow targets to expand pseudo instructions depending on the choice of
1146     // registers before MachineCopyPropagation.
1147     derived().addPostRewrite(addPass);
1148 
1149     // Copy propagate to forward register uses and try to eliminate COPYs that
1150     // were not coalesced.
1151     addPass(MachineCopyPropagationPass());
1152 
1153     // Run post-ra machine LICM to hoist reloads / remats.
1154     //
1155     // FIXME: can this move into MachineLateOptimization?
1156     addPass(MachineLICMPass());
1157   }
1158 }
1159 
1160 //===---------------------------------------------------------------------===//
1161 /// Post RegAlloc Pass Configuration
1162 //===---------------------------------------------------------------------===//
1163 
1164 /// Add passes that optimize machine instructions after register allocation.
1165 template <typename Derived, typename TargetMachineT>
1166 void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineLateOptimization(
1167     AddMachinePass &addPass) const {
1168   // Branch folding must be run after regalloc and prolog/epilog insertion.
1169   addPass(BranchFolderPass());
1170 
1171   // Tail duplication.
1172   // Note that duplicating tail just increases code size and degrades
1173   // performance for targets that require Structured Control Flow.
1174   // In addition it can also make CFG irreducible. Thus we disable it.
1175   if (!TM.requiresStructuredCFG())
1176     addPass(TailDuplicatePass());
1177 
1178   // Cleanup of redundant (identical) address/immediate loads.
1179   addPass(MachineLateInstrsCleanupPass());
1180 
1181   // Copy propagation.
1182   addPass(MachineCopyPropagationPass());
1183 }
1184 
1185 /// Add standard basic block placement passes.
1186 template <typename Derived, typename TargetMachineT>
1187 void CodeGenPassBuilder<Derived, TargetMachineT>::addBlockPlacement(
1188     AddMachinePass &addPass) const {
1189   addPass(MachineBlockPlacementPass());
1190   // Run a separate pass to collect block placement statistics.
1191   if (Opt.EnableBlockPlacementStats)
1192     addPass(MachineBlockPlacementStatsPass());
1193 }
1194 
1195 } // namespace llvm
1196 
1197 #endif // LLVM_PASSES_CODEGENPASSBUILDER_H
1198