1 //===- Construction of codegen pass pipelines ------------------*- C++ -*--===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// Interfaces for registering analysis passes, producing common pass manager
11 /// configurations, and parsing of pass pipelines.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_CODEGENPASSBUILDER_H
16 #define LLVM_CODEGEN_CODEGENPASSBUILDER_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/Analysis/AliasAnalysis.h"
21 #include "llvm/Analysis/BasicAliasAnalysis.h"
22 #include "llvm/Analysis/ScopedNoAliasAA.h"
23 #include "llvm/Analysis/TargetTransformInfo.h"
24 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
25 #include "llvm/CodeGen/ExpandReductions.h"
26 #include "llvm/CodeGen/MachinePassManager.h"
27 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
28 #include "llvm/CodeGen/ReplaceWithVeclib.h"
29 #include "llvm/CodeGen/UnreachableBlockElim.h"
30 #include "llvm/IR/PassManager.h"
31 #include "llvm/IR/Verifier.h"
32 #include "llvm/IRPrinter/IRPrintingPasses.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCTargetOptions.h"
35 #include "llvm/Support/CodeGen.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/Error.h"
38 #include "llvm/Support/ErrorHandling.h"
39 #include "llvm/Target/CGPassBuilderOption.h"
40 #include "llvm/Target/TargetMachine.h"
41 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
42 #include "llvm/Transforms/Scalar/LoopPassManager.h"
43 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
44 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
45 #include "llvm/Transforms/Scalar/MergeICmps.h"
46 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
47 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
48 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
49 #include "llvm/Transforms/Utils/LowerInvoke.h"
50 #include <cassert>
51 #include <type_traits>
52 #include <utility>
53
54 namespace llvm {
55
56 // FIXME: Dummy target independent passes definitions that have not yet been
57 // ported to new pass manager. Once they do, remove these.
58 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
59 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
60 template <typename... Ts> PASS_NAME(Ts &&...) {} \
61 PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \
62 return PreservedAnalyses::all(); \
63 } \
64 };
65 #define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
66 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
67 template <typename... Ts> PASS_NAME(Ts &&...) {} \
68 PreservedAnalyses run(Module &, ModuleAnalysisManager &) { \
69 return PreservedAnalyses::all(); \
70 } \
71 };
72 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
73 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
74 template <typename... Ts> PASS_NAME(Ts &&...) {} \
75 Error run(Module &, MachineFunctionAnalysisManager &) { \
76 return Error::success(); \
77 } \
78 PreservedAnalyses run(MachineFunction &, \
79 MachineFunctionAnalysisManager &) { \
80 llvm_unreachable("this api is to make new PM api happy"); \
81 } \
82 static AnalysisKey Key; \
83 };
84 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
85 struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \
86 template <typename... Ts> PASS_NAME(Ts &&...) {} \
87 PreservedAnalyses run(MachineFunction &, \
88 MachineFunctionAnalysisManager &) { \
89 return PreservedAnalyses::all(); \
90 } \
91 static AnalysisKey Key; \
92 };
93 #include "MachinePassRegistry.def"
94
95 /// This class provides access to building LLVM's passes.
96 ///
97 /// Its members provide the baseline state available to passes during their
98 /// construction. The \c MachinePassRegistry.def file specifies how to construct
99 /// all of the built-in passes, and those may reference these members during
100 /// construction.
101 template <typename DerivedT> class CodeGenPassBuilder {
102 public:
CodeGenPassBuilder(LLVMTargetMachine & TM,CGPassBuilderOption Opts,PassInstrumentationCallbacks * PIC)103 explicit CodeGenPassBuilder(LLVMTargetMachine &TM, CGPassBuilderOption Opts,
104 PassInstrumentationCallbacks *PIC)
105 : TM(TM), Opt(Opts), PIC(PIC) {
106 // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve
107 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID)
108
109 // Target should override TM.Options.EnableIPRA in their target-specific
110 // LLVMTM ctor. See TargetMachine::setGlobalISel for example.
111 if (Opt.EnableIPRA)
112 TM.Options.EnableIPRA = *Opt.EnableIPRA;
113
114 if (Opt.EnableGlobalISelAbort)
115 TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort;
116
117 if (!Opt.OptimizeRegAlloc)
118 Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOpt::None;
119 }
120
121 Error buildPipeline(ModulePassManager &MPM, MachineFunctionPassManager &MFPM,
122 raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
123 CodeGenFileType FileType) const;
124
125 void registerModuleAnalyses(ModuleAnalysisManager &) const;
126 void registerFunctionAnalyses(FunctionAnalysisManager &) const;
127 void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const;
128 std::pair<StringRef, bool> getPassNameFromLegacyName(StringRef) const;
129
registerAnalyses(MachineFunctionAnalysisManager & MFAM)130 void registerAnalyses(MachineFunctionAnalysisManager &MFAM) const {
131 registerModuleAnalyses(*MFAM.MAM);
132 registerFunctionAnalyses(*MFAM.FAM);
133 registerMachineFunctionAnalyses(MFAM);
134 }
135
getPassInstrumentationCallbacks()136 PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const {
137 return PIC;
138 }
139
140 protected:
141 template <typename PassT> using has_key_t = decltype(PassT::Key);
142
143 template <typename PassT>
144 using is_module_pass_t = decltype(std::declval<PassT &>().run(
145 std::declval<Module &>(), std::declval<ModuleAnalysisManager &>()));
146
147 template <typename PassT>
148 using is_function_pass_t = decltype(std::declval<PassT &>().run(
149 std::declval<Function &>(), std::declval<FunctionAnalysisManager &>()));
150
151 // Function object to maintain state while adding codegen IR passes.
152 class AddIRPass {
153 public:
154 AddIRPass(ModulePassManager &MPM, bool DebugPM, bool Check = true)
MPM(MPM)155 : MPM(MPM) {
156 if (Check)
157 AddingFunctionPasses = false;
158 }
~AddIRPass()159 ~AddIRPass() {
160 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
161 }
162
163 // Add Function Pass
164 template <typename PassT>
165 std::enable_if_t<is_detected<is_function_pass_t, PassT>::value>
operator()166 operator()(PassT &&Pass) {
167 if (AddingFunctionPasses && !*AddingFunctionPasses)
168 AddingFunctionPasses = true;
169 FPM.addPass(std::forward<PassT>(Pass));
170 }
171
172 // Add Module Pass
173 template <typename PassT>
174 std::enable_if_t<is_detected<is_module_pass_t, PassT>::value &&
175 !is_detected<is_function_pass_t, PassT>::value>
operator()176 operator()(PassT &&Pass) {
177 assert((!AddingFunctionPasses || !*AddingFunctionPasses) &&
178 "could not add module pass after adding function pass");
179 MPM.addPass(std::forward<PassT>(Pass));
180 }
181
182 private:
183 ModulePassManager &MPM;
184 FunctionPassManager FPM;
185 // The codegen IR pipeline are mostly function passes with the exceptions of
186 // a few loop and module passes. `AddingFunctionPasses` make sures that
187 // we could only add module passes at the beginning of the pipeline. Once
188 // we begin adding function passes, we could no longer add module passes.
189 // This special-casing introduces less adaptor passes. If we have the need
190 // of adding module passes after function passes, we could change the
191 // implementation to accommodate that.
192 std::optional<bool> AddingFunctionPasses;
193 };
194
195 // Function object to maintain state while adding codegen machine passes.
196 class AddMachinePass {
197 public:
AddMachinePass(MachineFunctionPassManager & PM)198 AddMachinePass(MachineFunctionPassManager &PM) : PM(PM) {}
199
operator()200 template <typename PassT> void operator()(PassT &&Pass) {
201 static_assert(
202 is_detected<has_key_t, PassT>::value,
203 "Machine function pass must define a static member variable `Key`.");
204 for (auto &C : BeforeCallbacks)
205 if (!C(&PassT::Key))
206 return;
207 PM.addPass(std::forward<PassT>(Pass));
208 for (auto &C : AfterCallbacks)
209 C(&PassT::Key);
210 }
211
insertPass(AnalysisKey * ID,PassT Pass)212 template <typename PassT> void insertPass(AnalysisKey *ID, PassT Pass) {
213 AfterCallbacks.emplace_back(
214 [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) {
215 if (PassID == ID)
216 this->PM.addPass(std::move(Pass));
217 });
218 }
219
disablePass(AnalysisKey * ID)220 void disablePass(AnalysisKey *ID) {
221 BeforeCallbacks.emplace_back(
222 [ID](AnalysisKey *PassID) { return PassID != ID; });
223 }
224
releasePM()225 MachineFunctionPassManager releasePM() { return std::move(PM); }
226
227 private:
228 MachineFunctionPassManager &PM;
229 SmallVector<llvm::unique_function<bool(AnalysisKey *)>, 4> BeforeCallbacks;
230 SmallVector<llvm::unique_function<void(AnalysisKey *)>, 4> AfterCallbacks;
231 };
232
233 LLVMTargetMachine &TM;
234 CGPassBuilderOption Opt;
235 PassInstrumentationCallbacks *PIC;
236
237 /// Target override these hooks to parse target-specific analyses.
registerTargetAnalysis(ModuleAnalysisManager &)238 void registerTargetAnalysis(ModuleAnalysisManager &) const {}
registerTargetAnalysis(FunctionAnalysisManager &)239 void registerTargetAnalysis(FunctionAnalysisManager &) const {}
registerTargetAnalysis(MachineFunctionAnalysisManager &)240 void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {}
getTargetPassNameFromLegacyName(StringRef)241 std::pair<StringRef, bool> getTargetPassNameFromLegacyName(StringRef) const {
242 return {"", false};
243 }
244
getTM()245 template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); }
getOptLevel()246 CodeGenOpt::Level getOptLevel() const { return TM.getOptLevel(); }
247
248 /// Check whether or not GlobalISel should abort on error.
249 /// When this is disabled, GlobalISel will fall back on SDISel instead of
250 /// erroring out.
isGlobalISelAbortEnabled()251 bool isGlobalISelAbortEnabled() const {
252 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
253 }
254
255 /// Check whether or not a diagnostic should be emitted when GlobalISel
256 /// uses the fallback path. In other words, it will emit a diagnostic
257 /// when GlobalISel failed and isGlobalISelAbortEnabled is false.
reportDiagnosticWhenGlobalISelFallback()258 bool reportDiagnosticWhenGlobalISelFallback() const {
259 return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
260 }
261
262 /// addInstSelector - This method should install an instruction selector pass,
263 /// which converts from LLVM code to machine instructions.
addInstSelector(AddMachinePass &)264 Error addInstSelector(AddMachinePass &) const {
265 return make_error<StringError>("addInstSelector is not overridden",
266 inconvertibleErrorCode());
267 }
268
269 /// Add passes that optimize instruction level parallelism for out-of-order
270 /// targets. These passes are run while the machine code is still in SSA
271 /// form, so they can use MachineTraceMetrics to control their heuristics.
272 ///
273 /// All passes added here should preserve the MachineDominatorTree,
274 /// MachineLoopInfo, and MachineTraceMetrics analyses.
addILPOpts(AddMachinePass &)275 void addILPOpts(AddMachinePass &) const {}
276
277 /// This method may be implemented by targets that want to run passes
278 /// immediately before register allocation.
addPreRegAlloc(AddMachinePass &)279 void addPreRegAlloc(AddMachinePass &) const {}
280
281 /// addPreRewrite - Add passes to the optimized register allocation pipeline
282 /// after register allocation is complete, but before virtual registers are
283 /// rewritten to physical registers.
284 ///
285 /// These passes must preserve VirtRegMap and LiveIntervals, and when running
286 /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
287 /// When these passes run, VirtRegMap contains legal physreg assignments for
288 /// all virtual registers.
289 ///
290 /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not
291 /// be honored. This is also not generally used for the the fast variant,
292 /// where the allocation and rewriting are done in one pass.
addPreRewrite(AddMachinePass &)293 void addPreRewrite(AddMachinePass &) const {}
294
295 /// Add passes to be run immediately after virtual registers are rewritten
296 /// to physical registers.
addPostRewrite(AddMachinePass &)297 void addPostRewrite(AddMachinePass &) const {}
298
299 /// This method may be implemented by targets that want to run passes after
300 /// register allocation pass pipeline but before prolog-epilog insertion.
addPostRegAlloc(AddMachinePass &)301 void addPostRegAlloc(AddMachinePass &) const {}
302
303 /// This method may be implemented by targets that want to run passes after
304 /// prolog-epilog insertion and before the second instruction scheduling pass.
addPreSched2(AddMachinePass &)305 void addPreSched2(AddMachinePass &) const {}
306
307 /// This pass may be implemented by targets that want to run passes
308 /// immediately before machine code is emitted.
addPreEmitPass(AddMachinePass &)309 void addPreEmitPass(AddMachinePass &) const {}
310
311 /// Targets may add passes immediately before machine code is emitted in this
312 /// callback. This is called even later than `addPreEmitPass`.
313 // FIXME: Rename `addPreEmitPass` to something more sensible given its actual
314 // position and remove the `2` suffix here as this callback is what
315 // `addPreEmitPass` *should* be but in reality isn't.
addPreEmitPass2(AddMachinePass &)316 void addPreEmitPass2(AddMachinePass &) const {}
317
318 /// {{@ For GlobalISel
319 ///
320
321 /// addPreISel - This method should add any "last minute" LLVM->LLVM
322 /// passes (which are run just before instruction selector).
addPreISel(AddIRPass &)323 void addPreISel(AddIRPass &) const {
324 llvm_unreachable("addPreISel is not overridden");
325 }
326
327 /// This method should install an IR translator pass, which converts from
328 /// LLVM code to machine instructions with possibly generic opcodes.
addIRTranslator(AddMachinePass &)329 Error addIRTranslator(AddMachinePass &) const {
330 return make_error<StringError>("addIRTranslator is not overridden",
331 inconvertibleErrorCode());
332 }
333
334 /// This method may be implemented by targets that want to run passes
335 /// immediately before legalization.
addPreLegalizeMachineIR(AddMachinePass &)336 void addPreLegalizeMachineIR(AddMachinePass &) const {}
337
338 /// This method should install a legalize pass, which converts the instruction
339 /// sequence into one that can be selected by the target.
addLegalizeMachineIR(AddMachinePass &)340 Error addLegalizeMachineIR(AddMachinePass &) const {
341 return make_error<StringError>("addLegalizeMachineIR is not overridden",
342 inconvertibleErrorCode());
343 }
344
345 /// This method may be implemented by targets that want to run passes
346 /// immediately before the register bank selection.
addPreRegBankSelect(AddMachinePass &)347 void addPreRegBankSelect(AddMachinePass &) const {}
348
349 /// This method should install a register bank selector pass, which
350 /// assigns register banks to virtual registers without a register
351 /// class or register banks.
addRegBankSelect(AddMachinePass &)352 Error addRegBankSelect(AddMachinePass &) const {
353 return make_error<StringError>("addRegBankSelect is not overridden",
354 inconvertibleErrorCode());
355 }
356
357 /// This method may be implemented by targets that want to run passes
358 /// immediately before the (global) instruction selection.
addPreGlobalInstructionSelect(AddMachinePass &)359 void addPreGlobalInstructionSelect(AddMachinePass &) const {}
360
361 /// This method should install a (global) instruction selector pass, which
362 /// converts possibly generic instructions to fully target-specific
363 /// instructions, thereby constraining all generic virtual registers to
364 /// register classes.
addGlobalInstructionSelect(AddMachinePass &)365 Error addGlobalInstructionSelect(AddMachinePass &) const {
366 return make_error<StringError>(
367 "addGlobalInstructionSelect is not overridden",
368 inconvertibleErrorCode());
369 }
370 /// @}}
371
372 /// High level function that adds all passes necessary to go from llvm IR
373 /// representation to the MI representation.
374 /// Adds IR based lowering and target specific optimization passes and finally
375 /// the core instruction selection passes.
376 void addISelPasses(AddIRPass &) const;
377
378 /// Add the actual instruction selection passes. This does not include
379 /// preparation passes on IR.
380 Error addCoreISelPasses(AddMachinePass &) const;
381
382 /// Add the complete, standard set of LLVM CodeGen passes.
383 /// Fully developed targets will not generally override this.
384 Error addMachinePasses(AddMachinePass &) const;
385
386 /// Add passes to lower exception handling for the code generator.
387 void addPassesToHandleExceptions(AddIRPass &) const;
388
389 /// Add common target configurable passes that perform LLVM IR to IR
390 /// transforms following machine independent optimization.
391 void addIRPasses(AddIRPass &) const;
392
393 /// Add pass to prepare the LLVM IR for code generation. This should be done
394 /// before exception handling preparation passes.
395 void addCodeGenPrepare(AddIRPass &) const;
396
397 /// Add common passes that perform LLVM IR to IR transforms in preparation for
398 /// instruction selection.
399 void addISelPrepare(AddIRPass &) const;
400
401 /// Methods with trivial inline returns are convenient points in the common
402 /// codegen pass pipeline where targets may insert passes. Methods with
403 /// out-of-line standard implementations are major CodeGen stages called by
404 /// addMachinePasses. Some targets may override major stages when inserting
405 /// passes is insufficient, but maintaining overriden stages is more work.
406 ///
407
408 /// addMachineSSAOptimization - Add standard passes that optimize machine
409 /// instructions in SSA form.
410 void addMachineSSAOptimization(AddMachinePass &) const;
411
412 /// addFastRegAlloc - Add the minimum set of target-independent passes that
413 /// are required for fast register allocation.
414 Error addFastRegAlloc(AddMachinePass &) const;
415
416 /// addOptimizedRegAlloc - Add passes related to register allocation.
417 /// LLVMTargetMachine provides standard regalloc passes for most targets.
418 void addOptimizedRegAlloc(AddMachinePass &) const;
419
420 /// Add passes that optimize machine instructions after register allocation.
421 void addMachineLateOptimization(AddMachinePass &) const;
422
423 /// addGCPasses - Add late codegen passes that analyze code for garbage
424 /// collection. This should return true if GC info should be printed after
425 /// these passes.
addGCPasses(AddMachinePass &)426 void addGCPasses(AddMachinePass &) const {}
427
428 /// Add standard basic block placement passes.
429 void addBlockPlacement(AddMachinePass &) const;
430
431 using CreateMCStreamer =
432 std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>;
addAsmPrinter(AddMachinePass &,CreateMCStreamer)433 void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const {
434 llvm_unreachable("addAsmPrinter is not overridden");
435 }
436
437 /// Utilities for targets to add passes to the pass manager.
438 ///
439
440 /// createTargetRegisterAllocator - Create the register allocator pass for
441 /// this target at the current optimization level.
442 void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const;
443
444 /// addMachinePasses helper to create the target-selected or overriden
445 /// regalloc pass.
446 void addRegAllocPass(AddMachinePass &, bool Optimized) const;
447
448 /// Add core register alloator passes which do the actual register assignment
449 /// and rewriting. \returns true if any passes were added.
450 Error addRegAssignmentFast(AddMachinePass &) const;
451 Error addRegAssignmentOptimized(AddMachinePass &) const;
452
453 private:
derived()454 DerivedT &derived() { return static_cast<DerivedT &>(*this); }
derived()455 const DerivedT &derived() const {
456 return static_cast<const DerivedT &>(*this);
457 }
458 };
459
460 template <typename Derived>
buildPipeline(ModulePassManager & MPM,MachineFunctionPassManager & MFPM,raw_pwrite_stream & Out,raw_pwrite_stream * DwoOut,CodeGenFileType FileType)461 Error CodeGenPassBuilder<Derived>::buildPipeline(
462 ModulePassManager &MPM, MachineFunctionPassManager &MFPM,
463 raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
464 CodeGenFileType FileType) const {
465 AddIRPass addIRPass(MPM, Opt.DebugPM);
466 addISelPasses(addIRPass);
467
468 AddMachinePass addPass(MFPM);
469 if (auto Err = addCoreISelPasses(addPass))
470 return std::move(Err);
471
472 if (auto Err = derived().addMachinePasses(addPass))
473 return std::move(Err);
474
475 derived().addAsmPrinter(
476 addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) {
477 return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx);
478 });
479
480 addPass(FreeMachineFunctionPass());
481 return Error::success();
482 }
483
registerAAAnalyses()484 static inline AAManager registerAAAnalyses() {
485 AAManager AA;
486
487 // The order in which these are registered determines their priority when
488 // being queried.
489
490 // Basic AliasAnalysis support.
491 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
492 // BasicAliasAnalysis wins if they disagree. This is intended to help
493 // support "obvious" type-punning idioms.
494 AA.registerFunctionAnalysis<TypeBasedAA>();
495 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
496 AA.registerFunctionAnalysis<BasicAA>();
497
498 return AA;
499 }
500
501 template <typename Derived>
registerModuleAnalyses(ModuleAnalysisManager & MAM)502 void CodeGenPassBuilder<Derived>::registerModuleAnalyses(
503 ModuleAnalysisManager &MAM) const {
504 #define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
505 MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
506 #include "MachinePassRegistry.def"
507 derived().registerTargetAnalysis(MAM);
508 }
509
510 template <typename Derived>
registerFunctionAnalyses(FunctionAnalysisManager & FAM)511 void CodeGenPassBuilder<Derived>::registerFunctionAnalyses(
512 FunctionAnalysisManager &FAM) const {
513 FAM.registerPass([this] { return registerAAAnalyses(); });
514
515 #define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
516 FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
517 #include "MachinePassRegistry.def"
518 derived().registerTargetAnalysis(FAM);
519 }
520
521 template <typename Derived>
registerMachineFunctionAnalyses(MachineFunctionAnalysisManager & MFAM)522 void CodeGenPassBuilder<Derived>::registerMachineFunctionAnalyses(
523 MachineFunctionAnalysisManager &MFAM) const {
524 #define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \
525 MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; });
526 #include "MachinePassRegistry.def"
527 derived().registerTargetAnalysis(MFAM);
528 }
529
530 // FIXME: For new PM, use pass name directly in commandline seems good.
531 // Translate stringfied pass name to its old commandline name. Returns the
532 // matching legacy name and a boolean value indicating if the pass is a machine
533 // pass.
534 template <typename Derived>
535 std::pair<StringRef, bool>
getPassNameFromLegacyName(StringRef Name)536 CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {
537 std::pair<StringRef, bool> Ret;
538 if (Name.empty())
539 return Ret;
540
541 #define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
542 if (Name == NAME) \
543 Ret = {#PASS_NAME, false};
544 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
545 if (Name == NAME) \
546 Ret = {#PASS_NAME, false};
547 #define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
548 if (Name == NAME) \
549 Ret = {#PASS_NAME, false};
550 #define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
551 if (Name == NAME) \
552 Ret = {#PASS_NAME, false};
553 #define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
554 if (Name == NAME) \
555 Ret = {#PASS_NAME, true};
556 #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
557 if (Name == NAME) \
558 Ret = {#PASS_NAME, true};
559 #define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
560 if (Name == NAME) \
561 Ret = {#PASS_NAME, true};
562 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \
563 if (Name == NAME) \
564 Ret = {#PASS_NAME, true};
565 #include "llvm/CodeGen/MachinePassRegistry.def"
566
567 if (Ret.first.empty())
568 Ret = derived().getTargetPassNameFromLegacyName(Name);
569
570 if (Ret.first.empty())
571 report_fatal_error(Twine('\"') + Twine(Name) +
572 Twine("\" pass could not be found."));
573
574 return Ret;
575 }
576
577 template <typename Derived>
addISelPasses(AddIRPass & addPass)578 void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
579 if (TM.useEmulatedTLS())
580 addPass(LowerEmuTLSPass());
581
582 addPass(PreISelIntrinsicLoweringPass());
583
584 derived().addIRPasses(addPass);
585 derived().addCodeGenPrepare(addPass);
586 addPassesToHandleExceptions(addPass);
587 derived().addISelPrepare(addPass);
588 }
589
590 /// Add common target configurable passes that perform LLVM IR to IR transforms
591 /// following machine independent optimization.
592 template <typename Derived>
addIRPasses(AddIRPass & addPass)593 void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
594 // Before running any passes, run the verifier to determine if the input
595 // coming from the front-end and/or optimizer is valid.
596 if (!Opt.DisableVerify)
597 addPass(VerifierPass());
598
599 // Run loop strength reduction before anything else.
600 if (getOptLevel() != CodeGenOpt::None && !Opt.DisableLSR) {
601 addPass(createFunctionToLoopPassAdaptor(
602 LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM));
603 // FIXME: use -stop-after so we could remove PrintLSR
604 if (Opt.PrintLSR)
605 addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
606 }
607
608 if (getOptLevel() != CodeGenOpt::None) {
609 // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
610 // loads and compares. ExpandMemCmpPass then tries to expand those calls
611 // into optimally-sized loads and compares. The transforms are enabled by a
612 // target lowering hook.
613 if (!Opt.DisableMergeICmps)
614 addPass(MergeICmpsPass());
615 addPass(ExpandMemCmpPass());
616 }
617
618 // Run GC lowering passes for builtin collectors
619 // TODO: add a pass insertion point here
620 addPass(GCLoweringPass());
621 addPass(ShadowStackGCLoweringPass());
622 addPass(LowerConstantIntrinsicsPass());
623
624 // Make sure that no unreachable blocks are instruction selected.
625 addPass(UnreachableBlockElimPass());
626
627 // Prepare expensive constants for SelectionDAG.
628 if (getOptLevel() != CodeGenOpt::None && !Opt.DisableConstantHoisting)
629 addPass(ConstantHoistingPass());
630
631 // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector
632 // operands with calls to the corresponding functions in a vector library.
633 if (getOptLevel() != CodeGenOpt::None)
634 addPass(ReplaceWithVeclib());
635
636 if (getOptLevel() != CodeGenOpt::None && !Opt.DisablePartialLibcallInlining)
637 addPass(PartiallyInlineLibCallsPass());
638
639 // Instrument function entry and exit, e.g. with calls to mcount().
640 addPass(EntryExitInstrumenterPass(/*PostInlining=*/true));
641
642 // Add scalarization of target's unsupported masked memory intrinsics pass.
643 // the unsupported intrinsic will be replaced with a chain of basic blocks,
644 // that stores/loads element one-by-one if the appropriate mask bit is set.
645 addPass(ScalarizeMaskedMemIntrinPass());
646
647 // Expand reduction intrinsics into shuffle sequences if the target wants to.
648 addPass(ExpandReductionsPass());
649
650 // Convert conditional moves to conditional jumps when profitable.
651 if (getOptLevel() != CodeGenOpt::None && !Opt.DisableSelectOptimize)
652 addPass(SelectOptimizePass());
653 }
654
655 /// Turn exception handling constructs into something the code generators can
656 /// handle.
657 template <typename Derived>
addPassesToHandleExceptions(AddIRPass & addPass)658 void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
659 AddIRPass &addPass) const {
660 const MCAsmInfo *MCAI = TM.getMCAsmInfo();
661 assert(MCAI && "No MCAsmInfo");
662 switch (MCAI->getExceptionHandlingType()) {
663 case ExceptionHandling::SjLj:
664 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
665 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
666 // catch info can get misplaced when a selector ends up more than one block
667 // removed from the parent invoke(s). This could happen when a landing
668 // pad is shared by multiple invokes and is also a target of a normal
669 // edge from elsewhere.
670 addPass(SjLjEHPreparePass());
671 [[fallthrough]];
672 case ExceptionHandling::DwarfCFI:
673 case ExceptionHandling::ARM:
674 case ExceptionHandling::AIX:
675 addPass(DwarfEHPass(getOptLevel()));
676 break;
677 case ExceptionHandling::WinEH:
678 // We support using both GCC-style and MSVC-style exceptions on Windows, so
679 // add both preparation passes. Each pass will only actually run if it
680 // recognizes the personality function.
681 addPass(WinEHPass());
682 addPass(DwarfEHPass(getOptLevel()));
683 break;
684 case ExceptionHandling::Wasm:
685 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
686 // on catchpads and cleanuppads because it does not outline them into
687 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
688 // should remove PHIs there.
689 addPass(WinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
690 addPass(WasmEHPass());
691 break;
692 case ExceptionHandling::None:
693 addPass(LowerInvokePass());
694
695 // The lower invoke pass may create unreachable code. Remove it.
696 addPass(UnreachableBlockElimPass());
697 break;
698 }
699 }
700
701 /// Add pass to prepare the LLVM IR for code generation. This should be done
702 /// before exception handling preparation passes.
703 template <typename Derived>
addCodeGenPrepare(AddIRPass & addPass)704 void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
705 if (getOptLevel() != CodeGenOpt::None && !Opt.DisableCGP)
706 addPass(CodeGenPreparePass());
707 // TODO: Default ctor'd RewriteSymbolPass is no-op.
708 // addPass(RewriteSymbolPass());
709 }
710
711 /// Add common passes that perform LLVM IR to IR transforms in preparation for
712 /// instruction selection.
713 template <typename Derived>
addISelPrepare(AddIRPass & addPass)714 void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
715 derived().addPreISel(addPass);
716
717 // Add both the safe stack and the stack protection passes: each of them will
718 // only protect functions that have corresponding attributes.
719 addPass(SafeStackPass());
720 addPass(StackProtectorPass());
721
722 if (Opt.PrintISelInput)
723 addPass(PrintFunctionPass(dbgs(),
724 "\n\n*** Final LLVM Code input to ISel ***\n"));
725
726 // All passes which modify the LLVM IR are now complete; run the verifier
727 // to ensure that the IR is valid.
728 if (!Opt.DisableVerify)
729 addPass(VerifierPass());
730 }
731
732 template <typename Derived>
addCoreISelPasses(AddMachinePass & addPass)733 Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
734 AddMachinePass &addPass) const {
735 // Enable FastISel with -fast-isel, but allow that to be overridden.
736 TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true));
737
738 // Determine an instruction selector.
739 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
740 SelectorType Selector;
741
742 if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true)
743 Selector = SelectorType::FastISel;
744 else if ((Opt.EnableGlobalISelOption &&
745 *Opt.EnableGlobalISelOption == true) ||
746 (TM.Options.EnableGlobalISel &&
747 (!Opt.EnableGlobalISelOption ||
748 *Opt.EnableGlobalISelOption == false)))
749 Selector = SelectorType::GlobalISel;
750 else if (TM.getOptLevel() == CodeGenOpt::None && TM.getO0WantsFastISel())
751 Selector = SelectorType::FastISel;
752 else
753 Selector = SelectorType::SelectionDAG;
754
755 // Set consistently TM.Options.EnableFastISel and EnableGlobalISel.
756 if (Selector == SelectorType::FastISel) {
757 TM.setFastISel(true);
758 TM.setGlobalISel(false);
759 } else if (Selector == SelectorType::GlobalISel) {
760 TM.setFastISel(false);
761 TM.setGlobalISel(true);
762 }
763
764 // Add instruction selector passes.
765 if (Selector == SelectorType::GlobalISel) {
766 if (auto Err = derived().addIRTranslator(addPass))
767 return std::move(Err);
768
769 derived().addPreLegalizeMachineIR(addPass);
770
771 if (auto Err = derived().addLegalizeMachineIR(addPass))
772 return std::move(Err);
773
774 // Before running the register bank selector, ask the target if it
775 // wants to run some passes.
776 derived().addPreRegBankSelect(addPass);
777
778 if (auto Err = derived().addRegBankSelect(addPass))
779 return std::move(Err);
780
781 derived().addPreGlobalInstructionSelect(addPass);
782
783 if (auto Err = derived().addGlobalInstructionSelect(addPass))
784 return std::move(Err);
785
786 // Pass to reset the MachineFunction if the ISel failed.
787 addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(),
788 isGlobalISelAbortEnabled()));
789
790 // Provide a fallback path when we do not want to abort on
791 // not-yet-supported input.
792 if (!isGlobalISelAbortEnabled())
793 if (auto Err = derived().addInstSelector(addPass))
794 return std::move(Err);
795
796 } else if (auto Err = derived().addInstSelector(addPass))
797 return std::move(Err);
798
799 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
800 // FinalizeISel.
801 addPass(FinalizeISelPass());
802
803 // // Print the instruction selected machine code...
804 // printAndVerify("After Instruction Selection");
805
806 return Error::success();
807 }
808
809 /// Add the complete set of target-independent postISel code generator passes.
810 ///
811 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
812 /// with nontrivial configuration or multiple passes are broken out below in
813 /// add%Stage routines.
814 ///
815 /// Any CodeGenPassBuilder<Derived>::addXX routine may be overriden by the
816 /// Target. The addPre/Post methods with empty header implementations allow
817 /// injecting target-specific fixups just before or after major stages.
818 /// Additionally, targets have the flexibility to change pass order within a
819 /// stage by overriding default implementation of add%Stage routines below. Each
820 /// technique has maintainability tradeoffs because alternate pass orders are
821 /// not well supported. addPre/Post works better if the target pass is easily
822 /// tied to a common pass. But if it has subtle dependencies on multiple passes,
823 /// the target should override the stage instead.
824 template <typename Derived>
addMachinePasses(AddMachinePass & addPass)825 Error CodeGenPassBuilder<Derived>::addMachinePasses(
826 AddMachinePass &addPass) const {
827 // Add passes that optimize machine instructions in SSA form.
828 if (getOptLevel() != CodeGenOpt::None) {
829 derived().addMachineSSAOptimization(addPass);
830 } else {
831 // If the target requests it, assign local variables to stack slots relative
832 // to one another and simplify frame index references where possible.
833 addPass(LocalStackSlotPass());
834 }
835
836 if (TM.Options.EnableIPRA)
837 addPass(RegUsageInfoPropagationPass());
838
839 // Run pre-ra passes.
840 derived().addPreRegAlloc(addPass);
841
842 // Run register allocation and passes that are tightly coupled with it,
843 // including phi elimination and scheduling.
844 if (*Opt.OptimizeRegAlloc) {
845 derived().addOptimizedRegAlloc(addPass);
846 } else {
847 if (auto Err = derived().addFastRegAlloc(addPass))
848 return Err;
849 }
850
851 // Run post-ra passes.
852 derived().addPostRegAlloc(addPass);
853
854 addPass(RemoveRedundantDebugValuesPass());
855
856 // Insert prolog/epilog code. Eliminate abstract frame index references...
857 if (getOptLevel() != CodeGenOpt::None) {
858 addPass(PostRAMachineSinkingPass());
859 addPass(ShrinkWrapPass());
860 }
861
862 addPass(PrologEpilogInserterPass());
863
864 /// Add passes that optimize machine instructions after register allocation.
865 if (getOptLevel() != CodeGenOpt::None)
866 derived().addMachineLateOptimization(addPass);
867
868 // Expand pseudo instructions before second scheduling pass.
869 addPass(ExpandPostRAPseudosPass());
870
871 // Run pre-sched2 passes.
872 derived().addPreSched2(addPass);
873
874 if (Opt.EnableImplicitNullChecks)
875 addPass(ImplicitNullChecksPass());
876
877 // Second pass scheduler.
878 // Let Target optionally insert this pass by itself at some other
879 // point.
880 if (getOptLevel() != CodeGenOpt::None &&
881 !TM.targetSchedulesPostRAScheduling()) {
882 if (Opt.MISchedPostRA)
883 addPass(PostMachineSchedulerPass());
884 else
885 addPass(PostRASchedulerPass());
886 }
887
888 // GC
889 derived().addGCPasses(addPass);
890
891 // Basic block placement.
892 if (getOptLevel() != CodeGenOpt::None)
893 derived().addBlockPlacement(addPass);
894
895 // Insert before XRay Instrumentation.
896 addPass(FEntryInserterPass());
897
898 addPass(XRayInstrumentationPass());
899 addPass(PatchableFunctionPass());
900
901 derived().addPreEmitPass(addPass);
902
903 if (TM.Options.EnableIPRA)
904 // Collect register usage information and produce a register mask of
905 // clobbered registers, to be used to optimize call sites.
906 addPass(RegUsageInfoCollectorPass());
907
908 addPass(FuncletLayoutPass());
909
910 addPass(StackMapLivenessPass());
911 addPass(LiveDebugValuesPass());
912 addPass(MachineSanitizerBinaryMetadata());
913
914 if (TM.Options.EnableMachineOutliner && getOptLevel() != CodeGenOpt::None &&
915 Opt.EnableMachineOutliner != RunOutliner::NeverOutline) {
916 bool RunOnAllFunctions =
917 (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline);
918 bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining;
919 if (AddOutliner)
920 addPass(MachineOutlinerPass(RunOnAllFunctions));
921 }
922
923 // Add passes that directly emit MI after all other MI passes.
924 derived().addPreEmitPass2(addPass);
925
926 return Error::success();
927 }
928
929 /// Add passes that optimize machine instructions in SSA form.
930 template <typename Derived>
addMachineSSAOptimization(AddMachinePass & addPass)931 void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
932 AddMachinePass &addPass) const {
933 // Pre-ra tail duplication.
934 addPass(EarlyTailDuplicatePass());
935
936 // Optimize PHIs before DCE: removing dead PHI cycles may make more
937 // instructions dead.
938 addPass(OptimizePHIsPass());
939
940 // This pass merges large allocas. StackSlotColoring is a different pass
941 // which merges spill slots.
942 addPass(StackColoringPass());
943
944 // If the target requests it, assign local variables to stack slots relative
945 // to one another and simplify frame index references where possible.
946 addPass(LocalStackSlotPass());
947
948 // With optimization, dead code should already be eliminated. However
949 // there is one known exception: lowered code for arguments that are only
950 // used by tail calls, where the tail calls reuse the incoming stack
951 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
952 addPass(DeadMachineInstructionElimPass());
953
954 // Allow targets to insert passes that improve instruction level parallelism,
955 // like if-conversion. Such passes will typically need dominator trees and
956 // loop info, just like LICM and CSE below.
957 derived().addILPOpts(addPass);
958
959 addPass(EarlyMachineLICMPass());
960 addPass(MachineCSEPass());
961
962 addPass(MachineSinkingPass());
963
964 addPass(PeepholeOptimizerPass());
965 // Clean-up the dead code that may have been generated by peephole
966 // rewriting.
967 addPass(DeadMachineInstructionElimPass());
968 }
969
970 //===---------------------------------------------------------------------===//
971 /// Register Allocation Pass Configuration
972 //===---------------------------------------------------------------------===//
973
974 /// Instantiate the default register allocator pass for this target for either
975 /// the optimized or unoptimized allocation path. This will be added to the pass
976 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
977 /// in the optimized case.
978 ///
979 /// A target that uses the standard regalloc pass order for fast or optimized
980 /// allocation may still override this for per-target regalloc
981 /// selection. But -regalloc=... always takes precedence.
982 template <typename Derived>
addTargetRegisterAllocator(AddMachinePass & addPass,bool Optimized)983 void CodeGenPassBuilder<Derived>::addTargetRegisterAllocator(
984 AddMachinePass &addPass, bool Optimized) const {
985 if (Optimized)
986 addPass(RAGreedyPass());
987 else
988 addPass(RAFastPass());
989 }
990
991 /// Find and instantiate the register allocation pass requested by this target
992 /// at the current optimization level. Different register allocators are
993 /// defined as separate passes because they may require different analysis.
994 template <typename Derived>
addRegAllocPass(AddMachinePass & addPass,bool Optimized)995 void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
996 bool Optimized) const {
997 if (Opt.RegAlloc == RegAllocType::Default)
998 // With no -regalloc= override, ask the target for a regalloc pass.
999 derived().addTargetRegisterAllocator(addPass, Optimized);
1000 else if (Opt.RegAlloc == RegAllocType::Basic)
1001 addPass(RABasicPass());
1002 else if (Opt.RegAlloc == RegAllocType::Fast)
1003 addPass(RAFastPass());
1004 else if (Opt.RegAlloc == RegAllocType::Greedy)
1005 addPass(RAGreedyPass());
1006 else if (Opt.RegAlloc == RegAllocType::PBQP)
1007 addPass(RAPBQPPass());
1008 else
1009 llvm_unreachable("unknonwn register allocator type");
1010 }
1011
1012 template <typename Derived>
addRegAssignmentFast(AddMachinePass & addPass)1013 Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
1014 AddMachinePass &addPass) const {
1015 if (Opt.RegAlloc != RegAllocType::Default &&
1016 Opt.RegAlloc != RegAllocType::Fast)
1017 return make_error<StringError>(
1018 "Must use fast (default) register allocator for unoptimized regalloc.",
1019 inconvertibleErrorCode());
1020
1021 addRegAllocPass(addPass, false);
1022 return Error::success();
1023 }
1024
1025 template <typename Derived>
addRegAssignmentOptimized(AddMachinePass & addPass)1026 Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
1027 AddMachinePass &addPass) const {
1028 // Add the selected register allocation pass.
1029 addRegAllocPass(addPass, true);
1030
1031 // Allow targets to change the register assignments before rewriting.
1032 derived().addPreRewrite(addPass);
1033
1034 // Finally rewrite virtual registers.
1035 addPass(VirtRegRewriterPass());
1036 // Perform stack slot coloring and post-ra machine LICM.
1037 //
1038 // FIXME: Re-enable coloring with register when it's capable of adding
1039 // kill markers.
1040 addPass(StackSlotColoringPass());
1041
1042 return Error::success();
1043 }
1044
1045 /// Add the minimum set of target-independent passes that are required for
1046 /// register allocation. No coalescing or scheduling.
1047 template <typename Derived>
addFastRegAlloc(AddMachinePass & addPass)1048 Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
1049 AddMachinePass &addPass) const {
1050 addPass(PHIEliminationPass());
1051 addPass(TwoAddressInstructionPass());
1052 return derived().addRegAssignmentFast(addPass);
1053 }
1054
1055 /// Add standard target-independent passes that are tightly coupled with
1056 /// optimized register allocation, including coalescing, machine instruction
1057 /// scheduling, and register allocation itself.
1058 template <typename Derived>
addOptimizedRegAlloc(AddMachinePass & addPass)1059 void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
1060 AddMachinePass &addPass) const {
1061 addPass(DetectDeadLanesPass());
1062
1063 addPass(ProcessImplicitDefsPass());
1064
1065 // Edge splitting is smarter with machine loop info.
1066 addPass(PHIEliminationPass());
1067
1068 // Eventually, we want to run LiveIntervals before PHI elimination.
1069 if (Opt.EarlyLiveIntervals)
1070 addPass(LiveIntervalsPass());
1071
1072 addPass(TwoAddressInstructionPass());
1073 addPass(RegisterCoalescerPass());
1074
1075 // The machine scheduler may accidentally create disconnected components
1076 // when moving subregister definitions around, avoid this by splitting them to
1077 // separate vregs before. Splitting can also improve reg. allocation quality.
1078 addPass(RenameIndependentSubregsPass());
1079
1080 // PreRA instruction scheduling.
1081 addPass(MachineSchedulerPass());
1082
1083 if (derived().addRegAssignmentOptimized(addPass)) {
1084 // Allow targets to expand pseudo instructions depending on the choice of
1085 // registers before MachineCopyPropagation.
1086 derived().addPostRewrite(addPass);
1087
1088 // Copy propagate to forward register uses and try to eliminate COPYs that
1089 // were not coalesced.
1090 addPass(MachineCopyPropagationPass());
1091
1092 // Run post-ra machine LICM to hoist reloads / remats.
1093 //
1094 // FIXME: can this move into MachineLateOptimization?
1095 addPass(MachineLICMPass());
1096 }
1097 }
1098
1099 //===---------------------------------------------------------------------===//
1100 /// Post RegAlloc Pass Configuration
1101 //===---------------------------------------------------------------------===//
1102
1103 /// Add passes that optimize machine instructions after register allocation.
1104 template <typename Derived>
addMachineLateOptimization(AddMachinePass & addPass)1105 void CodeGenPassBuilder<Derived>::addMachineLateOptimization(
1106 AddMachinePass &addPass) const {
1107 // Branch folding must be run after regalloc and prolog/epilog insertion.
1108 addPass(BranchFolderPass());
1109
1110 // Tail duplication.
1111 // Note that duplicating tail just increases code size and degrades
1112 // performance for targets that require Structured Control Flow.
1113 // In addition it can also make CFG irreducible. Thus we disable it.
1114 if (!TM.requiresStructuredCFG())
1115 addPass(TailDuplicatePass());
1116
1117 // Cleanup of redundant (identical) address/immediate loads.
1118 addPass(MachineLateInstrsCleanupPass());
1119
1120 // Copy propagation.
1121 addPass(MachineCopyPropagationPass());
1122 }
1123
1124 /// Add standard basic block placement passes.
1125 template <typename Derived>
addBlockPlacement(AddMachinePass & addPass)1126 void CodeGenPassBuilder<Derived>::addBlockPlacement(
1127 AddMachinePass &addPass) const {
1128 addPass(MachineBlockPlacementPass());
1129 // Run a separate pass to collect block placement statistics.
1130 if (Opt.EnableBlockPlacementStats)
1131 addPass(MachineBlockPlacementStatsPass());
1132 }
1133
1134 } // namespace llvm
1135
1136 #endif // LLVM_CODEGEN_CODEGENPASSBUILDER_H
1137