xref: /llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp (revision 0cbccb13d6757b0ea7f2a7f29bb598e1935bcf37) 
1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVTargetObjectFile.h"
18 #include "RISCVTargetTransformInfo.h"
19 #include "TargetInfo/RISCVTargetInfo.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/Analysis/TargetTransformInfo.h"
22 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
23 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
24 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
25 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
26 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
27 #include "llvm/CodeGen/MIRParser/MIParser.h"
28 #include "llvm/CodeGen/MIRYamlMapping.h"
29 #include "llvm/CodeGen/MachineScheduler.h"
30 #include "llvm/CodeGen/MacroFusion.h"
31 #include "llvm/CodeGen/Passes.h"
32 #include "llvm/CodeGen/RegAllocRegistry.h"
33 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
34 #include "llvm/CodeGen/TargetPassConfig.h"
35 #include "llvm/InitializePasses.h"
36 #include "llvm/MC/TargetRegistry.h"
37 #include "llvm/Passes/PassBuilder.h"
38 #include "llvm/Support/FormattedStream.h"
39 #include "llvm/Target/TargetOptions.h"
40 #include "llvm/Transforms/IPO.h"
41 #include "llvm/Transforms/Scalar.h"
42 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43 #include <optional>
44 using namespace llvm;
45 
46 static cl::opt<bool> EnableRedundantCopyElimination(
47     "riscv-enable-copyelim",
48     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
49     cl::Hidden);
50 
51 // FIXME: Unify control over GlobalMerge.
52 static cl::opt<cl::boolOrDefault>
53     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
54                       cl::desc("Enable the global merge pass"));
55 
56 static cl::opt<bool>
57     EnableMachineCombiner("riscv-enable-machine-combiner",
58                           cl::desc("Enable the machine combiner pass"),
59                           cl::init(true), cl::Hidden);
60 
61 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
62     "riscv-v-vector-bits-max",
63     cl::desc("Assume V extension vector registers are at most this big, "
64              "with zero meaning no maximum size is assumed."),
65     cl::init(0), cl::Hidden);
66 
67 static cl::opt<int> RVVVectorBitsMinOpt(
68     "riscv-v-vector-bits-min",
69     cl::desc("Assume V extension vector registers are at least this big, "
70              "with zero meaning no minimum size is assumed. A value of -1 "
71              "means use Zvl*b extension. This is primarily used to enable "
72              "autovectorization with fixed width vectors."),
73     cl::init(-1), cl::Hidden);
74 
75 static cl::opt<bool> EnableRISCVCopyPropagation(
76     "riscv-enable-copy-propagation",
77     cl::desc("Enable the copy propagation with RISC-V copy instr"),
78     cl::init(true), cl::Hidden);
79 
80 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
81     "riscv-enable-dead-defs", cl::Hidden,
82     cl::desc("Enable the pass that removes dead"
83              " definitons and replaces stores to"
84              " them with stores to x0"),
85     cl::init(true));
86 
87 static cl::opt<bool>
88     EnableSinkFold("riscv-enable-sink-fold",
89                    cl::desc("Enable sinking and folding of instruction copies"),
90                    cl::init(true), cl::Hidden);
91 
92 static cl::opt<bool>
93     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
94                            cl::desc("Enable the loop data prefetch pass"),
95                            cl::init(true));
96 
97 static cl::opt<bool> EnableMISchedLoadStoreClustering(
98     "riscv-misched-load-store-clustering", cl::Hidden,
99     cl::desc("Enable load and store clustering in the machine scheduler"),
100     cl::init(true));
101 
102 static cl::opt<bool>
103     EnableVLOptimizer("riscv-enable-vl-optimizer",
104                       cl::desc("Enable the RISC-V VL Optimizer pass"),
105                       cl::init(false), cl::Hidden);
106 
107 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
108   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
109   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
110   auto *PR = PassRegistry::getPassRegistry();
111   initializeGlobalISel(*PR);
112   initializeRISCVO0PreLegalizerCombinerPass(*PR);
113   initializeRISCVPreLegalizerCombinerPass(*PR);
114   initializeRISCVPostLegalizerCombinerPass(*PR);
115   initializeKCFIPass(*PR);
116   initializeRISCVDeadRegisterDefinitionsPass(*PR);
117   initializeRISCVMakeCompressibleOptPass(*PR);
118   initializeRISCVGatherScatterLoweringPass(*PR);
119   initializeRISCVCodeGenPreparePass(*PR);
120   initializeRISCVPostRAExpandPseudoPass(*PR);
121   initializeRISCVMergeBaseOffsetOptPass(*PR);
122   initializeRISCVOptWInstrsPass(*PR);
123   initializeRISCVPreRAExpandPseudoPass(*PR);
124   initializeRISCVExpandPseudoPass(*PR);
125   initializeRISCVVectorPeepholePass(*PR);
126   initializeRISCVVLOptimizerPass(*PR);
127   initializeRISCVInsertVSETVLIPass(*PR);
128   initializeRISCVInsertReadWriteCSRPass(*PR);
129   initializeRISCVInsertWriteVXRMPass(*PR);
130   initializeRISCVDAGToDAGISelLegacyPass(*PR);
131   initializeRISCVMoveMergePass(*PR);
132   initializeRISCVPushPopOptPass(*PR);
133 }
134 
135 static StringRef computeDataLayout(const Triple &TT,
136                                    const TargetOptions &Options) {
137   StringRef ABIName = Options.MCOptions.getABIName();
138   if (TT.isArch64Bit()) {
139     if (ABIName == "lp64e")
140       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
141 
142     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
143   }
144   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
145 
146   if (ABIName == "ilp32e")
147     return "e-m:e-p:32:32-i64:64-n32-S32";
148 
149   return "e-m:e-p:32:32-i64:64-n32-S128";
150 }
151 
152 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
153                                            std::optional<Reloc::Model> RM) {
154   return RM.value_or(Reloc::Static);
155 }
156 
157 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
158                                        StringRef CPU, StringRef FS,
159                                        const TargetOptions &Options,
160                                        std::optional<Reloc::Model> RM,
161                                        std::optional<CodeModel::Model> CM,
162                                        CodeGenOptLevel OL, bool JIT)
163     : LLVMTargetMachine(T, computeDataLayout(TT, Options), TT, CPU, FS, Options,
164                         getEffectiveRelocModel(TT, RM),
165                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
166       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
167   initAsmInfo();
168 
169   // RISC-V supports the MachineOutliner.
170   setMachineOutliner(true);
171   setSupportsDefaultOutlining(true);
172 
173   if (TT.isOSFuchsia() && !TT.isArch64Bit())
174     report_fatal_error("Fuchsia is only supported for 64-bit");
175 }
176 
177 const RISCVSubtarget *
178 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
179   Attribute CPUAttr = F.getFnAttribute("target-cpu");
180   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
181   Attribute FSAttr = F.getFnAttribute("target-features");
182 
183   std::string CPU =
184       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
185   std::string TuneCPU =
186       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
187   std::string FS =
188       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
189 
190   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
191   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
192 
193   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
194   if (VScaleRangeAttr.isValid()) {
195     if (!RVVVectorBitsMinOpt.getNumOccurrences())
196       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
197     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
198     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
199       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
200   }
201 
202   if (RVVBitsMin != -1U) {
203     // FIXME: Change to >= 32 when VLEN = 32 is supported.
204     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
205                                 isPowerOf2_32(RVVBitsMin))) &&
206            "V or Zve* extension requires vector length to be in the range of "
207            "64 to 65536 and a power 2!");
208     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
209            "Minimum V extension vector length should not be larger than its "
210            "maximum!");
211   }
212   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
213                               isPowerOf2_32(RVVBitsMax))) &&
214          "V or Zve* extension requires vector length to be in the range of "
215          "64 to 65536 and a power 2!");
216 
217   if (RVVBitsMin != -1U) {
218     if (RVVBitsMax != 0) {
219       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
220       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
221     }
222 
223     RVVBitsMin = llvm::bit_floor(
224         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
225   }
226   RVVBitsMax =
227       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
228 
229   SmallString<512> Key;
230   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
231                            << CPU << TuneCPU << FS;
232   auto &I = SubtargetMap[Key];
233   if (!I) {
234     // This needs to be done before we create a new subtarget since any
235     // creation will depend on the TM and the code generation flags on the
236     // function that reside in TargetOptions.
237     resetTargetOptions(F);
238     auto ABIName = Options.MCOptions.getABIName();
239     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
240             F.getParent()->getModuleFlag("target-abi"))) {
241       auto TargetABI = RISCVABI::getTargetABI(ABIName);
242       if (TargetABI != RISCVABI::ABI_Unknown &&
243           ModuleTargetABI->getString() != ABIName) {
244         report_fatal_error("-target-abi option != target-abi module flag");
245       }
246       ABIName = ModuleTargetABI->getString();
247     }
248     I = std::make_unique<RISCVSubtarget>(
249         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
250   }
251   return I.get();
252 }
253 
254 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
255     BumpPtrAllocator &Allocator, const Function &F,
256     const TargetSubtargetInfo *STI) const {
257   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
258                                                                     F, STI);
259 }
260 
261 TargetTransformInfo
262 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
263   return TargetTransformInfo(RISCVTTIImpl(this, F));
264 }
265 
266 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
267 // for all memory accesses, so it is reasonable to assume that an
268 // implementation has no-op address space casts. If an implementation makes a
269 // change to this, they can override it here.
270 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
271                                              unsigned DstAS) const {
272   return true;
273 }
274 
275 namespace {
276 
277 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
278 public:
279   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
280       : RegisterRegAllocBase(N, D, C) {}
281 };
282 
283 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
284                                const MachineRegisterInfo &MRI,
285                                const Register Reg) {
286   const TargetRegisterClass *RC = MRI.getRegClass(Reg);
287   return RISCVRegisterInfo::isRVVRegClass(RC);
288 }
289 
290 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
291 
292 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
293 
294 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
295 /// This option could designate the rvv register allocator only.
296 /// For example: -riscv-rvv-regalloc=basic
297 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
298                RegisterPassParser<RVVRegisterRegAlloc>>
299     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
300                 cl::init(&useDefaultRegisterAllocator),
301                 cl::desc("Register allocator to use for RVV register."));
302 
303 static void initializeDefaultRVVRegisterAllocatorOnce() {
304   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
305 
306   if (!Ctor) {
307     Ctor = RVVRegAlloc;
308     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
309   }
310 }
311 
312 static FunctionPass *createBasicRVVRegisterAllocator() {
313   return createBasicRegisterAllocator(onlyAllocateRVVReg);
314 }
315 
316 static FunctionPass *createGreedyRVVRegisterAllocator() {
317   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
318 }
319 
320 static FunctionPass *createFastRVVRegisterAllocator() {
321   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
322 }
323 
324 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
325                                                "basic register allocator",
326                                                createBasicRVVRegisterAllocator);
327 static RVVRegisterRegAlloc
328     greedyRegAllocRVVReg("greedy", "greedy register allocator",
329                          createGreedyRVVRegisterAllocator);
330 
331 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
332                                               createFastRVVRegisterAllocator);
333 
334 class RISCVPassConfig : public TargetPassConfig {
335 public:
336   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
337       : TargetPassConfig(TM, PM) {
338     if (TM.getOptLevel() != CodeGenOptLevel::None)
339       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
340     setEnableSinkAndFold(EnableSinkFold);
341     EnableLoopTermFold = true;
342   }
343 
344   RISCVTargetMachine &getRISCVTargetMachine() const {
345     return getTM<RISCVTargetMachine>();
346   }
347 
348   ScheduleDAGInstrs *
349   createMachineScheduler(MachineSchedContext *C) const override {
350     ScheduleDAGMILive *DAG = nullptr;
351     if (EnableMISchedLoadStoreClustering) {
352       DAG = createGenericSchedLive(C);
353       DAG->addMutation(createLoadClusterDAGMutation(
354           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
355       DAG->addMutation(createStoreClusterDAGMutation(
356           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
357     }
358     return DAG;
359   }
360 
361   void addIRPasses() override;
362   bool addPreISel() override;
363   void addCodeGenPrepare() override;
364   bool addInstSelector() override;
365   bool addIRTranslator() override;
366   void addPreLegalizeMachineIR() override;
367   bool addLegalizeMachineIR() override;
368   void addPreRegBankSelect() override;
369   bool addRegBankSelect() override;
370   bool addGlobalInstructionSelect() override;
371   void addPreEmitPass() override;
372   void addPreEmitPass2() override;
373   void addPreSched2() override;
374   void addMachineSSAOptimization() override;
375   FunctionPass *createRVVRegAllocPass(bool Optimized);
376   bool addRegAssignAndRewriteFast() override;
377   bool addRegAssignAndRewriteOptimized() override;
378   void addPreRegAlloc() override;
379   void addPostRegAlloc() override;
380   void addFastRegAlloc() override;
381 
382   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
383 };
384 } // namespace
385 
386 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
387   return new RISCVPassConfig(*this, PM);
388 }
389 
390 std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
391   return getStandardCSEConfigForOpt(TM->getOptLevel());
392 }
393 
394 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
395   // Initialize the global default.
396   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
397                   initializeDefaultRVVRegisterAllocatorOnce);
398 
399   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
400   if (Ctor != useDefaultRegisterAllocator)
401     return Ctor();
402 
403   if (Optimized)
404     return createGreedyRVVRegisterAllocator();
405 
406   return createFastRVVRegisterAllocator();
407 }
408 
409 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
410   addPass(createRVVRegAllocPass(false));
411   addPass(createRISCVInsertVSETVLIPass());
412   if (TM->getOptLevel() != CodeGenOptLevel::None &&
413       EnableRISCVDeadRegisterElimination)
414     addPass(createRISCVDeadRegisterDefinitionsPass());
415   return TargetPassConfig::addRegAssignAndRewriteFast();
416 }
417 
418 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
419   addPass(createRVVRegAllocPass(true));
420   addPass(createVirtRegRewriter(false));
421   addPass(createRISCVInsertVSETVLIPass());
422   if (TM->getOptLevel() != CodeGenOptLevel::None &&
423       EnableRISCVDeadRegisterElimination)
424     addPass(createRISCVDeadRegisterDefinitionsPass());
425   return TargetPassConfig::addRegAssignAndRewriteOptimized();
426 }
427 
428 void RISCVPassConfig::addIRPasses() {
429   addPass(createAtomicExpandLegacyPass());
430   addPass(createRISCVZacasABIFixPass());
431 
432   if (getOptLevel() != CodeGenOptLevel::None) {
433     if (EnableLoopDataPrefetch)
434       addPass(createLoopDataPrefetchPass());
435 
436     addPass(createRISCVGatherScatterLoweringPass());
437     addPass(createInterleavedAccessPass());
438     addPass(createRISCVCodeGenPreparePass());
439   }
440 
441   TargetPassConfig::addIRPasses();
442 }
443 
444 bool RISCVPassConfig::addPreISel() {
445   if (TM->getOptLevel() != CodeGenOptLevel::None) {
446     // Add a barrier before instruction selection so that we will not get
447     // deleted block address after enabling default outlining. See D99707 for
448     // more details.
449     addPass(createBarrierNoopPass());
450   }
451 
452   if (EnableGlobalMerge == cl::BOU_TRUE) {
453     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
454                                   /* OnlyOptimizeForSize */ false,
455                                   /* MergeExternalByDefault */ true));
456   }
457 
458   return false;
459 }
460 
461 void RISCVPassConfig::addCodeGenPrepare() {
462   if (getOptLevel() != CodeGenOptLevel::None)
463     addPass(createTypePromotionLegacyPass());
464   TargetPassConfig::addCodeGenPrepare();
465 }
466 
467 bool RISCVPassConfig::addInstSelector() {
468   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
469 
470   return false;
471 }
472 
473 bool RISCVPassConfig::addIRTranslator() {
474   addPass(new IRTranslator(getOptLevel()));
475   return false;
476 }
477 
478 void RISCVPassConfig::addPreLegalizeMachineIR() {
479   if (getOptLevel() == CodeGenOptLevel::None) {
480     addPass(createRISCVO0PreLegalizerCombiner());
481   } else {
482     addPass(createRISCVPreLegalizerCombiner());
483   }
484 }
485 
486 bool RISCVPassConfig::addLegalizeMachineIR() {
487   addPass(new Legalizer());
488   return false;
489 }
490 
491 void RISCVPassConfig::addPreRegBankSelect() {
492   if (getOptLevel() != CodeGenOptLevel::None)
493     addPass(createRISCVPostLegalizerCombiner());
494 }
495 
496 bool RISCVPassConfig::addRegBankSelect() {
497   addPass(new RegBankSelect());
498   return false;
499 }
500 
501 bool RISCVPassConfig::addGlobalInstructionSelect() {
502   addPass(new InstructionSelect(getOptLevel()));
503   return false;
504 }
505 
506 void RISCVPassConfig::addPreSched2() {
507   addPass(createRISCVPostRAExpandPseudoPass());
508 
509   // Emit KCFI checks for indirect calls.
510   addPass(createKCFIPass());
511 }
512 
513 void RISCVPassConfig::addPreEmitPass() {
514   // TODO: It would potentially be better to schedule copy propagation after
515   // expanding pseudos (in addPreEmitPass2). However, performing copy
516   // propagation after the machine outliner (which runs after addPreEmitPass)
517   // currently leads to incorrect code-gen, where copies to registers within
518   // outlined functions are removed erroneously.
519   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
520       EnableRISCVCopyPropagation)
521     addPass(createMachineCopyPropagationPass(true));
522   addPass(&BranchRelaxationPassID);
523   addPass(createRISCVMakeCompressibleOptPass());
524 }
525 
526 void RISCVPassConfig::addPreEmitPass2() {
527   if (TM->getOptLevel() != CodeGenOptLevel::None) {
528     addPass(createRISCVMoveMergePass());
529     // Schedule PushPop Optimization before expansion of Pseudo instruction,
530     // ensuring return instruction is detected correctly.
531     addPass(createRISCVPushPopOptimizationPass());
532   }
533   addPass(createRISCVIndirectBranchTrackingPass());
534   addPass(createRISCVExpandPseudoPass());
535 
536   // Schedule the expansion of AMOs at the last possible moment, avoiding the
537   // possibility for other passes to break the requirements for forward
538   // progress in the LR/SC block.
539   addPass(createRISCVExpandAtomicPseudoPass());
540 
541   // KCFI indirect call checks are lowered to a bundle.
542   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
543     return MF.getFunction().getParent()->getModuleFlag("kcfi");
544   }));
545 }
546 
547 void RISCVPassConfig::addMachineSSAOptimization() {
548   addPass(createRISCVVectorPeepholePass());
549 
550   TargetPassConfig::addMachineSSAOptimization();
551 
552   if (EnableMachineCombiner)
553     addPass(&MachineCombinerID);
554 
555   if (TM->getTargetTriple().isRISCV64()) {
556     addPass(createRISCVOptWInstrsPass());
557   }
558 }
559 
560 void RISCVPassConfig::addPreRegAlloc() {
561   addPass(createRISCVPreRAExpandPseudoPass());
562   if (TM->getOptLevel() != CodeGenOptLevel::None) {
563     addPass(createRISCVMergeBaseOffsetOptPass());
564     if (EnableVLOptimizer)
565       addPass(createRISCVVLOptimizerPass());
566   }
567 
568   addPass(createRISCVInsertReadWriteCSRPass());
569   addPass(createRISCVInsertWriteVXRMPass());
570   addPass(createRISCVLandingPadSetupPass());
571 }
572 
573 void RISCVPassConfig::addFastRegAlloc() {
574   addPass(&InitUndefID);
575   TargetPassConfig::addFastRegAlloc();
576 }
577 
578 
579 void RISCVPassConfig::addPostRegAlloc() {
580   if (TM->getOptLevel() != CodeGenOptLevel::None &&
581       EnableRedundantCopyElimination)
582     addPass(createRISCVRedundantCopyEliminationPass());
583 }
584 
585 void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
586   PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,
587                                                  OptimizationLevel Level) {
588     LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
589   });
590 }
591 
592 yaml::MachineFunctionInfo *
593 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
594   return new yaml::RISCVMachineFunctionInfo();
595 }
596 
597 yaml::MachineFunctionInfo *
598 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
599   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
600   return new yaml::RISCVMachineFunctionInfo(*MFI);
601 }
602 
603 bool RISCVTargetMachine::parseMachineFunctionInfo(
604     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
605     SMDiagnostic &Error, SMRange &SourceRange) const {
606   const auto &YamlMFI =
607       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
608   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
609   return false;
610 }
611