xref: /llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp (revision 69d0bab82689d470e3fd68f50ca8b8d28f3e2294) 
1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVTargetObjectFile.h"
18 #include "RISCVTargetTransformInfo.h"
19 #include "TargetInfo/RISCVTargetInfo.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/Analysis/TargetTransformInfo.h"
22 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
23 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
24 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
25 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
26 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
27 #include "llvm/CodeGen/MIRParser/MIParser.h"
28 #include "llvm/CodeGen/MIRYamlMapping.h"
29 #include "llvm/CodeGen/MachineScheduler.h"
30 #include "llvm/CodeGen/MacroFusion.h"
31 #include "llvm/CodeGen/Passes.h"
32 #include "llvm/CodeGen/RegAllocRegistry.h"
33 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
34 #include "llvm/CodeGen/TargetPassConfig.h"
35 #include "llvm/InitializePasses.h"
36 #include "llvm/MC/TargetRegistry.h"
37 #include "llvm/Passes/PassBuilder.h"
38 #include "llvm/Support/FormattedStream.h"
39 #include "llvm/Target/TargetOptions.h"
40 #include "llvm/Transforms/IPO.h"
41 #include "llvm/Transforms/Scalar.h"
42 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43 #include <optional>
44 using namespace llvm;
45 
46 static cl::opt<bool> EnableRedundantCopyElimination(
47     "riscv-enable-copyelim",
48     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
49     cl::Hidden);
50 
51 // FIXME: Unify control over GlobalMerge.
52 static cl::opt<cl::boolOrDefault>
53     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
54                       cl::desc("Enable the global merge pass"));
55 
56 static cl::opt<bool>
57     EnableMachineCombiner("riscv-enable-machine-combiner",
58                           cl::desc("Enable the machine combiner pass"),
59                           cl::init(true), cl::Hidden);
60 
61 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
62     "riscv-v-vector-bits-max",
63     cl::desc("Assume V extension vector registers are at most this big, "
64              "with zero meaning no maximum size is assumed."),
65     cl::init(0), cl::Hidden);
66 
67 static cl::opt<int> RVVVectorBitsMinOpt(
68     "riscv-v-vector-bits-min",
69     cl::desc("Assume V extension vector registers are at least this big, "
70              "with zero meaning no minimum size is assumed. A value of -1 "
71              "means use Zvl*b extension. This is primarily used to enable "
72              "autovectorization with fixed width vectors."),
73     cl::init(-1), cl::Hidden);
74 
75 static cl::opt<bool> EnableRISCVCopyPropagation(
76     "riscv-enable-copy-propagation",
77     cl::desc("Enable the copy propagation with RISC-V copy instr"),
78     cl::init(true), cl::Hidden);
79 
80 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
81     "riscv-enable-dead-defs", cl::Hidden,
82     cl::desc("Enable the pass that removes dead"
83              " definitons and replaces stores to"
84              " them with stores to x0"),
85     cl::init(true));
86 
87 static cl::opt<bool>
88     EnableSinkFold("riscv-enable-sink-fold",
89                    cl::desc("Enable sinking and folding of instruction copies"),
90                    cl::init(true), cl::Hidden);
91 
92 static cl::opt<bool>
93     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
94                            cl::desc("Enable the loop data prefetch pass"),
95                            cl::init(true));
96 
97 static cl::opt<bool> EnableMISchedLoadStoreClustering(
98     "riscv-misched-load-store-clustering", cl::Hidden,
99     cl::desc("Enable load and store clustering in the machine scheduler"),
100     cl::init(true));
101 
102 static cl::opt<bool> EnablePostMISchedLoadStoreClustering(
103     "riscv-postmisched-load-store-clustering", cl::Hidden,
104     cl::desc("Enable PostRA load and store clustering in the machine scheduler"),
105     cl::init(true));
106 
107 static cl::opt<bool>
108     EnableVLOptimizer("riscv-enable-vl-optimizer",
109                       cl::desc("Enable the RISC-V VL Optimizer pass"),
110                       cl::init(false), cl::Hidden);
111 
112 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
113   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
114   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
115   auto *PR = PassRegistry::getPassRegistry();
116   initializeGlobalISel(*PR);
117   initializeRISCVO0PreLegalizerCombinerPass(*PR);
118   initializeRISCVPreLegalizerCombinerPass(*PR);
119   initializeRISCVPostLegalizerCombinerPass(*PR);
120   initializeKCFIPass(*PR);
121   initializeRISCVDeadRegisterDefinitionsPass(*PR);
122   initializeRISCVMakeCompressibleOptPass(*PR);
123   initializeRISCVGatherScatterLoweringPass(*PR);
124   initializeRISCVCodeGenPreparePass(*PR);
125   initializeRISCVPostRAExpandPseudoPass(*PR);
126   initializeRISCVMergeBaseOffsetOptPass(*PR);
127   initializeRISCVOptWInstrsPass(*PR);
128   initializeRISCVPreRAExpandPseudoPass(*PR);
129   initializeRISCVExpandPseudoPass(*PR);
130   initializeRISCVVectorPeepholePass(*PR);
131   initializeRISCVVLOptimizerPass(*PR);
132   initializeRISCVInsertVSETVLIPass(*PR);
133   initializeRISCVInsertReadWriteCSRPass(*PR);
134   initializeRISCVInsertWriteVXRMPass(*PR);
135   initializeRISCVDAGToDAGISelLegacyPass(*PR);
136   initializeRISCVMoveMergePass(*PR);
137   initializeRISCVPushPopOptPass(*PR);
138 }
139 
140 static StringRef computeDataLayout(const Triple &TT,
141                                    const TargetOptions &Options) {
142   StringRef ABIName = Options.MCOptions.getABIName();
143   if (TT.isArch64Bit()) {
144     if (ABIName == "lp64e")
145       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
146 
147     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
148   }
149   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
150 
151   if (ABIName == "ilp32e")
152     return "e-m:e-p:32:32-i64:64-n32-S32";
153 
154   return "e-m:e-p:32:32-i64:64-n32-S128";
155 }
156 
157 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
158                                            std::optional<Reloc::Model> RM) {
159   return RM.value_or(Reloc::Static);
160 }
161 
162 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
163                                        StringRef CPU, StringRef FS,
164                                        const TargetOptions &Options,
165                                        std::optional<Reloc::Model> RM,
166                                        std::optional<CodeModel::Model> CM,
167                                        CodeGenOptLevel OL, bool JIT)
168     : LLVMTargetMachine(T, computeDataLayout(TT, Options), TT, CPU, FS, Options,
169                         getEffectiveRelocModel(TT, RM),
170                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
171       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
172   initAsmInfo();
173 
174   // RISC-V supports the MachineOutliner.
175   setMachineOutliner(true);
176   setSupportsDefaultOutlining(true);
177 
178   if (TT.isOSFuchsia() && !TT.isArch64Bit())
179     report_fatal_error("Fuchsia is only supported for 64-bit");
180 
181   setCFIFixup(true);
182 }
183 
184 const RISCVSubtarget *
185 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
186   Attribute CPUAttr = F.getFnAttribute("target-cpu");
187   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
188   Attribute FSAttr = F.getFnAttribute("target-features");
189 
190   std::string CPU =
191       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
192   std::string TuneCPU =
193       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
194   std::string FS =
195       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
196 
197   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
198   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
199 
200   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
201   if (VScaleRangeAttr.isValid()) {
202     if (!RVVVectorBitsMinOpt.getNumOccurrences())
203       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
204     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
205     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
206       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
207   }
208 
209   if (RVVBitsMin != -1U) {
210     // FIXME: Change to >= 32 when VLEN = 32 is supported.
211     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
212                                 isPowerOf2_32(RVVBitsMin))) &&
213            "V or Zve* extension requires vector length to be in the range of "
214            "64 to 65536 and a power 2!");
215     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
216            "Minimum V extension vector length should not be larger than its "
217            "maximum!");
218   }
219   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
220                               isPowerOf2_32(RVVBitsMax))) &&
221          "V or Zve* extension requires vector length to be in the range of "
222          "64 to 65536 and a power 2!");
223 
224   if (RVVBitsMin != -1U) {
225     if (RVVBitsMax != 0) {
226       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
227       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
228     }
229 
230     RVVBitsMin = llvm::bit_floor(
231         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
232   }
233   RVVBitsMax =
234       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
235 
236   SmallString<512> Key;
237   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
238                            << CPU << TuneCPU << FS;
239   auto &I = SubtargetMap[Key];
240   if (!I) {
241     // This needs to be done before we create a new subtarget since any
242     // creation will depend on the TM and the code generation flags on the
243     // function that reside in TargetOptions.
244     resetTargetOptions(F);
245     auto ABIName = Options.MCOptions.getABIName();
246     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
247             F.getParent()->getModuleFlag("target-abi"))) {
248       auto TargetABI = RISCVABI::getTargetABI(ABIName);
249       if (TargetABI != RISCVABI::ABI_Unknown &&
250           ModuleTargetABI->getString() != ABIName) {
251         report_fatal_error("-target-abi option != target-abi module flag");
252       }
253       ABIName = ModuleTargetABI->getString();
254     }
255     I = std::make_unique<RISCVSubtarget>(
256         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
257   }
258   return I.get();
259 }
260 
261 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
262     BumpPtrAllocator &Allocator, const Function &F,
263     const TargetSubtargetInfo *STI) const {
264   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
265                                                                     F, STI);
266 }
267 
268 TargetTransformInfo
269 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
270   return TargetTransformInfo(RISCVTTIImpl(this, F));
271 }
272 
273 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
274 // for all memory accesses, so it is reasonable to assume that an
275 // implementation has no-op address space casts. If an implementation makes a
276 // change to this, they can override it here.
277 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
278                                              unsigned DstAS) const {
279   return true;
280 }
281 
282 namespace {
283 
284 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
285 public:
286   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
287       : RegisterRegAllocBase(N, D, C) {}
288 };
289 
290 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
291                                const MachineRegisterInfo &MRI,
292                                const Register Reg) {
293   const TargetRegisterClass *RC = MRI.getRegClass(Reg);
294   return RISCVRegisterInfo::isRVVRegClass(RC);
295 }
296 
297 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
298 
299 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
300 
301 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
302 /// This option could designate the rvv register allocator only.
303 /// For example: -riscv-rvv-regalloc=basic
304 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
305                RegisterPassParser<RVVRegisterRegAlloc>>
306     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
307                 cl::init(&useDefaultRegisterAllocator),
308                 cl::desc("Register allocator to use for RVV register."));
309 
310 static void initializeDefaultRVVRegisterAllocatorOnce() {
311   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
312 
313   if (!Ctor) {
314     Ctor = RVVRegAlloc;
315     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
316   }
317 }
318 
319 static FunctionPass *createBasicRVVRegisterAllocator() {
320   return createBasicRegisterAllocator(onlyAllocateRVVReg);
321 }
322 
323 static FunctionPass *createGreedyRVVRegisterAllocator() {
324   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
325 }
326 
327 static FunctionPass *createFastRVVRegisterAllocator() {
328   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
329 }
330 
331 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
332                                                "basic register allocator",
333                                                createBasicRVVRegisterAllocator);
334 static RVVRegisterRegAlloc
335     greedyRegAllocRVVReg("greedy", "greedy register allocator",
336                          createGreedyRVVRegisterAllocator);
337 
338 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
339                                               createFastRVVRegisterAllocator);
340 
341 class RISCVPassConfig : public TargetPassConfig {
342 public:
343   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
344       : TargetPassConfig(TM, PM) {
345     if (TM.getOptLevel() != CodeGenOptLevel::None)
346       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
347     setEnableSinkAndFold(EnableSinkFold);
348     EnableLoopTermFold = true;
349   }
350 
351   RISCVTargetMachine &getRISCVTargetMachine() const {
352     return getTM<RISCVTargetMachine>();
353   }
354 
355   ScheduleDAGInstrs *
356   createMachineScheduler(MachineSchedContext *C) const override {
357     ScheduleDAGMILive *DAG = nullptr;
358     if (EnableMISchedLoadStoreClustering) {
359       DAG = createGenericSchedLive(C);
360       DAG->addMutation(createLoadClusterDAGMutation(
361           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
362       DAG->addMutation(createStoreClusterDAGMutation(
363           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
364     }
365     return DAG;
366   }
367 
368   ScheduleDAGInstrs *
369   createPostMachineScheduler(MachineSchedContext *C) const override {
370     ScheduleDAGMI *DAG = nullptr;
371     if (EnablePostMISchedLoadStoreClustering) {
372       DAG = createGenericSchedPostRA(C);
373       DAG->addMutation(createLoadClusterDAGMutation(
374           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
375       DAG->addMutation(createStoreClusterDAGMutation(
376           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
377     }
378     return DAG;
379   }
380 
381   void addIRPasses() override;
382   bool addPreISel() override;
383   void addCodeGenPrepare() override;
384   bool addInstSelector() override;
385   bool addIRTranslator() override;
386   void addPreLegalizeMachineIR() override;
387   bool addLegalizeMachineIR() override;
388   void addPreRegBankSelect() override;
389   bool addRegBankSelect() override;
390   bool addGlobalInstructionSelect() override;
391   void addPreEmitPass() override;
392   void addPreEmitPass2() override;
393   void addPreSched2() override;
394   void addMachineSSAOptimization() override;
395   FunctionPass *createRVVRegAllocPass(bool Optimized);
396   bool addRegAssignAndRewriteFast() override;
397   bool addRegAssignAndRewriteOptimized() override;
398   void addPreRegAlloc() override;
399   void addPostRegAlloc() override;
400   void addFastRegAlloc() override;
401 
402   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
403 };
404 } // namespace
405 
406 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
407   return new RISCVPassConfig(*this, PM);
408 }
409 
410 std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
411   return getStandardCSEConfigForOpt(TM->getOptLevel());
412 }
413 
414 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
415   // Initialize the global default.
416   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
417                   initializeDefaultRVVRegisterAllocatorOnce);
418 
419   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
420   if (Ctor != useDefaultRegisterAllocator)
421     return Ctor();
422 
423   if (Optimized)
424     return createGreedyRVVRegisterAllocator();
425 
426   return createFastRVVRegisterAllocator();
427 }
428 
429 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
430   addPass(createRVVRegAllocPass(false));
431   addPass(createRISCVInsertVSETVLIPass());
432   if (TM->getOptLevel() != CodeGenOptLevel::None &&
433       EnableRISCVDeadRegisterElimination)
434     addPass(createRISCVDeadRegisterDefinitionsPass());
435   return TargetPassConfig::addRegAssignAndRewriteFast();
436 }
437 
438 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
439   addPass(createRVVRegAllocPass(true));
440   addPass(createVirtRegRewriter(false));
441   addPass(createRISCVInsertVSETVLIPass());
442   if (TM->getOptLevel() != CodeGenOptLevel::None &&
443       EnableRISCVDeadRegisterElimination)
444     addPass(createRISCVDeadRegisterDefinitionsPass());
445   return TargetPassConfig::addRegAssignAndRewriteOptimized();
446 }
447 
448 void RISCVPassConfig::addIRPasses() {
449   addPass(createAtomicExpandLegacyPass());
450   addPass(createRISCVZacasABIFixPass());
451 
452   if (getOptLevel() != CodeGenOptLevel::None) {
453     if (EnableLoopDataPrefetch)
454       addPass(createLoopDataPrefetchPass());
455 
456     addPass(createRISCVGatherScatterLoweringPass());
457     addPass(createInterleavedAccessPass());
458     addPass(createRISCVCodeGenPreparePass());
459   }
460 
461   TargetPassConfig::addIRPasses();
462 }
463 
464 bool RISCVPassConfig::addPreISel() {
465   if (TM->getOptLevel() != CodeGenOptLevel::None) {
466     // Add a barrier before instruction selection so that we will not get
467     // deleted block address after enabling default outlining. See D99707 for
468     // more details.
469     addPass(createBarrierNoopPass());
470   }
471 
472   if (EnableGlobalMerge == cl::BOU_TRUE) {
473     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
474                                   /* OnlyOptimizeForSize */ false,
475                                   /* MergeExternalByDefault */ true));
476   }
477 
478   return false;
479 }
480 
481 void RISCVPassConfig::addCodeGenPrepare() {
482   if (getOptLevel() != CodeGenOptLevel::None)
483     addPass(createTypePromotionLegacyPass());
484   TargetPassConfig::addCodeGenPrepare();
485 }
486 
487 bool RISCVPassConfig::addInstSelector() {
488   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
489 
490   return false;
491 }
492 
493 bool RISCVPassConfig::addIRTranslator() {
494   addPass(new IRTranslator(getOptLevel()));
495   return false;
496 }
497 
498 void RISCVPassConfig::addPreLegalizeMachineIR() {
499   if (getOptLevel() == CodeGenOptLevel::None) {
500     addPass(createRISCVO0PreLegalizerCombiner());
501   } else {
502     addPass(createRISCVPreLegalizerCombiner());
503   }
504 }
505 
506 bool RISCVPassConfig::addLegalizeMachineIR() {
507   addPass(new Legalizer());
508   return false;
509 }
510 
511 void RISCVPassConfig::addPreRegBankSelect() {
512   if (getOptLevel() != CodeGenOptLevel::None)
513     addPass(createRISCVPostLegalizerCombiner());
514 }
515 
516 bool RISCVPassConfig::addRegBankSelect() {
517   addPass(new RegBankSelect());
518   return false;
519 }
520 
521 bool RISCVPassConfig::addGlobalInstructionSelect() {
522   addPass(new InstructionSelect(getOptLevel()));
523   return false;
524 }
525 
526 void RISCVPassConfig::addPreSched2() {
527   addPass(createRISCVPostRAExpandPseudoPass());
528 
529   // Emit KCFI checks for indirect calls.
530   addPass(createKCFIPass());
531 }
532 
533 void RISCVPassConfig::addPreEmitPass() {
534   // TODO: It would potentially be better to schedule copy propagation after
535   // expanding pseudos (in addPreEmitPass2). However, performing copy
536   // propagation after the machine outliner (which runs after addPreEmitPass)
537   // currently leads to incorrect code-gen, where copies to registers within
538   // outlined functions are removed erroneously.
539   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
540       EnableRISCVCopyPropagation)
541     addPass(createMachineCopyPropagationPass(true));
542   addPass(&BranchRelaxationPassID);
543   addPass(createRISCVMakeCompressibleOptPass());
544 }
545 
546 void RISCVPassConfig::addPreEmitPass2() {
547   if (TM->getOptLevel() != CodeGenOptLevel::None) {
548     addPass(createRISCVMoveMergePass());
549     // Schedule PushPop Optimization before expansion of Pseudo instruction,
550     // ensuring return instruction is detected correctly.
551     addPass(createRISCVPushPopOptimizationPass());
552   }
553   addPass(createRISCVIndirectBranchTrackingPass());
554   addPass(createRISCVExpandPseudoPass());
555 
556   // Schedule the expansion of AMOs at the last possible moment, avoiding the
557   // possibility for other passes to break the requirements for forward
558   // progress in the LR/SC block.
559   addPass(createRISCVExpandAtomicPseudoPass());
560 
561   // KCFI indirect call checks are lowered to a bundle.
562   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
563     return MF.getFunction().getParent()->getModuleFlag("kcfi");
564   }));
565 }
566 
567 void RISCVPassConfig::addMachineSSAOptimization() {
568   addPass(createRISCVVectorPeepholePass());
569 
570   TargetPassConfig::addMachineSSAOptimization();
571 
572   if (EnableMachineCombiner)
573     addPass(&MachineCombinerID);
574 
575   if (TM->getTargetTriple().isRISCV64()) {
576     addPass(createRISCVOptWInstrsPass());
577   }
578 }
579 
580 void RISCVPassConfig::addPreRegAlloc() {
581   addPass(createRISCVPreRAExpandPseudoPass());
582   if (TM->getOptLevel() != CodeGenOptLevel::None) {
583     addPass(createRISCVMergeBaseOffsetOptPass());
584     if (EnableVLOptimizer)
585       addPass(createRISCVVLOptimizerPass());
586   }
587 
588   addPass(createRISCVInsertReadWriteCSRPass());
589   addPass(createRISCVInsertWriteVXRMPass());
590   addPass(createRISCVLandingPadSetupPass());
591 }
592 
593 void RISCVPassConfig::addFastRegAlloc() {
594   addPass(&InitUndefID);
595   TargetPassConfig::addFastRegAlloc();
596 }
597 
598 
599 void RISCVPassConfig::addPostRegAlloc() {
600   if (TM->getOptLevel() != CodeGenOptLevel::None &&
601       EnableRedundantCopyElimination)
602     addPass(createRISCVRedundantCopyEliminationPass());
603 }
604 
605 void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
606   PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,
607                                                  OptimizationLevel Level) {
608     LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
609   });
610 }
611 
612 yaml::MachineFunctionInfo *
613 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
614   return new yaml::RISCVMachineFunctionInfo();
615 }
616 
617 yaml::MachineFunctionInfo *
618 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
619   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
620   return new yaml::RISCVMachineFunctionInfo(*MFI);
621 }
622 
623 bool RISCVTargetMachine::parseMachineFunctionInfo(
624     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
625     SMDiagnostic &Error, SMRange &SourceRange) const {
626   const auto &YamlMFI =
627       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
628   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
629   return false;
630 }
631