xref: /llvm-project/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp (revision ce393beddfb810204ac0618c2fe365d4d21f5c32) 
1 //===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// RISC-V.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCVRegisterBankInfo.h"
16 #include "RISCVSubtarget.h"
17 #include "RISCVTargetMachine.h"
18 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
20 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24 #include "llvm/CodeGen/MachineJumpTableInfo.h"
25 #include "llvm/IR/IntrinsicsRISCV.h"
26 #include "llvm/Support/Debug.h"
27 
28 #define DEBUG_TYPE "riscv-isel"
29 
30 using namespace llvm;
31 using namespace MIPatternMatch;
32 
33 #define GET_GLOBALISEL_PREDICATE_BITSET
34 #include "RISCVGenGlobalISel.inc"
35 #undef GET_GLOBALISEL_PREDICATE_BITSET
36 
37 namespace {
38 
39 class RISCVInstructionSelector : public InstructionSelector {
40 public:
41   RISCVInstructionSelector(const RISCVTargetMachine &TM,
42                            const RISCVSubtarget &STI,
43                            const RISCVRegisterBankInfo &RBI);
44 
45   bool select(MachineInstr &MI) override;
46 
47   void setupMF(MachineFunction &MF, GISelKnownBits *KB,
48                CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49                BlockFrequencyInfo *BFI) override {
50     InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
51     MRI = &MF.getRegInfo();
52   }
53 
54   static const char *getName() { return DEBUG_TYPE; }
55 
56 private:
57   const TargetRegisterClass *
58   getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
59 
60   bool isRegInGprb(Register Reg) const;
61   bool isRegInFprb(Register Reg) const;
62 
63   // tblgen-erated 'select' implementation, used as the initial selector for
64   // the patterns that don't require complex C++.
65   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
66 
67   // A lowering phase that runs before any selection attempts.
68   // Returns true if the instruction was modified.
69   void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
70 
71   bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
72 
73   // Custom selection methods
74   bool selectCopy(MachineInstr &MI) const;
75   bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
76   bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
77   bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
78                   bool IsExternWeak = false) const;
79   bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
80   bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
81   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
82                  MachineIRBuilder &MIB) const;
83   bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
84   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
85 
86   ComplexRendererFns selectShiftMask(MachineOperand &Root,
87                                      unsigned ShiftWidth) const;
88   ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
89     return selectShiftMask(Root, STI.getXLen());
90   }
91   ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
92     return selectShiftMask(Root, 32);
93   }
94   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
95 
96   ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
97   template <unsigned Bits>
98   ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
99     return selectSExtBits(Root, Bits);
100   }
101 
102   ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
103   template <unsigned Bits>
104   ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
105     return selectZExtBits(Root, Bits);
106   }
107 
108   ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
109   template <unsigned ShAmt>
110   ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
111     return selectSHXADDOp(Root, ShAmt);
112   }
113 
114   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
115                                        unsigned ShAmt) const;
116   template <unsigned ShAmt>
117   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
118     return selectSHXADD_UWOp(Root, ShAmt);
119   }
120 
121   ComplexRendererFns renderVLOp(MachineOperand &Root) const;
122 
123   // Custom renderers for tablegen
124   void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
125                     int OpIdx) const;
126   void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
127                             int OpIdx) const;
128   void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
129                           int OpIdx) const;
130   void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
131                       int OpIdx) const;
132   void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
133                  int OpIdx) const;
134   void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
135                         int OpIdx) const;
136 
137   void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
138                            int OpIdx) const;
139   void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
140                                  const MachineInstr &MI, int OpIdx) const;
141 
142   void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
143                               int OpIdx) const;
144   void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
145                               int OpIdx) const;
146 
147   const RISCVSubtarget &STI;
148   const RISCVInstrInfo &TII;
149   const RISCVRegisterInfo &TRI;
150   const RISCVRegisterBankInfo &RBI;
151   const RISCVTargetMachine &TM;
152 
153   MachineRegisterInfo *MRI = nullptr;
154 
155   // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
156   // uses "STI." in the code generated by TableGen. We need to unify the name of
157   // Subtarget variable.
158   const RISCVSubtarget *Subtarget = &STI;
159 
160 #define GET_GLOBALISEL_PREDICATES_DECL
161 #include "RISCVGenGlobalISel.inc"
162 #undef GET_GLOBALISEL_PREDICATES_DECL
163 
164 #define GET_GLOBALISEL_TEMPORARIES_DECL
165 #include "RISCVGenGlobalISel.inc"
166 #undef GET_GLOBALISEL_TEMPORARIES_DECL
167 };
168 
169 } // end anonymous namespace
170 
171 #define GET_GLOBALISEL_IMPL
172 #include "RISCVGenGlobalISel.inc"
173 #undef GET_GLOBALISEL_IMPL
174 
175 RISCVInstructionSelector::RISCVInstructionSelector(
176     const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
177     const RISCVRegisterBankInfo &RBI)
178     : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
179       TM(TM),
180 
181 #define GET_GLOBALISEL_PREDICATES_INIT
182 #include "RISCVGenGlobalISel.inc"
183 #undef GET_GLOBALISEL_PREDICATES_INIT
184 #define GET_GLOBALISEL_TEMPORARIES_INIT
185 #include "RISCVGenGlobalISel.inc"
186 #undef GET_GLOBALISEL_TEMPORARIES_INIT
187 {
188 }
189 
190 InstructionSelector::ComplexRendererFns
191 RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
192                                           unsigned ShiftWidth) const {
193   if (!Root.isReg())
194     return std::nullopt;
195 
196   using namespace llvm::MIPatternMatch;
197 
198   Register ShAmtReg = Root.getReg();
199   // Peek through zext.
200   Register ZExtSrcReg;
201   if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
202     ShAmtReg = ZExtSrcReg;
203 
204   APInt AndMask;
205   Register AndSrcReg;
206   // Try to combine the following pattern (applicable to other shift
207   // instructions as well as 32-bit ones):
208   //
209   //   %4:gprb(s64) = G_AND %3, %2
210   //   %5:gprb(s64) = G_LSHR %1, %4(s64)
211   //
212   // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
213   // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
214   // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
215   // then it can be eliminated. Given register rs1 or rs2 holding a constant
216   // (the and mask), there are two cases G_AND can be erased:
217   //
218   // 1. the lowest log2(XLEN) bits of the and mask are all set
219   // 2. the bits of the register being masked are already unset (zero set)
220   if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
221     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
222     if (ShMask.isSubsetOf(AndMask)) {
223       ShAmtReg = AndSrcReg;
224     } else {
225       // SimplifyDemandedBits may have optimized the mask so try restoring any
226       // bits that are known zero.
227       KnownBits Known = KB->getKnownBits(AndSrcReg);
228       if (ShMask.isSubsetOf(AndMask | Known.Zero))
229         ShAmtReg = AndSrcReg;
230     }
231   }
232 
233   APInt Imm;
234   Register Reg;
235   if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
236     if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
237       // If we are shifting by X+N where N == 0 mod Size, then just shift by X
238       // to avoid the ADD.
239       ShAmtReg = Reg;
240   } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
241     if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
242       // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
243       // to generate a NEG instead of a SUB of a constant.
244       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
245       unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
246       return {{[=](MachineInstrBuilder &MIB) {
247         MachineIRBuilder(*MIB.getInstr())
248             .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
249         MIB.addReg(ShAmtReg);
250       }}};
251     }
252     if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
253       // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
254       // to generate a NOT instead of a SUB of a constant.
255       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
256       return {{[=](MachineInstrBuilder &MIB) {
257         MachineIRBuilder(*MIB.getInstr())
258             .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
259             .addImm(-1);
260         MIB.addReg(ShAmtReg);
261       }}};
262     }
263   }
264 
265   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
266 }
267 
268 InstructionSelector::ComplexRendererFns
269 RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
270                                          unsigned Bits) const {
271   if (!Root.isReg())
272     return std::nullopt;
273   Register RootReg = Root.getReg();
274   MachineInstr *RootDef = MRI->getVRegDef(RootReg);
275 
276   if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
277       RootDef->getOperand(2).getImm() == Bits) {
278     return {
279         {[=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); }}};
280   }
281 
282   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
283   if ((Size - KB->computeNumSignBits(RootReg)) < Bits)
284     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
285 
286   return std::nullopt;
287 }
288 
289 InstructionSelector::ComplexRendererFns
290 RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
291                                          unsigned Bits) const {
292   if (!Root.isReg())
293     return std::nullopt;
294   Register RootReg = Root.getReg();
295 
296   Register RegX;
297   uint64_t Mask = maskTrailingOnes<uint64_t>(Bits);
298   if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {
299     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
300   }
301 
302   if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&
303       MRI->getType(RegX).getScalarSizeInBits() == Bits)
304     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
305 
306   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
307   if (KB->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))
308     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
309 
310   return std::nullopt;
311 }
312 
313 InstructionSelector::ComplexRendererFns
314 RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
315                                          unsigned ShAmt) const {
316   using namespace llvm::MIPatternMatch;
317 
318   if (!Root.isReg())
319     return std::nullopt;
320   Register RootReg = Root.getReg();
321 
322   const unsigned XLen = STI.getXLen();
323   APInt Mask, C2;
324   Register RegY;
325   std::optional<bool> LeftShift;
326   // (and (shl y, c2), mask)
327   if (mi_match(RootReg, *MRI,
328                m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
329     LeftShift = true;
330   // (and (lshr y, c2), mask)
331   else if (mi_match(RootReg, *MRI,
332                     m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
333     LeftShift = false;
334 
335   if (LeftShift.has_value()) {
336     if (*LeftShift)
337       Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
338     else
339       Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
340 
341     if (Mask.isShiftedMask()) {
342       unsigned Leading = XLen - Mask.getActiveBits();
343       unsigned Trailing = Mask.countr_zero();
344       // Given (and (shl y, c2), mask) in which mask has no leading zeros and
345       // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
346       if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
347         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
348         return {{[=](MachineInstrBuilder &MIB) {
349           MachineIRBuilder(*MIB.getInstr())
350               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
351               .addImm(Trailing - C2.getLimitedValue());
352           MIB.addReg(DstReg);
353         }}};
354       }
355 
356       // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
357       // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
358       if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
359         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
360         return {{[=](MachineInstrBuilder &MIB) {
361           MachineIRBuilder(*MIB.getInstr())
362               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
363               .addImm(Leading + Trailing);
364           MIB.addReg(DstReg);
365         }}};
366       }
367     }
368   }
369 
370   LeftShift.reset();
371 
372   // (shl (and y, mask), c2)
373   if (mi_match(RootReg, *MRI,
374                m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
375                       m_ICst(C2))))
376     LeftShift = true;
377   // (lshr (and y, mask), c2)
378   else if (mi_match(RootReg, *MRI,
379                     m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
380                             m_ICst(C2))))
381     LeftShift = false;
382 
383   if (LeftShift.has_value() && Mask.isShiftedMask()) {
384     unsigned Leading = XLen - Mask.getActiveBits();
385     unsigned Trailing = Mask.countr_zero();
386 
387     // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
388     // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
389     bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
390                 (Trailing + C2.getLimitedValue()) == ShAmt;
391     if (!Cond)
392       // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
393       // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
394       Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
395              (Trailing - C2.getLimitedValue()) == ShAmt;
396 
397     if (Cond) {
398       Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
399       return {{[=](MachineInstrBuilder &MIB) {
400         MachineIRBuilder(*MIB.getInstr())
401             .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
402             .addImm(Trailing);
403         MIB.addReg(DstReg);
404       }}};
405     }
406   }
407 
408   return std::nullopt;
409 }
410 
411 InstructionSelector::ComplexRendererFns
412 RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
413                                             unsigned ShAmt) const {
414   using namespace llvm::MIPatternMatch;
415 
416   if (!Root.isReg())
417     return std::nullopt;
418   Register RootReg = Root.getReg();
419 
420   // Given (and (shl x, c2), mask) in which mask is a shifted mask with
421   // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
422   // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
423   APInt Mask, C2;
424   Register RegX;
425   if (mi_match(
426           RootReg, *MRI,
427           m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
428                                 m_ICst(Mask))))) {
429     Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
430 
431     if (Mask.isShiftedMask()) {
432       unsigned Leading = Mask.countl_zero();
433       unsigned Trailing = Mask.countr_zero();
434       if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
435         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
436         return {{[=](MachineInstrBuilder &MIB) {
437           MachineIRBuilder(*MIB.getInstr())
438               .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
439               .addImm(C2.getLimitedValue() - ShAmt);
440           MIB.addReg(DstReg);
441         }}};
442       }
443     }
444   }
445 
446   return std::nullopt;
447 }
448 
449 InstructionSelector::ComplexRendererFns
450 RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
451   assert(Root.isReg() && "Expected operand to be a Register");
452   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
453 
454   if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
455     auto C = RootDef->getOperand(1).getCImm();
456     if (C->getValue().isAllOnes())
457       // If the operand is a G_CONSTANT with value of all ones it is larger than
458       // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
459       // recognized specially by the vsetvli insertion pass.
460       return {{[=](MachineInstrBuilder &MIB) {
461         MIB.addImm(RISCV::VLMaxSentinel);
462       }}};
463 
464     if (isUInt<5>(C->getZExtValue())) {
465       uint64_t ZExtC = C->getZExtValue();
466       return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
467     }
468   }
469   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
470 }
471 
472 InstructionSelector::ComplexRendererFns
473 RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
474   if (!Root.isReg())
475     return std::nullopt;
476 
477   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
478   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
479     return {{
480         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
481         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
482     }};
483   }
484 
485   if (isBaseWithConstantOffset(Root, *MRI)) {
486     MachineOperand &LHS = RootDef->getOperand(1);
487     MachineOperand &RHS = RootDef->getOperand(2);
488     MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
489     MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
490 
491     int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
492     if (isInt<12>(RHSC)) {
493       if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
494         return {{
495             [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
496             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
497         }};
498 
499       return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
500                [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
501     }
502   }
503 
504   // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
505   // the combiner?
506   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
507            [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
508 }
509 
510 /// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
511 /// CC Must be an ICMP Predicate.
512 static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
513   switch (CC) {
514   default:
515     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
516   case CmpInst::Predicate::ICMP_EQ:
517     return RISCVCC::COND_EQ;
518   case CmpInst::Predicate::ICMP_NE:
519     return RISCVCC::COND_NE;
520   case CmpInst::Predicate::ICMP_ULT:
521     return RISCVCC::COND_LTU;
522   case CmpInst::Predicate::ICMP_SLT:
523     return RISCVCC::COND_LT;
524   case CmpInst::Predicate::ICMP_UGE:
525     return RISCVCC::COND_GEU;
526   case CmpInst::Predicate::ICMP_SGE:
527     return RISCVCC::COND_GE;
528   }
529 }
530 
531 static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
532                                  Register &LHS, Register &RHS,
533                                  MachineRegisterInfo &MRI) {
534   // Try to fold an ICmp. If that fails, use a NE compare with X0.
535   CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
536   if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
537     LHS = CondReg;
538     RHS = RISCV::X0;
539     CC = RISCVCC::COND_NE;
540     return;
541   }
542 
543   // We found an ICmp, do some canonicalizations.
544 
545   // Adjust comparisons to use comparison with 0 if possible.
546   if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
547     switch (Pred) {
548     case CmpInst::Predicate::ICMP_SGT:
549       // Convert X > -1 to X >= 0
550       if (*Constant == -1) {
551         CC = RISCVCC::COND_GE;
552         RHS = RISCV::X0;
553         return;
554       }
555       break;
556     case CmpInst::Predicate::ICMP_SLT:
557       // Convert X < 1 to 0 >= X
558       if (*Constant == 1) {
559         CC = RISCVCC::COND_GE;
560         RHS = LHS;
561         LHS = RISCV::X0;
562         return;
563       }
564       break;
565     default:
566       break;
567     }
568   }
569 
570   switch (Pred) {
571   default:
572     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
573   case CmpInst::Predicate::ICMP_EQ:
574   case CmpInst::Predicate::ICMP_NE:
575   case CmpInst::Predicate::ICMP_ULT:
576   case CmpInst::Predicate::ICMP_SLT:
577   case CmpInst::Predicate::ICMP_UGE:
578   case CmpInst::Predicate::ICMP_SGE:
579     // These CCs are supported directly by RISC-V branches.
580     break;
581   case CmpInst::Predicate::ICMP_SGT:
582   case CmpInst::Predicate::ICMP_SLE:
583   case CmpInst::Predicate::ICMP_UGT:
584   case CmpInst::Predicate::ICMP_ULE:
585     // These CCs are not supported directly by RISC-V branches, but changing the
586     // direction of the CC and swapping LHS and RHS are.
587     Pred = CmpInst::getSwappedPredicate(Pred);
588     std::swap(LHS, RHS);
589     break;
590   }
591 
592   CC = getRISCVCCFromICmp(Pred);
593 }
594 
595 bool RISCVInstructionSelector::select(MachineInstr &MI) {
596   MachineIRBuilder MIB(MI);
597 
598   preISelLower(MI, MIB);
599   const unsigned Opc = MI.getOpcode();
600 
601   if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
602     if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
603       const Register DefReg = MI.getOperand(0).getReg();
604       const LLT DefTy = MRI->getType(DefReg);
605 
606       const RegClassOrRegBank &RegClassOrBank =
607           MRI->getRegClassOrRegBank(DefReg);
608 
609       const TargetRegisterClass *DefRC =
610           dyn_cast<const TargetRegisterClass *>(RegClassOrBank);
611       if (!DefRC) {
612         if (!DefTy.isValid()) {
613           LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
614           return false;
615         }
616 
617         const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
618         DefRC = getRegClassForTypeOnBank(DefTy, RB);
619         if (!DefRC) {
620           LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
621           return false;
622         }
623       }
624 
625       MI.setDesc(TII.get(TargetOpcode::PHI));
626       return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
627     }
628 
629     // Certain non-generic instructions also need some special handling.
630     if (MI.isCopy())
631       return selectCopy(MI);
632 
633     return true;
634   }
635 
636   if (selectImpl(MI, *CoverageInfo))
637     return true;
638 
639   switch (Opc) {
640   case TargetOpcode::G_ANYEXT:
641   case TargetOpcode::G_PTRTOINT:
642   case TargetOpcode::G_INTTOPTR:
643   case TargetOpcode::G_TRUNC:
644   case TargetOpcode::G_FREEZE:
645     return selectCopy(MI);
646   case TargetOpcode::G_CONSTANT: {
647     Register DstReg = MI.getOperand(0).getReg();
648     int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
649 
650     if (!materializeImm(DstReg, Imm, MIB))
651       return false;
652 
653     MI.eraseFromParent();
654     return true;
655   }
656   case TargetOpcode::G_FCONSTANT: {
657     // TODO: Use constant pool for complext constants.
658     // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
659     Register DstReg = MI.getOperand(0).getReg();
660     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
661     APInt Imm = FPimm.bitcastToAPInt();
662     unsigned Size = MRI->getType(DstReg).getSizeInBits();
663     if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
664       Register GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
665       if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
666         return false;
667 
668       unsigned Opcode = Size == 64   ? RISCV::FMV_D_X
669                         : Size == 32 ? RISCV::FMV_W_X
670                                      : RISCV::FMV_H_X;
671       auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
672       if (!FMV.constrainAllUses(TII, TRI, RBI))
673         return false;
674     } else {
675       assert(Size == 64 && !Subtarget->is64Bit() &&
676              "Unexpected size or subtarget");
677       // Split into two pieces and build through the stack.
678       Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
679       Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
680       if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
681                           MIB))
682         return false;
683       if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
684         return false;
685       MachineInstrBuilder PairF64 = MIB.buildInstr(
686           RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
687       if (!PairF64.constrainAllUses(TII, TRI, RBI))
688         return false;
689     }
690 
691     MI.eraseFromParent();
692     return true;
693   }
694   case TargetOpcode::G_GLOBAL_VALUE: {
695     auto *GV = MI.getOperand(1).getGlobal();
696     if (GV->isThreadLocal()) {
697       // TODO: implement this case.
698       return false;
699     }
700 
701     return selectAddr(MI, MIB, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
702   }
703   case TargetOpcode::G_JUMP_TABLE:
704   case TargetOpcode::G_CONSTANT_POOL:
705     return selectAddr(MI, MIB, MRI);
706   case TargetOpcode::G_BRCOND: {
707     Register LHS, RHS;
708     RISCVCC::CondCode CC;
709     getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
710 
711     auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
712                    .addMBB(MI.getOperand(1).getMBB());
713     MI.eraseFromParent();
714     return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
715   }
716   case TargetOpcode::G_BRINDIRECT:
717     MI.setDesc(TII.get(RISCV::PseudoBRIND));
718     MI.addOperand(MachineOperand::CreateImm(0));
719     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
720   case TargetOpcode::G_SELECT:
721     return selectSelect(MI, MIB);
722   case TargetOpcode::G_FCMP:
723     return selectFPCompare(MI, MIB);
724   case TargetOpcode::G_FENCE: {
725     AtomicOrdering FenceOrdering =
726         static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
727     SyncScope::ID FenceSSID =
728         static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
729     emitFence(FenceOrdering, FenceSSID, MIB);
730     MI.eraseFromParent();
731     return true;
732   }
733   case TargetOpcode::G_IMPLICIT_DEF:
734     return selectImplicitDef(MI, MIB);
735   case TargetOpcode::G_MERGE_VALUES:
736     return selectMergeValues(MI, MIB);
737   case TargetOpcode::G_UNMERGE_VALUES:
738     return selectUnmergeValues(MI, MIB);
739   default:
740     return false;
741   }
742 }
743 
744 bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI,
745                                                  MachineIRBuilder &MIB) const {
746   assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
747 
748   // Build a F64 Pair from operands
749   if (MI.getNumOperands() != 3)
750     return false;
751   Register Dst = MI.getOperand(0).getReg();
752   Register Lo = MI.getOperand(1).getReg();
753   Register Hi = MI.getOperand(2).getReg();
754   if (!isRegInFprb(Dst) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
755     return false;
756   MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
757   return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
758 }
759 
760 bool RISCVInstructionSelector::selectUnmergeValues(
761     MachineInstr &MI, MachineIRBuilder &MIB) const {
762   assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
763 
764   // Split F64 Src into two s32 parts
765   if (MI.getNumOperands() != 3)
766     return false;
767   Register Src = MI.getOperand(2).getReg();
768   Register Lo = MI.getOperand(0).getReg();
769   Register Hi = MI.getOperand(1).getReg();
770   if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
771     return false;
772   MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
773   return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
774 }
775 
776 bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
777                                                  MachineIRBuilder &MIB) {
778   Register PtrReg = Op.getReg();
779   assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
780 
781   const LLT sXLen = LLT::scalar(STI.getXLen());
782   auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
783   MRI->setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
784   Op.setReg(PtrToInt.getReg(0));
785   return select(*PtrToInt);
786 }
787 
788 void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
789                                             MachineIRBuilder &MIB) {
790   switch (MI.getOpcode()) {
791   case TargetOpcode::G_PTR_ADD: {
792     Register DstReg = MI.getOperand(0).getReg();
793     const LLT sXLen = LLT::scalar(STI.getXLen());
794 
795     replacePtrWithInt(MI.getOperand(1), MIB);
796     MI.setDesc(TII.get(TargetOpcode::G_ADD));
797     MRI->setType(DstReg, sXLen);
798     break;
799   }
800   case TargetOpcode::G_PTRMASK: {
801     Register DstReg = MI.getOperand(0).getReg();
802     const LLT sXLen = LLT::scalar(STI.getXLen());
803     replacePtrWithInt(MI.getOperand(1), MIB);
804     MI.setDesc(TII.get(TargetOpcode::G_AND));
805     MRI->setType(DstReg, sXLen);
806     break;
807   }
808   }
809 }
810 
811 void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
812                                             const MachineInstr &MI,
813                                             int OpIdx) const {
814   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
815          "Expected G_CONSTANT");
816   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
817   MIB.addImm(-CstVal);
818 }
819 
820 void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
821                                                     const MachineInstr &MI,
822                                                     int OpIdx) const {
823   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
824          "Expected G_CONSTANT");
825   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
826   MIB.addImm(STI.getXLen() - CstVal);
827 }
828 
829 void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
830                                                   const MachineInstr &MI,
831                                                   int OpIdx) const {
832   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
833          "Expected G_CONSTANT");
834   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
835   MIB.addImm(32 - CstVal);
836 }
837 
838 void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
839                                               const MachineInstr &MI,
840                                               int OpIdx) const {
841   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
842          "Expected G_CONSTANT");
843   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
844   MIB.addImm(CstVal + 1);
845 }
846 
847 void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
848                                          const MachineInstr &MI,
849                                          int OpIdx) const {
850   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
851          "Expected G_CONSTANT");
852   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
853   MIB.addImm(CstVal);
854 }
855 
856 void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
857                                                 const MachineInstr &MI,
858                                                 int OpIdx) const {
859   assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
860          "Expected G_FRAME_INDEX");
861   MIB.add(MI.getOperand(1));
862 }
863 
864 void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
865                                                    const MachineInstr &MI,
866                                                    int OpIdx) const {
867   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
868          "Expected G_CONSTANT");
869   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
870   MIB.addImm(llvm::countr_zero(C));
871 }
872 
873 void RISCVInstructionSelector::renderXLenSubTrailingOnes(
874     MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
875   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
876          "Expected G_CONSTANT");
877   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
878   MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
879 }
880 
881 void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
882                                                       const MachineInstr &MI,
883                                                       int OpIdx) const {
884   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
885          "Expected G_CONSTANT");
886   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
887   int64_t Adj = Imm < 0 ? -2048 : 2047;
888   MIB.addImm(Imm - Adj);
889 }
890 
891 void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
892                                                       const MachineInstr &MI,
893                                                       int OpIdx) const {
894   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
895          "Expected G_CONSTANT");
896   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
897   MIB.addImm(Imm);
898 }
899 
900 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
901     LLT Ty, const RegisterBank &RB) const {
902   if (RB.getID() == RISCV::GPRBRegBankID) {
903     if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
904       return &RISCV::GPRRegClass;
905   }
906 
907   if (RB.getID() == RISCV::FPRBRegBankID) {
908     if (Ty.getSizeInBits() == 16)
909       return &RISCV::FPR16RegClass;
910     if (Ty.getSizeInBits() == 32)
911       return &RISCV::FPR32RegClass;
912     if (Ty.getSizeInBits() == 64)
913       return &RISCV::FPR64RegClass;
914   }
915 
916   if (RB.getID() == RISCV::VRBRegBankID) {
917     if (Ty.getSizeInBits().getKnownMinValue() <= 64)
918       return &RISCV::VRRegClass;
919 
920     if (Ty.getSizeInBits().getKnownMinValue() == 128)
921       return &RISCV::VRM2RegClass;
922 
923     if (Ty.getSizeInBits().getKnownMinValue() == 256)
924       return &RISCV::VRM4RegClass;
925 
926     if (Ty.getSizeInBits().getKnownMinValue() == 512)
927       return &RISCV::VRM8RegClass;
928   }
929 
930   return nullptr;
931 }
932 
933 bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
934   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
935 }
936 
937 bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
938   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
939 }
940 
941 bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
942   Register DstReg = MI.getOperand(0).getReg();
943 
944   if (DstReg.isPhysical())
945     return true;
946 
947   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
948       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
949   assert(DstRC &&
950          "Register class not available for LLT, register bank combination");
951 
952   // No need to constrain SrcReg. It will get constrained when
953   // we hit another of its uses or its defs.
954   // Copies do not have constraints.
955   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
956     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
957                       << " operand\n");
958     return false;
959   }
960 
961   MI.setDesc(TII.get(RISCV::COPY));
962   return true;
963 }
964 
965 bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
966                                                  MachineIRBuilder &MIB) const {
967   assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
968 
969   const Register DstReg = MI.getOperand(0).getReg();
970   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
971       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
972 
973   assert(DstRC &&
974          "Register class not available for LLT, register bank combination");
975 
976   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
977     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
978                       << " operand\n");
979   }
980   MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
981   return true;
982 }
983 
984 bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
985                                               MachineIRBuilder &MIB) const {
986   if (Imm == 0) {
987     MIB.buildCopy(DstReg, Register(RISCV::X0));
988     RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
989     return true;
990   }
991 
992   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
993   unsigned NumInsts = Seq.size();
994   Register SrcReg = RISCV::X0;
995 
996   for (unsigned i = 0; i < NumInsts; i++) {
997     Register TmpReg = i < NumInsts - 1
998                           ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
999                           : DstReg;
1000     const RISCVMatInt::Inst &I = Seq[i];
1001     MachineInstr *Result;
1002 
1003     switch (I.getOpndKind()) {
1004     case RISCVMatInt::Imm:
1005       // clang-format off
1006       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
1007                    .addImm(I.getImm());
1008       // clang-format on
1009       break;
1010     case RISCVMatInt::RegX0:
1011       Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
1012                               {SrcReg, Register(RISCV::X0)});
1013       break;
1014     case RISCVMatInt::RegReg:
1015       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
1016       break;
1017     case RISCVMatInt::RegImm:
1018       Result =
1019           MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
1020       break;
1021     }
1022 
1023     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1024       return false;
1025 
1026     SrcReg = TmpReg;
1027   }
1028 
1029   return true;
1030 }
1031 
1032 bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
1033                                           MachineIRBuilder &MIB, bool IsLocal,
1034                                           bool IsExternWeak) const {
1035   assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1036           MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1037           MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1038          "Unexpected opcode");
1039 
1040   const MachineOperand &DispMO = MI.getOperand(1);
1041 
1042   Register DefReg = MI.getOperand(0).getReg();
1043   const LLT DefTy = MRI->getType(DefReg);
1044 
1045   // When HWASAN is used and tagging of global variables is enabled
1046   // they should be accessed via the GOT, since the tagged address of a global
1047   // is incompatible with existing code models. This also applies to non-pic
1048   // mode.
1049   if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1050     if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1051       // Use PC-relative addressing to access the symbol. This generates the
1052       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1053       // %pcrel_lo(auipc)).
1054       MI.setDesc(TII.get(RISCV::PseudoLLA));
1055       return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1056     }
1057 
1058     // Use PC-relative addressing to access the GOT for this symbol, then
1059     // load the address from the GOT. This generates the pattern (PseudoLGA
1060     // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1061     // %pcrel_lo(auipc))).
1062     MachineFunction &MF = *MI.getParent()->getParent();
1063     MachineMemOperand *MemOp = MF.getMachineMemOperand(
1064         MachinePointerInfo::getGOT(MF),
1065         MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1066             MachineMemOperand::MOInvariant,
1067         DefTy, Align(DefTy.getSizeInBits() / 8));
1068 
1069     auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1070                       .addDisp(DispMO, 0)
1071                       .addMemOperand(MemOp);
1072 
1073     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1074       return false;
1075 
1076     MI.eraseFromParent();
1077     return true;
1078   }
1079 
1080   switch (TM.getCodeModel()) {
1081   default: {
1082     reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1083                        getName(), "Unsupported code model for lowering", MI);
1084     return false;
1085   }
1086   case CodeModel::Small: {
1087     // Must lie within a single 2 GiB address range and must lie between
1088     // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1089     // (lui %hi(sym)) %lo(sym)).
1090     Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1091     MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1092                                .addDisp(DispMO, 0, RISCVII::MO_HI);
1093 
1094     if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1095       return false;
1096 
1097     auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1098                       .addDisp(DispMO, 0, RISCVII::MO_LO);
1099 
1100     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1101       return false;
1102 
1103     MI.eraseFromParent();
1104     return true;
1105   }
1106   case CodeModel::Medium:
1107     // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1108     // relocation needs to reference a label that points to the auipc
1109     // instruction itself, not the global. This cannot be done inside the
1110     // instruction selector.
1111     if (IsExternWeak) {
1112       // An extern weak symbol may be undefined, i.e. have value 0, which may
1113       // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1114       // symbol. This generates the pattern (PseudoLGA sym), which expands to
1115       // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1116       MachineFunction &MF = *MI.getParent()->getParent();
1117       MachineMemOperand *MemOp = MF.getMachineMemOperand(
1118           MachinePointerInfo::getGOT(MF),
1119           MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1120               MachineMemOperand::MOInvariant,
1121           DefTy, Align(DefTy.getSizeInBits() / 8));
1122 
1123       auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1124                         .addDisp(DispMO, 0)
1125                         .addMemOperand(MemOp);
1126 
1127       if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1128         return false;
1129 
1130       MI.eraseFromParent();
1131       return true;
1132     }
1133 
1134     // Generate a sequence for accessing addresses within any 2GiB range
1135     // within the address space. This generates the pattern (PseudoLLA sym),
1136     // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1137     MI.setDesc(TII.get(RISCV::PseudoLLA));
1138     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1139   }
1140 
1141   return false;
1142 }
1143 
1144 bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1145                                             MachineIRBuilder &MIB) const {
1146   auto &SelectMI = cast<GSelect>(MI);
1147 
1148   Register LHS, RHS;
1149   RISCVCC::CondCode CC;
1150   getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
1151 
1152   Register DstReg = SelectMI.getReg(0);
1153 
1154   unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1155   if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1156     unsigned Size = MRI->getType(DstReg).getSizeInBits();
1157     Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1158                      : RISCV::Select_FPR64_Using_CC_GPR;
1159   }
1160 
1161   MachineInstr *Result = MIB.buildInstr(Opc)
1162                              .addDef(DstReg)
1163                              .addReg(LHS)
1164                              .addReg(RHS)
1165                              .addImm(CC)
1166                              .addReg(SelectMI.getTrueReg())
1167                              .addReg(SelectMI.getFalseReg());
1168   MI.eraseFromParent();
1169   return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1170 }
1171 
1172 // Convert an FCMP predicate to one of the supported F or D instructions.
1173 static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1174   assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1175   switch (Pred) {
1176   default:
1177     llvm_unreachable("Unsupported predicate");
1178   case CmpInst::FCMP_OLT:
1179     return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1180   case CmpInst::FCMP_OLE:
1181     return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1182   case CmpInst::FCMP_OEQ:
1183     return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1184   }
1185 }
1186 
1187 // Try legalizing an FCMP by swapping or inverting the predicate to one that
1188 // is supported.
1189 static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1190                                   CmpInst::Predicate &Pred, bool &NeedInvert) {
1191   auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1192     return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1193            Pred == CmpInst::FCMP_OEQ;
1194   };
1195 
1196   assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1197 
1198   CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
1199   if (isLegalFCmpPredicate(InvPred)) {
1200     Pred = InvPred;
1201     std::swap(LHS, RHS);
1202     return true;
1203   }
1204 
1205   InvPred = CmpInst::getInversePredicate(Pred);
1206   NeedInvert = true;
1207   if (isLegalFCmpPredicate(InvPred)) {
1208     Pred = InvPred;
1209     return true;
1210   }
1211   InvPred = CmpInst::getSwappedPredicate(InvPred);
1212   if (isLegalFCmpPredicate(InvPred)) {
1213     Pred = InvPred;
1214     std::swap(LHS, RHS);
1215     return true;
1216   }
1217 
1218   return false;
1219 }
1220 
1221 // Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1222 // the result in DstReg.
1223 // FIXME: Maybe we should expand this earlier.
1224 bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1225                                                MachineIRBuilder &MIB) const {
1226   auto &CmpMI = cast<GFCmp>(MI);
1227   CmpInst::Predicate Pred = CmpMI.getCond();
1228 
1229   Register DstReg = CmpMI.getReg(0);
1230   Register LHS = CmpMI.getLHSReg();
1231   Register RHS = CmpMI.getRHSReg();
1232 
1233   unsigned Size = MRI->getType(LHS).getSizeInBits();
1234   assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1235 
1236   Register TmpReg = DstReg;
1237 
1238   bool NeedInvert = false;
1239   // First try swapping operands or inverting.
1240   if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1241     if (NeedInvert)
1242       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1243     auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1244     if (!Cmp.constrainAllUses(TII, TRI, RBI))
1245       return false;
1246   } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1247     // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1248     NeedInvert = Pred == CmpInst::FCMP_UEQ;
1249     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1250                                {&RISCV::GPRRegClass}, {LHS, RHS});
1251     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1252       return false;
1253     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1254                                {&RISCV::GPRRegClass}, {RHS, LHS});
1255     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1256       return false;
1257     if (NeedInvert)
1258       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1259     auto Or =
1260         MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1261     if (!Or.constrainAllUses(TII, TRI, RBI))
1262       return false;
1263   } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1264     // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1265     // FIXME: If LHS and RHS are the same we can use a single FEQ.
1266     NeedInvert = Pred == CmpInst::FCMP_UNO;
1267     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1268                                {&RISCV::GPRRegClass}, {LHS, LHS});
1269     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1270       return false;
1271     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1272                                {&RISCV::GPRRegClass}, {RHS, RHS});
1273     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1274       return false;
1275     if (NeedInvert)
1276       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1277     auto And =
1278         MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1279     if (!And.constrainAllUses(TII, TRI, RBI))
1280       return false;
1281   } else
1282     llvm_unreachable("Unhandled predicate");
1283 
1284   // Emit an XORI to invert the result if needed.
1285   if (NeedInvert) {
1286     auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1287     if (!Xor.constrainAllUses(TII, TRI, RBI))
1288       return false;
1289   }
1290 
1291   MI.eraseFromParent();
1292   return true;
1293 }
1294 
1295 void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1296                                          SyncScope::ID FenceSSID,
1297                                          MachineIRBuilder &MIB) const {
1298   if (STI.hasStdExtZtso()) {
1299     // The only fence that needs an instruction is a sequentially-consistent
1300     // cross-thread fence.
1301     if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1302         FenceSSID == SyncScope::System) {
1303       // fence rw, rw
1304       MIB.buildInstr(RISCV::FENCE, {}, {})
1305           .addImm(RISCVFenceField::R | RISCVFenceField::W)
1306           .addImm(RISCVFenceField::R | RISCVFenceField::W);
1307       return;
1308     }
1309 
1310     // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1311     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1312     return;
1313   }
1314 
1315   // singlethread fences only synchronize with signal handlers on the same
1316   // thread and thus only need to preserve instruction order, not actually
1317   // enforce memory ordering.
1318   if (FenceSSID == SyncScope::SingleThread) {
1319     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1320     return;
1321   }
1322 
1323   // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1324   // Manual: Volume I.
1325   unsigned Pred, Succ;
1326   switch (FenceOrdering) {
1327   default:
1328     llvm_unreachable("Unexpected ordering");
1329   case AtomicOrdering::AcquireRelease:
1330     // fence acq_rel -> fence.tso
1331     MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1332     return;
1333   case AtomicOrdering::Acquire:
1334     // fence acquire -> fence r, rw
1335     Pred = RISCVFenceField::R;
1336     Succ = RISCVFenceField::R | RISCVFenceField::W;
1337     break;
1338   case AtomicOrdering::Release:
1339     // fence release -> fence rw, w
1340     Pred = RISCVFenceField::R | RISCVFenceField::W;
1341     Succ = RISCVFenceField::W;
1342     break;
1343   case AtomicOrdering::SequentiallyConsistent:
1344     // fence seq_cst -> fence rw, rw
1345     Pred = RISCVFenceField::R | RISCVFenceField::W;
1346     Succ = RISCVFenceField::R | RISCVFenceField::W;
1347     break;
1348   }
1349   MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1350 }
1351 
1352 namespace llvm {
1353 InstructionSelector *
1354 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1355                                const RISCVSubtarget &Subtarget,
1356                                const RISCVRegisterBankInfo &RBI) {
1357   return new RISCVInstructionSelector(TM, Subtarget, RBI);
1358 }
1359 } // end namespace llvm
1360