xref: /llvm-project/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp (revision 785b16ad04a741dce65ebaa11ee86d9dd19dd699) 
1 //===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// RISC-V.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCVRegisterBankInfo.h"
16 #include "RISCVSubtarget.h"
17 #include "RISCVTargetMachine.h"
18 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
20 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24 #include "llvm/CodeGen/MachineJumpTableInfo.h"
25 #include "llvm/IR/IntrinsicsRISCV.h"
26 #include "llvm/Support/Debug.h"
27 
28 #define DEBUG_TYPE "riscv-isel"
29 
30 using namespace llvm;
31 using namespace MIPatternMatch;
32 
33 #define GET_GLOBALISEL_PREDICATE_BITSET
34 #include "RISCVGenGlobalISel.inc"
35 #undef GET_GLOBALISEL_PREDICATE_BITSET
36 
37 namespace {
38 
39 class RISCVInstructionSelector : public InstructionSelector {
40 public:
41   RISCVInstructionSelector(const RISCVTargetMachine &TM,
42                            const RISCVSubtarget &STI,
43                            const RISCVRegisterBankInfo &RBI);
44 
45   bool select(MachineInstr &MI) override;
46 
47   void setupMF(MachineFunction &MF, GISelKnownBits *KB,
48                CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49                BlockFrequencyInfo *BFI) override {
50     InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
51     MRI = &MF.getRegInfo();
52   }
53 
54   static const char *getName() { return DEBUG_TYPE; }
55 
56 private:
57   const TargetRegisterClass *
58   getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
59 
60   bool isRegInGprb(Register Reg) const;
61   bool isRegInFprb(Register Reg) const;
62 
63   // tblgen-erated 'select' implementation, used as the initial selector for
64   // the patterns that don't require complex C++.
65   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
66 
67   // A lowering phase that runs before any selection attempts.
68   // Returns true if the instruction was modified.
69   void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
70 
71   bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
72 
73   // Custom selection methods
74   bool selectCopy(MachineInstr &MI) const;
75   bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
76   bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
77   bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
78                   bool IsExternWeak = false) const;
79   bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
80   bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
81   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
82                  MachineIRBuilder &MIB) const;
83   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
84 
85   ComplexRendererFns selectShiftMask(MachineOperand &Root,
86                                      unsigned ShiftWidth) const;
87   ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
88     return selectShiftMask(Root, STI.getXLen());
89   }
90   ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
91     return selectShiftMask(Root, 32);
92   }
93   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
94 
95   ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
96   template <unsigned Bits>
97   ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
98     return selectSExtBits(Root, Bits);
99   }
100 
101   ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
102   template <unsigned Bits>
103   ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
104     return selectZExtBits(Root, Bits);
105   }
106 
107   ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
108   template <unsigned ShAmt>
109   ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
110     return selectSHXADDOp(Root, ShAmt);
111   }
112 
113   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
114                                        unsigned ShAmt) const;
115   template <unsigned ShAmt>
116   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
117     return selectSHXADD_UWOp(Root, ShAmt);
118   }
119 
120   ComplexRendererFns renderVLOp(MachineOperand &Root) const;
121 
122   // Custom renderers for tablegen
123   void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
124                     int OpIdx) const;
125   void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
126                             int OpIdx) const;
127   void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
128                           int OpIdx) const;
129   void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
130                       int OpIdx) const;
131   void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
132                  int OpIdx) const;
133   void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
134                         int OpIdx) const;
135 
136   void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
137                            int OpIdx) const;
138   void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
139                                  const MachineInstr &MI, int OpIdx) const;
140 
141   void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
142                               int OpIdx) const;
143   void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
144                               int OpIdx) const;
145 
146   const RISCVSubtarget &STI;
147   const RISCVInstrInfo &TII;
148   const RISCVRegisterInfo &TRI;
149   const RISCVRegisterBankInfo &RBI;
150   const RISCVTargetMachine &TM;
151 
152   MachineRegisterInfo *MRI = nullptr;
153 
154   // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
155   // uses "STI." in the code generated by TableGen. We need to unify the name of
156   // Subtarget variable.
157   const RISCVSubtarget *Subtarget = &STI;
158 
159 #define GET_GLOBALISEL_PREDICATES_DECL
160 #include "RISCVGenGlobalISel.inc"
161 #undef GET_GLOBALISEL_PREDICATES_DECL
162 
163 #define GET_GLOBALISEL_TEMPORARIES_DECL
164 #include "RISCVGenGlobalISel.inc"
165 #undef GET_GLOBALISEL_TEMPORARIES_DECL
166 };
167 
168 } // end anonymous namespace
169 
170 #define GET_GLOBALISEL_IMPL
171 #include "RISCVGenGlobalISel.inc"
172 #undef GET_GLOBALISEL_IMPL
173 
174 RISCVInstructionSelector::RISCVInstructionSelector(
175     const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
176     const RISCVRegisterBankInfo &RBI)
177     : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
178       TM(TM),
179 
180 #define GET_GLOBALISEL_PREDICATES_INIT
181 #include "RISCVGenGlobalISel.inc"
182 #undef GET_GLOBALISEL_PREDICATES_INIT
183 #define GET_GLOBALISEL_TEMPORARIES_INIT
184 #include "RISCVGenGlobalISel.inc"
185 #undef GET_GLOBALISEL_TEMPORARIES_INIT
186 {
187 }
188 
189 InstructionSelector::ComplexRendererFns
190 RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
191                                           unsigned ShiftWidth) const {
192   if (!Root.isReg())
193     return std::nullopt;
194 
195   using namespace llvm::MIPatternMatch;
196 
197   Register ShAmtReg = Root.getReg();
198   // Peek through zext.
199   Register ZExtSrcReg;
200   if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
201     ShAmtReg = ZExtSrcReg;
202 
203   APInt AndMask;
204   Register AndSrcReg;
205   // Try to combine the following pattern (applicable to other shift
206   // instructions as well as 32-bit ones):
207   //
208   //   %4:gprb(s64) = G_AND %3, %2
209   //   %5:gprb(s64) = G_LSHR %1, %4(s64)
210   //
211   // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
212   // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
213   // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
214   // then it can be eliminated. Given register rs1 or rs2 holding a constant
215   // (the and mask), there are two cases G_AND can be erased:
216   //
217   // 1. the lowest log2(XLEN) bits of the and mask are all set
218   // 2. the bits of the register being masked are already unset (zero set)
219   if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
220     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
221     if (ShMask.isSubsetOf(AndMask)) {
222       ShAmtReg = AndSrcReg;
223     } else {
224       // SimplifyDemandedBits may have optimized the mask so try restoring any
225       // bits that are known zero.
226       KnownBits Known = KB->getKnownBits(AndSrcReg);
227       if (ShMask.isSubsetOf(AndMask | Known.Zero))
228         ShAmtReg = AndSrcReg;
229     }
230   }
231 
232   APInt Imm;
233   Register Reg;
234   if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
235     if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
236       // If we are shifting by X+N where N == 0 mod Size, then just shift by X
237       // to avoid the ADD.
238       ShAmtReg = Reg;
239   } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
240     if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
241       // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
242       // to generate a NEG instead of a SUB of a constant.
243       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
244       unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
245       return {{[=](MachineInstrBuilder &MIB) {
246         MachineIRBuilder(*MIB.getInstr())
247             .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
248         MIB.addReg(ShAmtReg);
249       }}};
250     }
251     if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
252       // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
253       // to generate a NOT instead of a SUB of a constant.
254       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
255       return {{[=](MachineInstrBuilder &MIB) {
256         MachineIRBuilder(*MIB.getInstr())
257             .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
258             .addImm(-1);
259         MIB.addReg(ShAmtReg);
260       }}};
261     }
262   }
263 
264   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
265 }
266 
267 InstructionSelector::ComplexRendererFns
268 RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
269                                          unsigned Bits) const {
270   if (!Root.isReg())
271     return std::nullopt;
272   Register RootReg = Root.getReg();
273   MachineInstr *RootDef = MRI->getVRegDef(RootReg);
274 
275   if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
276       RootDef->getOperand(2).getImm() == Bits) {
277     return {
278         {[=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); }}};
279   }
280 
281   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
282   if ((Size - KB->computeNumSignBits(RootReg)) < Bits)
283     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
284 
285   return std::nullopt;
286 }
287 
288 InstructionSelector::ComplexRendererFns
289 RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
290                                          unsigned Bits) const {
291   if (!Root.isReg())
292     return std::nullopt;
293   Register RootReg = Root.getReg();
294 
295   Register RegX;
296   uint64_t Mask = maskTrailingOnes<uint64_t>(Bits);
297   if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {
298     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
299   }
300 
301   if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&
302       MRI->getType(RegX).getScalarSizeInBits() == Bits)
303     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
304 
305   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
306   if (KB->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))
307     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
308 
309   return std::nullopt;
310 }
311 
312 InstructionSelector::ComplexRendererFns
313 RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
314                                          unsigned ShAmt) const {
315   using namespace llvm::MIPatternMatch;
316 
317   if (!Root.isReg())
318     return std::nullopt;
319   Register RootReg = Root.getReg();
320 
321   const unsigned XLen = STI.getXLen();
322   APInt Mask, C2;
323   Register RegY;
324   std::optional<bool> LeftShift;
325   // (and (shl y, c2), mask)
326   if (mi_match(RootReg, *MRI,
327                m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
328     LeftShift = true;
329   // (and (lshr y, c2), mask)
330   else if (mi_match(RootReg, *MRI,
331                     m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
332     LeftShift = false;
333 
334   if (LeftShift.has_value()) {
335     if (*LeftShift)
336       Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
337     else
338       Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
339 
340     if (Mask.isShiftedMask()) {
341       unsigned Leading = XLen - Mask.getActiveBits();
342       unsigned Trailing = Mask.countr_zero();
343       // Given (and (shl y, c2), mask) in which mask has no leading zeros and
344       // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
345       if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
346         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
347         return {{[=](MachineInstrBuilder &MIB) {
348           MachineIRBuilder(*MIB.getInstr())
349               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
350               .addImm(Trailing - C2.getLimitedValue());
351           MIB.addReg(DstReg);
352         }}};
353       }
354 
355       // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
356       // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
357       if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
358         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
359         return {{[=](MachineInstrBuilder &MIB) {
360           MachineIRBuilder(*MIB.getInstr())
361               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
362               .addImm(Leading + Trailing);
363           MIB.addReg(DstReg);
364         }}};
365       }
366     }
367   }
368 
369   LeftShift.reset();
370 
371   // (shl (and y, mask), c2)
372   if (mi_match(RootReg, *MRI,
373                m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
374                       m_ICst(C2))))
375     LeftShift = true;
376   // (lshr (and y, mask), c2)
377   else if (mi_match(RootReg, *MRI,
378                     m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
379                             m_ICst(C2))))
380     LeftShift = false;
381 
382   if (LeftShift.has_value() && Mask.isShiftedMask()) {
383     unsigned Leading = XLen - Mask.getActiveBits();
384     unsigned Trailing = Mask.countr_zero();
385 
386     // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
387     // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
388     bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
389                 (Trailing + C2.getLimitedValue()) == ShAmt;
390     if (!Cond)
391       // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
392       // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
393       Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
394              (Trailing - C2.getLimitedValue()) == ShAmt;
395 
396     if (Cond) {
397       Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
398       return {{[=](MachineInstrBuilder &MIB) {
399         MachineIRBuilder(*MIB.getInstr())
400             .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
401             .addImm(Trailing);
402         MIB.addReg(DstReg);
403       }}};
404     }
405   }
406 
407   return std::nullopt;
408 }
409 
410 InstructionSelector::ComplexRendererFns
411 RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
412                                             unsigned ShAmt) const {
413   using namespace llvm::MIPatternMatch;
414 
415   if (!Root.isReg())
416     return std::nullopt;
417   Register RootReg = Root.getReg();
418 
419   // Given (and (shl x, c2), mask) in which mask is a shifted mask with
420   // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
421   // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
422   APInt Mask, C2;
423   Register RegX;
424   if (mi_match(
425           RootReg, *MRI,
426           m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
427                                 m_ICst(Mask))))) {
428     Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
429 
430     if (Mask.isShiftedMask()) {
431       unsigned Leading = Mask.countl_zero();
432       unsigned Trailing = Mask.countr_zero();
433       if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
434         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
435         return {{[=](MachineInstrBuilder &MIB) {
436           MachineIRBuilder(*MIB.getInstr())
437               .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
438               .addImm(C2.getLimitedValue() - ShAmt);
439           MIB.addReg(DstReg);
440         }}};
441       }
442     }
443   }
444 
445   return std::nullopt;
446 }
447 
448 InstructionSelector::ComplexRendererFns
449 RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
450   assert(Root.isReg() && "Expected operand to be a Register");
451   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
452 
453   if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
454     auto C = RootDef->getOperand(1).getCImm();
455     if (C->getValue().isAllOnes())
456       // If the operand is a G_CONSTANT with value of all ones it is larger than
457       // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
458       // recognized specially by the vsetvli insertion pass.
459       return {{[=](MachineInstrBuilder &MIB) {
460         MIB.addImm(RISCV::VLMaxSentinel);
461       }}};
462 
463     if (isUInt<5>(C->getZExtValue())) {
464       uint64_t ZExtC = C->getZExtValue();
465       return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
466     }
467   }
468   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
469 }
470 
471 InstructionSelector::ComplexRendererFns
472 RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
473   if (!Root.isReg())
474     return std::nullopt;
475 
476   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
477   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
478     return {{
479         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
480         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
481     }};
482   }
483 
484   if (isBaseWithConstantOffset(Root, *MRI)) {
485     MachineOperand &LHS = RootDef->getOperand(1);
486     MachineOperand &RHS = RootDef->getOperand(2);
487     MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
488     MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
489 
490     int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
491     if (isInt<12>(RHSC)) {
492       if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
493         return {{
494             [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
495             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
496         }};
497 
498       return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
499                [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
500     }
501   }
502 
503   // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
504   // the combiner?
505   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
506            [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
507 }
508 
509 /// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
510 /// CC Must be an ICMP Predicate.
511 static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
512   switch (CC) {
513   default:
514     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
515   case CmpInst::Predicate::ICMP_EQ:
516     return RISCVCC::COND_EQ;
517   case CmpInst::Predicate::ICMP_NE:
518     return RISCVCC::COND_NE;
519   case CmpInst::Predicate::ICMP_ULT:
520     return RISCVCC::COND_LTU;
521   case CmpInst::Predicate::ICMP_SLT:
522     return RISCVCC::COND_LT;
523   case CmpInst::Predicate::ICMP_UGE:
524     return RISCVCC::COND_GEU;
525   case CmpInst::Predicate::ICMP_SGE:
526     return RISCVCC::COND_GE;
527   }
528 }
529 
530 static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
531                                  Register &LHS, Register &RHS,
532                                  MachineRegisterInfo &MRI) {
533   // Try to fold an ICmp. If that fails, use a NE compare with X0.
534   CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
535   if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
536     LHS = CondReg;
537     RHS = RISCV::X0;
538     CC = RISCVCC::COND_NE;
539     return;
540   }
541 
542   // We found an ICmp, do some canonicalizations.
543 
544   // Adjust comparisons to use comparison with 0 if possible.
545   if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
546     switch (Pred) {
547     case CmpInst::Predicate::ICMP_SGT:
548       // Convert X > -1 to X >= 0
549       if (*Constant == -1) {
550         CC = RISCVCC::COND_GE;
551         RHS = RISCV::X0;
552         return;
553       }
554       break;
555     case CmpInst::Predicate::ICMP_SLT:
556       // Convert X < 1 to 0 >= X
557       if (*Constant == 1) {
558         CC = RISCVCC::COND_GE;
559         RHS = LHS;
560         LHS = RISCV::X0;
561         return;
562       }
563       break;
564     default:
565       break;
566     }
567   }
568 
569   switch (Pred) {
570   default:
571     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
572   case CmpInst::Predicate::ICMP_EQ:
573   case CmpInst::Predicate::ICMP_NE:
574   case CmpInst::Predicate::ICMP_ULT:
575   case CmpInst::Predicate::ICMP_SLT:
576   case CmpInst::Predicate::ICMP_UGE:
577   case CmpInst::Predicate::ICMP_SGE:
578     // These CCs are supported directly by RISC-V branches.
579     break;
580   case CmpInst::Predicate::ICMP_SGT:
581   case CmpInst::Predicate::ICMP_SLE:
582   case CmpInst::Predicate::ICMP_UGT:
583   case CmpInst::Predicate::ICMP_ULE:
584     // These CCs are not supported directly by RISC-V branches, but changing the
585     // direction of the CC and swapping LHS and RHS are.
586     Pred = CmpInst::getSwappedPredicate(Pred);
587     std::swap(LHS, RHS);
588     break;
589   }
590 
591   CC = getRISCVCCFromICmp(Pred);
592 }
593 
594 bool RISCVInstructionSelector::select(MachineInstr &MI) {
595   MachineIRBuilder MIB(MI);
596 
597   preISelLower(MI, MIB);
598   const unsigned Opc = MI.getOpcode();
599 
600   if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
601     if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
602       const Register DefReg = MI.getOperand(0).getReg();
603       const LLT DefTy = MRI->getType(DefReg);
604 
605       const RegClassOrRegBank &RegClassOrBank =
606           MRI->getRegClassOrRegBank(DefReg);
607 
608       const TargetRegisterClass *DefRC =
609           dyn_cast<const TargetRegisterClass *>(RegClassOrBank);
610       if (!DefRC) {
611         if (!DefTy.isValid()) {
612           LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
613           return false;
614         }
615 
616         const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
617         DefRC = getRegClassForTypeOnBank(DefTy, RB);
618         if (!DefRC) {
619           LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
620           return false;
621         }
622       }
623 
624       MI.setDesc(TII.get(TargetOpcode::PHI));
625       return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
626     }
627 
628     // Certain non-generic instructions also need some special handling.
629     if (MI.isCopy())
630       return selectCopy(MI);
631 
632     return true;
633   }
634 
635   if (selectImpl(MI, *CoverageInfo))
636     return true;
637 
638   switch (Opc) {
639   case TargetOpcode::G_ANYEXT:
640   case TargetOpcode::G_PTRTOINT:
641   case TargetOpcode::G_INTTOPTR:
642   case TargetOpcode::G_TRUNC:
643   case TargetOpcode::G_FREEZE:
644     return selectCopy(MI);
645   case TargetOpcode::G_CONSTANT: {
646     Register DstReg = MI.getOperand(0).getReg();
647     int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
648 
649     if (!materializeImm(DstReg, Imm, MIB))
650       return false;
651 
652     MI.eraseFromParent();
653     return true;
654   }
655   case TargetOpcode::G_FCONSTANT: {
656     // TODO: Use constant pool for complext constants.
657     // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
658     Register DstReg = MI.getOperand(0).getReg();
659     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
660     APInt Imm = FPimm.bitcastToAPInt();
661     unsigned Size = MRI->getType(DstReg).getSizeInBits();
662     if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
663       Register GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
664       if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
665         return false;
666 
667       unsigned Opcode = Size == 64   ? RISCV::FMV_D_X
668                         : Size == 32 ? RISCV::FMV_W_X
669                                      : RISCV::FMV_H_X;
670       auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
671       if (!FMV.constrainAllUses(TII, TRI, RBI))
672         return false;
673     } else {
674       assert(Size == 64 && !Subtarget->is64Bit() &&
675              "Unexpected size or subtarget");
676       // Split into two pieces and build through the stack.
677       Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
678       Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
679       if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
680                           MIB))
681         return false;
682       if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
683         return false;
684       MachineInstrBuilder PairF64 = MIB.buildInstr(
685           RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
686       if (!PairF64.constrainAllUses(TII, TRI, RBI))
687         return false;
688     }
689 
690     MI.eraseFromParent();
691     return true;
692   }
693   case TargetOpcode::G_GLOBAL_VALUE: {
694     auto *GV = MI.getOperand(1).getGlobal();
695     if (GV->isThreadLocal()) {
696       // TODO: implement this case.
697       return false;
698     }
699 
700     return selectAddr(MI, MIB, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
701   }
702   case TargetOpcode::G_JUMP_TABLE:
703   case TargetOpcode::G_CONSTANT_POOL:
704     return selectAddr(MI, MIB, MRI);
705   case TargetOpcode::G_BRCOND: {
706     Register LHS, RHS;
707     RISCVCC::CondCode CC;
708     getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
709 
710     auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
711                    .addMBB(MI.getOperand(1).getMBB());
712     MI.eraseFromParent();
713     return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
714   }
715   case TargetOpcode::G_BRINDIRECT:
716     MI.setDesc(TII.get(RISCV::PseudoBRIND));
717     MI.addOperand(MachineOperand::CreateImm(0));
718     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
719   case TargetOpcode::G_SELECT:
720     return selectSelect(MI, MIB);
721   case TargetOpcode::G_FCMP:
722     return selectFPCompare(MI, MIB);
723   case TargetOpcode::G_FENCE: {
724     AtomicOrdering FenceOrdering =
725         static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
726     SyncScope::ID FenceSSID =
727         static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
728     emitFence(FenceOrdering, FenceSSID, MIB);
729     MI.eraseFromParent();
730     return true;
731   }
732   case TargetOpcode::G_IMPLICIT_DEF:
733     return selectImplicitDef(MI, MIB);
734   case TargetOpcode::G_UNMERGE_VALUES:
735     return selectUnmergeValues(MI, MIB);
736   default:
737     return false;
738   }
739 }
740 
741 bool RISCVInstructionSelector::selectUnmergeValues(
742     MachineInstr &MI, MachineIRBuilder &MIB) const {
743   assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
744 
745   if (!Subtarget->hasStdExtZfa())
746     return false;
747 
748   // Split F64 Src into two s32 parts
749   if (MI.getNumOperands() != 3)
750     return false;
751   Register Src = MI.getOperand(2).getReg();
752   Register Lo = MI.getOperand(0).getReg();
753   Register Hi = MI.getOperand(1).getReg();
754   if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
755     return false;
756 
757   MachineInstr *ExtractLo = MIB.buildInstr(RISCV::FMV_X_W_FPR64, {Lo}, {Src});
758   if (!constrainSelectedInstRegOperands(*ExtractLo, TII, TRI, RBI))
759     return false;
760 
761   MachineInstr *ExtractHi = MIB.buildInstr(RISCV::FMVH_X_D, {Hi}, {Src});
762   if (!constrainSelectedInstRegOperands(*ExtractHi, TII, TRI, RBI))
763     return false;
764 
765   MI.eraseFromParent();
766   return true;
767 }
768 
769 bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
770                                                  MachineIRBuilder &MIB) {
771   Register PtrReg = Op.getReg();
772   assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
773 
774   const LLT sXLen = LLT::scalar(STI.getXLen());
775   auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
776   MRI->setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
777   Op.setReg(PtrToInt.getReg(0));
778   return select(*PtrToInt);
779 }
780 
781 void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
782                                             MachineIRBuilder &MIB) {
783   switch (MI.getOpcode()) {
784   case TargetOpcode::G_PTR_ADD: {
785     Register DstReg = MI.getOperand(0).getReg();
786     const LLT sXLen = LLT::scalar(STI.getXLen());
787 
788     replacePtrWithInt(MI.getOperand(1), MIB);
789     MI.setDesc(TII.get(TargetOpcode::G_ADD));
790     MRI->setType(DstReg, sXLen);
791     break;
792   }
793   case TargetOpcode::G_PTRMASK: {
794     Register DstReg = MI.getOperand(0).getReg();
795     const LLT sXLen = LLT::scalar(STI.getXLen());
796     replacePtrWithInt(MI.getOperand(1), MIB);
797     MI.setDesc(TII.get(TargetOpcode::G_AND));
798     MRI->setType(DstReg, sXLen);
799     break;
800   }
801   }
802 }
803 
804 void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
805                                             const MachineInstr &MI,
806                                             int OpIdx) const {
807   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
808          "Expected G_CONSTANT");
809   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
810   MIB.addImm(-CstVal);
811 }
812 
813 void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
814                                                     const MachineInstr &MI,
815                                                     int OpIdx) const {
816   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
817          "Expected G_CONSTANT");
818   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
819   MIB.addImm(STI.getXLen() - CstVal);
820 }
821 
822 void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
823                                                   const MachineInstr &MI,
824                                                   int OpIdx) const {
825   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
826          "Expected G_CONSTANT");
827   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
828   MIB.addImm(32 - CstVal);
829 }
830 
831 void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
832                                               const MachineInstr &MI,
833                                               int OpIdx) const {
834   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
835          "Expected G_CONSTANT");
836   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
837   MIB.addImm(CstVal + 1);
838 }
839 
840 void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
841                                          const MachineInstr &MI,
842                                          int OpIdx) const {
843   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
844          "Expected G_CONSTANT");
845   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
846   MIB.addImm(CstVal);
847 }
848 
849 void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
850                                                 const MachineInstr &MI,
851                                                 int OpIdx) const {
852   assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
853          "Expected G_FRAME_INDEX");
854   MIB.add(MI.getOperand(1));
855 }
856 
857 void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
858                                                    const MachineInstr &MI,
859                                                    int OpIdx) const {
860   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
861          "Expected G_CONSTANT");
862   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
863   MIB.addImm(llvm::countr_zero(C));
864 }
865 
866 void RISCVInstructionSelector::renderXLenSubTrailingOnes(
867     MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
868   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
869          "Expected G_CONSTANT");
870   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
871   MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
872 }
873 
874 void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
875                                                       const MachineInstr &MI,
876                                                       int OpIdx) const {
877   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
878          "Expected G_CONSTANT");
879   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
880   int64_t Adj = Imm < 0 ? -2048 : 2047;
881   MIB.addImm(Imm - Adj);
882 }
883 
884 void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
885                                                       const MachineInstr &MI,
886                                                       int OpIdx) const {
887   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
888          "Expected G_CONSTANT");
889   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
890   MIB.addImm(Imm);
891 }
892 
893 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
894     LLT Ty, const RegisterBank &RB) const {
895   if (RB.getID() == RISCV::GPRBRegBankID) {
896     if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
897       return &RISCV::GPRRegClass;
898   }
899 
900   if (RB.getID() == RISCV::FPRBRegBankID) {
901     if (Ty.getSizeInBits() == 16)
902       return &RISCV::FPR16RegClass;
903     if (Ty.getSizeInBits() == 32)
904       return &RISCV::FPR32RegClass;
905     if (Ty.getSizeInBits() == 64)
906       return &RISCV::FPR64RegClass;
907   }
908 
909   if (RB.getID() == RISCV::VRBRegBankID) {
910     if (Ty.getSizeInBits().getKnownMinValue() <= 64)
911       return &RISCV::VRRegClass;
912 
913     if (Ty.getSizeInBits().getKnownMinValue() == 128)
914       return &RISCV::VRM2RegClass;
915 
916     if (Ty.getSizeInBits().getKnownMinValue() == 256)
917       return &RISCV::VRM4RegClass;
918 
919     if (Ty.getSizeInBits().getKnownMinValue() == 512)
920       return &RISCV::VRM8RegClass;
921   }
922 
923   return nullptr;
924 }
925 
926 bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
927   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
928 }
929 
930 bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
931   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
932 }
933 
934 bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
935   Register DstReg = MI.getOperand(0).getReg();
936 
937   if (DstReg.isPhysical())
938     return true;
939 
940   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
941       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
942   assert(DstRC &&
943          "Register class not available for LLT, register bank combination");
944 
945   // No need to constrain SrcReg. It will get constrained when
946   // we hit another of its uses or its defs.
947   // Copies do not have constraints.
948   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
949     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
950                       << " operand\n");
951     return false;
952   }
953 
954   MI.setDesc(TII.get(RISCV::COPY));
955   return true;
956 }
957 
958 bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
959                                                  MachineIRBuilder &MIB) const {
960   assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
961 
962   const Register DstReg = MI.getOperand(0).getReg();
963   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
964       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
965 
966   assert(DstRC &&
967          "Register class not available for LLT, register bank combination");
968 
969   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
970     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
971                       << " operand\n");
972   }
973   MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
974   return true;
975 }
976 
977 bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
978                                               MachineIRBuilder &MIB) const {
979   if (Imm == 0) {
980     MIB.buildCopy(DstReg, Register(RISCV::X0));
981     RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
982     return true;
983   }
984 
985   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
986   unsigned NumInsts = Seq.size();
987   Register SrcReg = RISCV::X0;
988 
989   for (unsigned i = 0; i < NumInsts; i++) {
990     Register TmpReg = i < NumInsts - 1
991                           ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
992                           : DstReg;
993     const RISCVMatInt::Inst &I = Seq[i];
994     MachineInstr *Result;
995 
996     switch (I.getOpndKind()) {
997     case RISCVMatInt::Imm:
998       // clang-format off
999       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
1000                    .addImm(I.getImm());
1001       // clang-format on
1002       break;
1003     case RISCVMatInt::RegX0:
1004       Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
1005                               {SrcReg, Register(RISCV::X0)});
1006       break;
1007     case RISCVMatInt::RegReg:
1008       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
1009       break;
1010     case RISCVMatInt::RegImm:
1011       Result =
1012           MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
1013       break;
1014     }
1015 
1016     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1017       return false;
1018 
1019     SrcReg = TmpReg;
1020   }
1021 
1022   return true;
1023 }
1024 
1025 bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
1026                                           MachineIRBuilder &MIB, bool IsLocal,
1027                                           bool IsExternWeak) const {
1028   assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1029           MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1030           MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1031          "Unexpected opcode");
1032 
1033   const MachineOperand &DispMO = MI.getOperand(1);
1034 
1035   Register DefReg = MI.getOperand(0).getReg();
1036   const LLT DefTy = MRI->getType(DefReg);
1037 
1038   // When HWASAN is used and tagging of global variables is enabled
1039   // they should be accessed via the GOT, since the tagged address of a global
1040   // is incompatible with existing code models. This also applies to non-pic
1041   // mode.
1042   if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1043     if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1044       // Use PC-relative addressing to access the symbol. This generates the
1045       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1046       // %pcrel_lo(auipc)).
1047       MI.setDesc(TII.get(RISCV::PseudoLLA));
1048       return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1049     }
1050 
1051     // Use PC-relative addressing to access the GOT for this symbol, then
1052     // load the address from the GOT. This generates the pattern (PseudoLGA
1053     // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1054     // %pcrel_lo(auipc))).
1055     MachineFunction &MF = *MI.getParent()->getParent();
1056     MachineMemOperand *MemOp = MF.getMachineMemOperand(
1057         MachinePointerInfo::getGOT(MF),
1058         MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1059             MachineMemOperand::MOInvariant,
1060         DefTy, Align(DefTy.getSizeInBits() / 8));
1061 
1062     auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1063                       .addDisp(DispMO, 0)
1064                       .addMemOperand(MemOp);
1065 
1066     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1067       return false;
1068 
1069     MI.eraseFromParent();
1070     return true;
1071   }
1072 
1073   switch (TM.getCodeModel()) {
1074   default: {
1075     reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1076                        getName(), "Unsupported code model for lowering", MI);
1077     return false;
1078   }
1079   case CodeModel::Small: {
1080     // Must lie within a single 2 GiB address range and must lie between
1081     // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1082     // (lui %hi(sym)) %lo(sym)).
1083     Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1084     MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1085                                .addDisp(DispMO, 0, RISCVII::MO_HI);
1086 
1087     if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1088       return false;
1089 
1090     auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1091                       .addDisp(DispMO, 0, RISCVII::MO_LO);
1092 
1093     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1094       return false;
1095 
1096     MI.eraseFromParent();
1097     return true;
1098   }
1099   case CodeModel::Medium:
1100     // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1101     // relocation needs to reference a label that points to the auipc
1102     // instruction itself, not the global. This cannot be done inside the
1103     // instruction selector.
1104     if (IsExternWeak) {
1105       // An extern weak symbol may be undefined, i.e. have value 0, which may
1106       // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1107       // symbol. This generates the pattern (PseudoLGA sym), which expands to
1108       // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1109       MachineFunction &MF = *MI.getParent()->getParent();
1110       MachineMemOperand *MemOp = MF.getMachineMemOperand(
1111           MachinePointerInfo::getGOT(MF),
1112           MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1113               MachineMemOperand::MOInvariant,
1114           DefTy, Align(DefTy.getSizeInBits() / 8));
1115 
1116       auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1117                         .addDisp(DispMO, 0)
1118                         .addMemOperand(MemOp);
1119 
1120       if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1121         return false;
1122 
1123       MI.eraseFromParent();
1124       return true;
1125     }
1126 
1127     // Generate a sequence for accessing addresses within any 2GiB range
1128     // within the address space. This generates the pattern (PseudoLLA sym),
1129     // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1130     MI.setDesc(TII.get(RISCV::PseudoLLA));
1131     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1132   }
1133 
1134   return false;
1135 }
1136 
1137 bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1138                                             MachineIRBuilder &MIB) const {
1139   auto &SelectMI = cast<GSelect>(MI);
1140 
1141   Register LHS, RHS;
1142   RISCVCC::CondCode CC;
1143   getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
1144 
1145   Register DstReg = SelectMI.getReg(0);
1146 
1147   unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1148   if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1149     unsigned Size = MRI->getType(DstReg).getSizeInBits();
1150     Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1151                      : RISCV::Select_FPR64_Using_CC_GPR;
1152   }
1153 
1154   MachineInstr *Result = MIB.buildInstr(Opc)
1155                              .addDef(DstReg)
1156                              .addReg(LHS)
1157                              .addReg(RHS)
1158                              .addImm(CC)
1159                              .addReg(SelectMI.getTrueReg())
1160                              .addReg(SelectMI.getFalseReg());
1161   MI.eraseFromParent();
1162   return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1163 }
1164 
1165 // Convert an FCMP predicate to one of the supported F or D instructions.
1166 static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1167   assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1168   switch (Pred) {
1169   default:
1170     llvm_unreachable("Unsupported predicate");
1171   case CmpInst::FCMP_OLT:
1172     return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1173   case CmpInst::FCMP_OLE:
1174     return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1175   case CmpInst::FCMP_OEQ:
1176     return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1177   }
1178 }
1179 
1180 // Try legalizing an FCMP by swapping or inverting the predicate to one that
1181 // is supported.
1182 static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1183                                   CmpInst::Predicate &Pred, bool &NeedInvert) {
1184   auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1185     return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1186            Pred == CmpInst::FCMP_OEQ;
1187   };
1188 
1189   assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1190 
1191   CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
1192   if (isLegalFCmpPredicate(InvPred)) {
1193     Pred = InvPred;
1194     std::swap(LHS, RHS);
1195     return true;
1196   }
1197 
1198   InvPred = CmpInst::getInversePredicate(Pred);
1199   NeedInvert = true;
1200   if (isLegalFCmpPredicate(InvPred)) {
1201     Pred = InvPred;
1202     return true;
1203   }
1204   InvPred = CmpInst::getSwappedPredicate(InvPred);
1205   if (isLegalFCmpPredicate(InvPred)) {
1206     Pred = InvPred;
1207     std::swap(LHS, RHS);
1208     return true;
1209   }
1210 
1211   return false;
1212 }
1213 
1214 // Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1215 // the result in DstReg.
1216 // FIXME: Maybe we should expand this earlier.
1217 bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1218                                                MachineIRBuilder &MIB) const {
1219   auto &CmpMI = cast<GFCmp>(MI);
1220   CmpInst::Predicate Pred = CmpMI.getCond();
1221 
1222   Register DstReg = CmpMI.getReg(0);
1223   Register LHS = CmpMI.getLHSReg();
1224   Register RHS = CmpMI.getRHSReg();
1225 
1226   unsigned Size = MRI->getType(LHS).getSizeInBits();
1227   assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1228 
1229   Register TmpReg = DstReg;
1230 
1231   bool NeedInvert = false;
1232   // First try swapping operands or inverting.
1233   if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1234     if (NeedInvert)
1235       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1236     auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1237     if (!Cmp.constrainAllUses(TII, TRI, RBI))
1238       return false;
1239   } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1240     // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1241     NeedInvert = Pred == CmpInst::FCMP_UEQ;
1242     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1243                                {&RISCV::GPRRegClass}, {LHS, RHS});
1244     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1245       return false;
1246     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1247                                {&RISCV::GPRRegClass}, {RHS, LHS});
1248     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1249       return false;
1250     if (NeedInvert)
1251       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1252     auto Or =
1253         MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1254     if (!Or.constrainAllUses(TII, TRI, RBI))
1255       return false;
1256   } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1257     // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1258     // FIXME: If LHS and RHS are the same we can use a single FEQ.
1259     NeedInvert = Pred == CmpInst::FCMP_UNO;
1260     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1261                                {&RISCV::GPRRegClass}, {LHS, LHS});
1262     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1263       return false;
1264     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1265                                {&RISCV::GPRRegClass}, {RHS, RHS});
1266     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1267       return false;
1268     if (NeedInvert)
1269       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1270     auto And =
1271         MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1272     if (!And.constrainAllUses(TII, TRI, RBI))
1273       return false;
1274   } else
1275     llvm_unreachable("Unhandled predicate");
1276 
1277   // Emit an XORI to invert the result if needed.
1278   if (NeedInvert) {
1279     auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1280     if (!Xor.constrainAllUses(TII, TRI, RBI))
1281       return false;
1282   }
1283 
1284   MI.eraseFromParent();
1285   return true;
1286 }
1287 
1288 void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1289                                          SyncScope::ID FenceSSID,
1290                                          MachineIRBuilder &MIB) const {
1291   if (STI.hasStdExtZtso()) {
1292     // The only fence that needs an instruction is a sequentially-consistent
1293     // cross-thread fence.
1294     if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1295         FenceSSID == SyncScope::System) {
1296       // fence rw, rw
1297       MIB.buildInstr(RISCV::FENCE, {}, {})
1298           .addImm(RISCVFenceField::R | RISCVFenceField::W)
1299           .addImm(RISCVFenceField::R | RISCVFenceField::W);
1300       return;
1301     }
1302 
1303     // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1304     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1305     return;
1306   }
1307 
1308   // singlethread fences only synchronize with signal handlers on the same
1309   // thread and thus only need to preserve instruction order, not actually
1310   // enforce memory ordering.
1311   if (FenceSSID == SyncScope::SingleThread) {
1312     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1313     return;
1314   }
1315 
1316   // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1317   // Manual: Volume I.
1318   unsigned Pred, Succ;
1319   switch (FenceOrdering) {
1320   default:
1321     llvm_unreachable("Unexpected ordering");
1322   case AtomicOrdering::AcquireRelease:
1323     // fence acq_rel -> fence.tso
1324     MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1325     return;
1326   case AtomicOrdering::Acquire:
1327     // fence acquire -> fence r, rw
1328     Pred = RISCVFenceField::R;
1329     Succ = RISCVFenceField::R | RISCVFenceField::W;
1330     break;
1331   case AtomicOrdering::Release:
1332     // fence release -> fence rw, w
1333     Pred = RISCVFenceField::R | RISCVFenceField::W;
1334     Succ = RISCVFenceField::W;
1335     break;
1336   case AtomicOrdering::SequentiallyConsistent:
1337     // fence seq_cst -> fence rw, rw
1338     Pred = RISCVFenceField::R | RISCVFenceField::W;
1339     Succ = RISCVFenceField::R | RISCVFenceField::W;
1340     break;
1341   }
1342   MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1343 }
1344 
1345 namespace llvm {
1346 InstructionSelector *
1347 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1348                                const RISCVSubtarget &Subtarget,
1349                                const RISCVRegisterBankInfo &RBI) {
1350   return new RISCVInstructionSelector(TM, Subtarget, RBI);
1351 }
1352 } // end namespace llvm
1353