xref: /llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision 0efe111365ae176671e01252d24028047d807a84)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/SmallSet.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/AliasAnalysis.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/ConstantFolding.h"
26 #include "llvm/Analysis/Loads.h"
27 #include "llvm/Analysis/MemoryLocation.h"
28 #include "llvm/Analysis/TargetLibraryInfo.h"
29 #include "llvm/Analysis/ValueTracking.h"
30 #include "llvm/Analysis/VectorUtils.h"
31 #include "llvm/CodeGen/Analysis.h"
32 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
33 #include "llvm/CodeGen/CodeGenCommonISel.h"
34 #include "llvm/CodeGen/FunctionLoweringInfo.h"
35 #include "llvm/CodeGen/GCMetadata.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFrameInfo.h"
38 #include "llvm/CodeGen/MachineFunction.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineInstrBundleIterator.h"
41 #include "llvm/CodeGen/MachineMemOperand.h"
42 #include "llvm/CodeGen/MachineModuleInfo.h"
43 #include "llvm/CodeGen/MachineOperand.h"
44 #include "llvm/CodeGen/MachineRegisterInfo.h"
45 #include "llvm/CodeGen/RuntimeLibcalls.h"
46 #include "llvm/CodeGen/SelectionDAG.h"
47 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
48 #include "llvm/CodeGen/StackMaps.h"
49 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
50 #include "llvm/CodeGen/TargetFrameLowering.h"
51 #include "llvm/CodeGen/TargetInstrInfo.h"
52 #include "llvm/CodeGen/TargetOpcodes.h"
53 #include "llvm/CodeGen/TargetRegisterInfo.h"
54 #include "llvm/CodeGen/TargetSubtargetInfo.h"
55 #include "llvm/CodeGen/WinEHFuncInfo.h"
56 #include "llvm/IR/Argument.h"
57 #include "llvm/IR/Attributes.h"
58 #include "llvm/IR/BasicBlock.h"
59 #include "llvm/IR/CFG.h"
60 #include "llvm/IR/CallingConv.h"
61 #include "llvm/IR/Constant.h"
62 #include "llvm/IR/ConstantRange.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
65 #include "llvm/IR/DebugInfo.h"
66 #include "llvm/IR/DebugInfoMetadata.h"
67 #include "llvm/IR/DerivedTypes.h"
68 #include "llvm/IR/DiagnosticInfo.h"
69 #include "llvm/IR/EHPersonalities.h"
70 #include "llvm/IR/Function.h"
71 #include "llvm/IR/GetElementPtrTypeIterator.h"
72 #include "llvm/IR/InlineAsm.h"
73 #include "llvm/IR/InstrTypes.h"
74 #include "llvm/IR/Instructions.h"
75 #include "llvm/IR/IntrinsicInst.h"
76 #include "llvm/IR/Intrinsics.h"
77 #include "llvm/IR/IntrinsicsAArch64.h"
78 #include "llvm/IR/IntrinsicsWebAssembly.h"
79 #include "llvm/IR/LLVMContext.h"
80 #include "llvm/IR/Metadata.h"
81 #include "llvm/IR/Module.h"
82 #include "llvm/IR/Operator.h"
83 #include "llvm/IR/PatternMatch.h"
84 #include "llvm/IR/Statepoint.h"
85 #include "llvm/IR/Type.h"
86 #include "llvm/IR/User.h"
87 #include "llvm/IR/Value.h"
88 #include "llvm/MC/MCContext.h"
89 #include "llvm/Support/AtomicOrdering.h"
90 #include "llvm/Support/Casting.h"
91 #include "llvm/Support/CommandLine.h"
92 #include "llvm/Support/Compiler.h"
93 #include "llvm/Support/Debug.h"
94 #include "llvm/Support/MathExtras.h"
95 #include "llvm/Support/raw_ostream.h"
96 #include "llvm/Target/TargetIntrinsicInfo.h"
97 #include "llvm/Target/TargetMachine.h"
98 #include "llvm/Target/TargetOptions.h"
99 #include "llvm/TargetParser/Triple.h"
100 #include "llvm/Transforms/Utils/Local.h"
101 #include <cstddef>
102 #include <iterator>
103 #include <limits>
104 #include <optional>
105 #include <tuple>
106 
107 using namespace llvm;
108 using namespace PatternMatch;
109 using namespace SwitchCG;
110 
111 #define DEBUG_TYPE "isel"
112 
113 /// LimitFloatPrecision - Generate low-precision inline sequences for
114 /// some float libcalls (6, 8 or 12 bits).
115 static unsigned LimitFloatPrecision;
116 
117 static cl::opt<bool>
118     InsertAssertAlign("insert-assert-align", cl::init(true),
119                       cl::desc("Insert the experimental `assertalign` node."),
120                       cl::ReallyHidden);
121 
122 static cl::opt<unsigned, true>
123     LimitFPPrecision("limit-float-precision",
124                      cl::desc("Generate low-precision inline sequences "
125                               "for some float libcalls"),
126                      cl::location(LimitFloatPrecision), cl::Hidden,
127                      cl::init(0));
128 
129 static cl::opt<unsigned> SwitchPeelThreshold(
130     "switch-peel-threshold", cl::Hidden, cl::init(66),
131     cl::desc("Set the case probability threshold for peeling the case from a "
132              "switch statement. A value greater than 100 will void this "
133              "optimization"));
134 
135 // Limit the width of DAG chains. This is important in general to prevent
136 // DAG-based analysis from blowing up. For example, alias analysis and
137 // load clustering may not complete in reasonable time. It is difficult to
138 // recognize and avoid this situation within each individual analysis, and
139 // future analyses are likely to have the same behavior. Limiting DAG width is
140 // the safe approach and will be especially important with global DAGs.
141 //
142 // MaxParallelChains default is arbitrarily high to avoid affecting
143 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
144 // sequence over this should have been converted to llvm.memcpy by the
145 // frontend. It is easy to induce this behavior with .ll code such as:
146 // %buffer = alloca [4096 x i8]
147 // %data = load [4096 x i8]* %argPtr
148 // store [4096 x i8] %data, [4096 x i8]* %buffer
149 static const unsigned MaxParallelChains = 64;
150 
151 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
152                                       const SDValue *Parts, unsigned NumParts,
153                                       MVT PartVT, EVT ValueVT, const Value *V,
154                                       std::optional<CallingConv::ID> CC);
155 
156 /// getCopyFromParts - Create a value that contains the specified legal parts
157 /// combined into the value they represent.  If the parts combine to a type
158 /// larger than ValueVT then AssertOp can be used to specify whether the extra
159 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
160 /// (ISD::AssertSext).
161 static SDValue
162 getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
163                  unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
164                  std::optional<CallingConv::ID> CC = std::nullopt,
165                  std::optional<ISD::NodeType> AssertOp = std::nullopt) {
166   // Let the target assemble the parts if it wants to
167   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
168   if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
169                                                    PartVT, ValueVT, CC))
170     return Val;
171 
172   if (ValueVT.isVector())
173     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
174                                   CC);
175 
176   assert(NumParts > 0 && "No parts to assemble!");
177   SDValue Val = Parts[0];
178 
179   if (NumParts > 1) {
180     // Assemble the value from multiple parts.
181     if (ValueVT.isInteger()) {
182       unsigned PartBits = PartVT.getSizeInBits();
183       unsigned ValueBits = ValueVT.getSizeInBits();
184 
185       // Assemble the power of 2 part.
186       unsigned RoundParts = llvm::bit_floor(NumParts);
187       unsigned RoundBits = PartBits * RoundParts;
188       EVT RoundVT = RoundBits == ValueBits ?
189         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
190       SDValue Lo, Hi;
191 
192       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
193 
194       if (RoundParts > 2) {
195         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
196                               PartVT, HalfVT, V);
197         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
198                               RoundParts / 2, PartVT, HalfVT, V);
199       } else {
200         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
201         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
202       }
203 
204       if (DAG.getDataLayout().isBigEndian())
205         std::swap(Lo, Hi);
206 
207       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
208 
209       if (RoundParts < NumParts) {
210         // Assemble the trailing non-power-of-2 part.
211         unsigned OddParts = NumParts - RoundParts;
212         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
213         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
214                               OddVT, V, CC);
215 
216         // Combine the round and odd parts.
217         Lo = Val;
218         if (DAG.getDataLayout().isBigEndian())
219           std::swap(Lo, Hi);
220         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
221         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
222         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
223                          DAG.getConstant(Lo.getValueSizeInBits(), DL,
224                                          TLI.getShiftAmountTy(
225                                              TotalVT, DAG.getDataLayout())));
226         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
227         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
228       }
229     } else if (PartVT.isFloatingPoint()) {
230       // FP split into multiple FP parts (for ppcf128)
231       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
232              "Unexpected split");
233       SDValue Lo, Hi;
234       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
235       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
236       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
237         std::swap(Lo, Hi);
238       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
239     } else {
240       // FP split into integer parts (soft fp)
241       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
242              !PartVT.isVector() && "Unexpected split");
243       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
244       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
245     }
246   }
247 
248   // There is now one part, held in Val.  Correct it to match ValueVT.
249   // PartEVT is the type of the register class that holds the value.
250   // ValueVT is the type of the inline asm operation.
251   EVT PartEVT = Val.getValueType();
252 
253   if (PartEVT == ValueVT)
254     return Val;
255 
256   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
257       ValueVT.bitsLT(PartEVT)) {
258     // For an FP value in an integer part, we need to truncate to the right
259     // width first.
260     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
261     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
262   }
263 
264   // Handle types that have the same size.
265   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
266     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
267 
268   // Handle types with different sizes.
269   if (PartEVT.isInteger() && ValueVT.isInteger()) {
270     if (ValueVT.bitsLT(PartEVT)) {
271       // For a truncate, see if we have any information to
272       // indicate whether the truncated bits will always be
273       // zero or sign-extension.
274       if (AssertOp)
275         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
276                           DAG.getValueType(ValueVT));
277       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
278     }
279     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
280   }
281 
282   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
283     // FP_ROUND's are always exact here.
284     if (ValueVT.bitsLT(Val.getValueType()))
285       return DAG.getNode(
286           ISD::FP_ROUND, DL, ValueVT, Val,
287           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
288 
289     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
290   }
291 
292   // Handle MMX to a narrower integer type by bitcasting MMX to integer and
293   // then truncating.
294   if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
295       ValueVT.bitsLT(PartEVT)) {
296     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
297     return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
298   }
299 
300   report_fatal_error("Unknown mismatch in getCopyFromParts!");
301 }
302 
303 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
304                                               const Twine &ErrMsg) {
305   const Instruction *I = dyn_cast_or_null<Instruction>(V);
306   if (!V)
307     return Ctx.emitError(ErrMsg);
308 
309   const char *AsmError = ", possible invalid constraint for vector type";
310   if (const CallInst *CI = dyn_cast<CallInst>(I))
311     if (CI->isInlineAsm())
312       return Ctx.emitError(I, ErrMsg + AsmError);
313 
314   return Ctx.emitError(I, ErrMsg);
315 }
316 
317 /// getCopyFromPartsVector - Create a value that contains the specified legal
318 /// parts combined into the value they represent.  If the parts combine to a
319 /// type larger than ValueVT then AssertOp can be used to specify whether the
320 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
321 /// ValueVT (ISD::AssertSext).
322 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
323                                       const SDValue *Parts, unsigned NumParts,
324                                       MVT PartVT, EVT ValueVT, const Value *V,
325                                       std::optional<CallingConv::ID> CallConv) {
326   assert(ValueVT.isVector() && "Not a vector value");
327   assert(NumParts > 0 && "No parts to assemble!");
328   const bool IsABIRegCopy = CallConv.has_value();
329 
330   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
331   SDValue Val = Parts[0];
332 
333   // Handle a multi-element vector.
334   if (NumParts > 1) {
335     EVT IntermediateVT;
336     MVT RegisterVT;
337     unsigned NumIntermediates;
338     unsigned NumRegs;
339 
340     if (IsABIRegCopy) {
341       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
342           *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
343           NumIntermediates, RegisterVT);
344     } else {
345       NumRegs =
346           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
347                                      NumIntermediates, RegisterVT);
348     }
349 
350     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
351     NumParts = NumRegs; // Silence a compiler warning.
352     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
353     assert(RegisterVT.getSizeInBits() ==
354            Parts[0].getSimpleValueType().getSizeInBits() &&
355            "Part type sizes don't match!");
356 
357     // Assemble the parts into intermediate operands.
358     SmallVector<SDValue, 8> Ops(NumIntermediates);
359     if (NumIntermediates == NumParts) {
360       // If the register was not expanded, truncate or copy the value,
361       // as appropriate.
362       for (unsigned i = 0; i != NumParts; ++i)
363         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
364                                   PartVT, IntermediateVT, V, CallConv);
365     } else if (NumParts > 0) {
366       // If the intermediate type was expanded, build the intermediate
367       // operands from the parts.
368       assert(NumParts % NumIntermediates == 0 &&
369              "Must expand into a divisible number of parts!");
370       unsigned Factor = NumParts / NumIntermediates;
371       for (unsigned i = 0; i != NumIntermediates; ++i)
372         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
373                                   PartVT, IntermediateVT, V, CallConv);
374     }
375 
376     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
377     // intermediate operands.
378     EVT BuiltVectorTy =
379         IntermediateVT.isVector()
380             ? EVT::getVectorVT(
381                   *DAG.getContext(), IntermediateVT.getScalarType(),
382                   IntermediateVT.getVectorElementCount() * NumParts)
383             : EVT::getVectorVT(*DAG.getContext(),
384                                IntermediateVT.getScalarType(),
385                                NumIntermediates);
386     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
387                                                 : ISD::BUILD_VECTOR,
388                       DL, BuiltVectorTy, Ops);
389   }
390 
391   // There is now one part, held in Val.  Correct it to match ValueVT.
392   EVT PartEVT = Val.getValueType();
393 
394   if (PartEVT == ValueVT)
395     return Val;
396 
397   if (PartEVT.isVector()) {
398     // Vector/Vector bitcast.
399     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
400       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
401 
402     // If the parts vector has more elements than the value vector, then we
403     // have a vector widening case (e.g. <2 x float> -> <4 x float>).
404     // Extract the elements we want.
405     if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
406       assert((PartEVT.getVectorElementCount().getKnownMinValue() >
407               ValueVT.getVectorElementCount().getKnownMinValue()) &&
408              (PartEVT.getVectorElementCount().isScalable() ==
409               ValueVT.getVectorElementCount().isScalable()) &&
410              "Cannot narrow, it would be a lossy transformation");
411       PartEVT =
412           EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
413                            ValueVT.getVectorElementCount());
414       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
415                         DAG.getVectorIdxConstant(0, DL));
416       if (PartEVT == ValueVT)
417         return Val;
418       if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
419         return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
420     }
421 
422     // Promoted vector extract
423     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
424   }
425 
426   // Trivial bitcast if the types are the same size and the destination
427   // vector type is legal.
428   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
429       TLI.isTypeLegal(ValueVT))
430     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
431 
432   if (ValueVT.getVectorNumElements() != 1) {
433      // Certain ABIs require that vectors are passed as integers. For vectors
434      // are the same size, this is an obvious bitcast.
435      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
436        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
437      } else if (ValueVT.bitsLT(PartEVT)) {
438        const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
439        EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
440        // Drop the extra bits.
441        Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
442        return DAG.getBitcast(ValueVT, Val);
443      }
444 
445      diagnosePossiblyInvalidConstraint(
446          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
447      return DAG.getUNDEF(ValueVT);
448   }
449 
450   // Handle cases such as i8 -> <1 x i1>
451   EVT ValueSVT = ValueVT.getVectorElementType();
452   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
453     unsigned ValueSize = ValueSVT.getSizeInBits();
454     if (ValueSize == PartEVT.getSizeInBits()) {
455       Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
456     } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
457       // It's possible a scalar floating point type gets softened to integer and
458       // then promoted to a larger integer. If PartEVT is the larger integer
459       // we need to truncate it and then bitcast to the FP type.
460       assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
461       EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462       Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
463       Val = DAG.getBitcast(ValueSVT, Val);
464     } else {
465       Val = ValueVT.isFloatingPoint()
466                 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
467                 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
468     }
469   }
470 
471   return DAG.getBuildVector(ValueVT, DL, Val);
472 }
473 
474 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
475                                  SDValue Val, SDValue *Parts, unsigned NumParts,
476                                  MVT PartVT, const Value *V,
477                                  std::optional<CallingConv::ID> CallConv);
478 
479 /// getCopyToParts - Create a series of nodes that contain the specified value
480 /// split into legal parts.  If the parts contain more bits than Val, then, for
481 /// integers, ExtendKind can be used to specify how to generate the extra bits.
482 static void
483 getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
484                unsigned NumParts, MVT PartVT, const Value *V,
485                std::optional<CallingConv::ID> CallConv = std::nullopt,
486                ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
487   // Let the target split the parts if it wants to
488   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
489   if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
490                                       CallConv))
491     return;
492   EVT ValueVT = Val.getValueType();
493 
494   // Handle the vector case separately.
495   if (ValueVT.isVector())
496     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
497                                 CallConv);
498 
499   unsigned OrigNumParts = NumParts;
500   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
501          "Copying to an illegal type!");
502 
503   if (NumParts == 0)
504     return;
505 
506   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
507   EVT PartEVT = PartVT;
508   if (PartEVT == ValueVT) {
509     assert(NumParts == 1 && "No-op copy with multiple parts!");
510     Parts[0] = Val;
511     return;
512   }
513 
514   unsigned PartBits = PartVT.getSizeInBits();
515   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
516     // If the parts cover more bits than the value has, promote the value.
517     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
518       assert(NumParts == 1 && "Do not know what to promote to!");
519       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
520     } else {
521       if (ValueVT.isFloatingPoint()) {
522         // FP values need to be bitcast, then extended if they are being put
523         // into a larger container.
524         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
525         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
526       }
527       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
528              ValueVT.isInteger() &&
529              "Unknown mismatch!");
530       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
531       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
532       if (PartVT == MVT::x86mmx)
533         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
534     }
535   } else if (PartBits == ValueVT.getSizeInBits()) {
536     // Different types of the same size.
537     assert(NumParts == 1 && PartEVT != ValueVT);
538     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
539   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
540     // If the parts cover less bits than value has, truncate the value.
541     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
542            ValueVT.isInteger() &&
543            "Unknown mismatch!");
544     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
545     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
546     if (PartVT == MVT::x86mmx)
547       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
548   }
549 
550   // The value may have changed - recompute ValueVT.
551   ValueVT = Val.getValueType();
552   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
553          "Failed to tile the value with PartVT!");
554 
555   if (NumParts == 1) {
556     if (PartEVT != ValueVT) {
557       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
558                                         "scalar-to-vector conversion failed");
559       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
560     }
561 
562     Parts[0] = Val;
563     return;
564   }
565 
566   // Expand the value into multiple parts.
567   if (NumParts & (NumParts - 1)) {
568     // The number of parts is not a power of 2.  Split off and copy the tail.
569     assert(PartVT.isInteger() && ValueVT.isInteger() &&
570            "Do not know what to expand to!");
571     unsigned RoundParts = llvm::bit_floor(NumParts);
572     unsigned RoundBits = RoundParts * PartBits;
573     unsigned OddParts = NumParts - RoundParts;
574     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
575       DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
576 
577     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
578                    CallConv);
579 
580     if (DAG.getDataLayout().isBigEndian())
581       // The odd parts were reversed by getCopyToParts - unreverse them.
582       std::reverse(Parts + RoundParts, Parts + NumParts);
583 
584     NumParts = RoundParts;
585     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
586     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
587   }
588 
589   // The number of parts is a power of 2.  Repeatedly bisect the value using
590   // EXTRACT_ELEMENT.
591   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
592                          EVT::getIntegerVT(*DAG.getContext(),
593                                            ValueVT.getSizeInBits()),
594                          Val);
595 
596   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
597     for (unsigned i = 0; i < NumParts; i += StepSize) {
598       unsigned ThisBits = StepSize * PartBits / 2;
599       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
600       SDValue &Part0 = Parts[i];
601       SDValue &Part1 = Parts[i+StepSize/2];
602 
603       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
605       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
606                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
607 
608       if (ThisBits == PartBits && ThisVT != PartVT) {
609         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
610         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
611       }
612     }
613   }
614 
615   if (DAG.getDataLayout().isBigEndian())
616     std::reverse(Parts, Parts + OrigNumParts);
617 }
618 
619 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
620                                      const SDLoc &DL, EVT PartVT) {
621   if (!PartVT.isVector())
622     return SDValue();
623 
624   EVT ValueVT = Val.getValueType();
625   ElementCount PartNumElts = PartVT.getVectorElementCount();
626   ElementCount ValueNumElts = ValueVT.getVectorElementCount();
627 
628   // We only support widening vectors with equivalent element types and
629   // fixed/scalable properties. If a target needs to widen a fixed-length type
630   // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
631   if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
632       PartNumElts.isScalable() != ValueNumElts.isScalable() ||
633       PartVT.getVectorElementType() != ValueVT.getVectorElementType())
634     return SDValue();
635 
636   // Widening a scalable vector to another scalable vector is done by inserting
637   // the vector into a larger undef one.
638   if (PartNumElts.isScalable())
639     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
640                        Val, DAG.getVectorIdxConstant(0, DL));
641 
642   EVT ElementVT = PartVT.getVectorElementType();
643   // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
644   // undef elements.
645   SmallVector<SDValue, 16> Ops;
646   DAG.ExtractVectorElements(Val, Ops);
647   SDValue EltUndef = DAG.getUNDEF(ElementVT);
648   Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
649 
650   // FIXME: Use CONCAT for 2x -> 4x.
651   return DAG.getBuildVector(PartVT, DL, Ops);
652 }
653 
654 /// getCopyToPartsVector - Create a series of nodes that contain the specified
655 /// value split into legal parts.
656 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
657                                  SDValue Val, SDValue *Parts, unsigned NumParts,
658                                  MVT PartVT, const Value *V,
659                                  std::optional<CallingConv::ID> CallConv) {
660   EVT ValueVT = Val.getValueType();
661   assert(ValueVT.isVector() && "Not a vector");
662   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
663   const bool IsABIRegCopy = CallConv.has_value();
664 
665   if (NumParts == 1) {
666     EVT PartEVT = PartVT;
667     if (PartEVT == ValueVT) {
668       // Nothing to do.
669     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
670       // Bitconvert vector->vector case.
671       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
672     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
673       Val = Widened;
674     } else if (PartVT.isVector() &&
675                PartEVT.getVectorElementType().bitsGE(
676                    ValueVT.getVectorElementType()) &&
677                PartEVT.getVectorElementCount() ==
678                    ValueVT.getVectorElementCount()) {
679 
680       // Promoted vector extract
681       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
682     } else if (PartEVT.isVector() &&
683                PartEVT.getVectorElementType() !=
684                    ValueVT.getVectorElementType() &&
685                TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
686                    TargetLowering::TypeWidenVector) {
687       // Combination of widening and promotion.
688       EVT WidenVT =
689           EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
690                            PartVT.getVectorElementCount());
691       SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
692       Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
693     } else {
694       // Don't extract an integer from a float vector. This can happen if the
695       // FP type gets softened to integer and then promoted. The promotion
696       // prevents it from being picked up by the earlier bitcast case.
697       if (ValueVT.getVectorElementCount().isScalar() &&
698           (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
699         Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
700                           DAG.getVectorIdxConstant(0, DL));
701       } else {
702         uint64_t ValueSize = ValueVT.getFixedSizeInBits();
703         assert(PartVT.getFixedSizeInBits() > ValueSize &&
704                "lossy conversion of vector to scalar type");
705         EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
706         Val = DAG.getBitcast(IntermediateType, Val);
707         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
708       }
709     }
710 
711     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
712     Parts[0] = Val;
713     return;
714   }
715 
716   // Handle a multi-element vector.
717   EVT IntermediateVT;
718   MVT RegisterVT;
719   unsigned NumIntermediates;
720   unsigned NumRegs;
721   if (IsABIRegCopy) {
722     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
723         *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
724         RegisterVT);
725   } else {
726     NumRegs =
727         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
728                                    NumIntermediates, RegisterVT);
729   }
730 
731   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
732   NumParts = NumRegs; // Silence a compiler warning.
733   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
734 
735   assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
736          "Mixing scalable and fixed vectors when copying in parts");
737 
738   std::optional<ElementCount> DestEltCnt;
739 
740   if (IntermediateVT.isVector())
741     DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
742   else
743     DestEltCnt = ElementCount::getFixed(NumIntermediates);
744 
745   EVT BuiltVectorTy = EVT::getVectorVT(
746       *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
747 
748   if (ValueVT == BuiltVectorTy) {
749     // Nothing to do.
750   } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
751     // Bitconvert vector->vector case.
752     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
753   } else {
754     if (BuiltVectorTy.getVectorElementType().bitsGT(
755             ValueVT.getVectorElementType())) {
756       // Integer promotion.
757       ValueVT = EVT::getVectorVT(*DAG.getContext(),
758                                  BuiltVectorTy.getVectorElementType(),
759                                  ValueVT.getVectorElementCount());
760       Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
761     }
762 
763     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
764       Val = Widened;
765     }
766   }
767 
768   assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
769 
770   // Split the vector into intermediate operands.
771   SmallVector<SDValue, 8> Ops(NumIntermediates);
772   for (unsigned i = 0; i != NumIntermediates; ++i) {
773     if (IntermediateVT.isVector()) {
774       // This does something sensible for scalable vectors - see the
775       // definition of EXTRACT_SUBVECTOR for further details.
776       unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
777       Ops[i] =
778           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
779                       DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
780     } else {
781       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
782                            DAG.getVectorIdxConstant(i, DL));
783     }
784   }
785 
786   // Split the intermediate operands into legal parts.
787   if (NumParts == NumIntermediates) {
788     // If the register was not expanded, promote or copy the value,
789     // as appropriate.
790     for (unsigned i = 0; i != NumParts; ++i)
791       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
792   } else if (NumParts > 0) {
793     // If the intermediate type was expanded, split each the value into
794     // legal parts.
795     assert(NumIntermediates != 0 && "division by zero");
796     assert(NumParts % NumIntermediates == 0 &&
797            "Must expand into a divisible number of parts!");
798     unsigned Factor = NumParts / NumIntermediates;
799     for (unsigned i = 0; i != NumIntermediates; ++i)
800       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
801                      CallConv);
802   }
803 }
804 
805 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
806                            EVT valuevt, std::optional<CallingConv::ID> CC)
807     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
808       RegCount(1, regs.size()), CallConv(CC) {}
809 
810 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
811                            const DataLayout &DL, unsigned Reg, Type *Ty,
812                            std::optional<CallingConv::ID> CC) {
813   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
814 
815   CallConv = CC;
816 
817   for (EVT ValueVT : ValueVTs) {
818     unsigned NumRegs =
819         isABIMangled()
820             ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
821             : TLI.getNumRegisters(Context, ValueVT);
822     MVT RegisterVT =
823         isABIMangled()
824             ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
825             : TLI.getRegisterType(Context, ValueVT);
826     for (unsigned i = 0; i != NumRegs; ++i)
827       Regs.push_back(Reg + i);
828     RegVTs.push_back(RegisterVT);
829     RegCount.push_back(NumRegs);
830     Reg += NumRegs;
831   }
832 }
833 
834 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
835                                       FunctionLoweringInfo &FuncInfo,
836                                       const SDLoc &dl, SDValue &Chain,
837                                       SDValue *Flag, const Value *V) const {
838   // A Value with type {} or [0 x %t] needs no registers.
839   if (ValueVTs.empty())
840     return SDValue();
841 
842   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
843 
844   // Assemble the legal parts into the final values.
845   SmallVector<SDValue, 4> Values(ValueVTs.size());
846   SmallVector<SDValue, 8> Parts;
847   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
848     // Copy the legal parts from the registers.
849     EVT ValueVT = ValueVTs[Value];
850     unsigned NumRegs = RegCount[Value];
851     MVT RegisterVT = isABIMangled()
852                          ? TLI.getRegisterTypeForCallingConv(
853                                *DAG.getContext(), *CallConv, RegVTs[Value])
854                          : RegVTs[Value];
855 
856     Parts.resize(NumRegs);
857     for (unsigned i = 0; i != NumRegs; ++i) {
858       SDValue P;
859       if (!Flag) {
860         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
861       } else {
862         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
863         *Flag = P.getValue(2);
864       }
865 
866       Chain = P.getValue(1);
867       Parts[i] = P;
868 
869       // If the source register was virtual and if we know something about it,
870       // add an assert node.
871       if (!Register::isVirtualRegister(Regs[Part + i]) ||
872           !RegisterVT.isInteger())
873         continue;
874 
875       const FunctionLoweringInfo::LiveOutInfo *LOI =
876         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
877       if (!LOI)
878         continue;
879 
880       unsigned RegSize = RegisterVT.getScalarSizeInBits();
881       unsigned NumSignBits = LOI->NumSignBits;
882       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
883 
884       if (NumZeroBits == RegSize) {
885         // The current value is a zero.
886         // Explicitly express that as it would be easier for
887         // optimizations to kick in.
888         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
889         continue;
890       }
891 
892       // FIXME: We capture more information than the dag can represent.  For
893       // now, just use the tightest assertzext/assertsext possible.
894       bool isSExt;
895       EVT FromVT(MVT::Other);
896       if (NumZeroBits) {
897         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
898         isSExt = false;
899       } else if (NumSignBits > 1) {
900         FromVT =
901             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
902         isSExt = true;
903       } else {
904         continue;
905       }
906       // Add an assertion node.
907       assert(FromVT != MVT::Other);
908       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
909                              RegisterVT, P, DAG.getValueType(FromVT));
910     }
911 
912     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
913                                      RegisterVT, ValueVT, V, CallConv);
914     Part += NumRegs;
915     Parts.clear();
916   }
917 
918   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
919 }
920 
921 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
922                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
923                                  const Value *V,
924                                  ISD::NodeType PreferredExtendType) const {
925   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
926   ISD::NodeType ExtendKind = PreferredExtendType;
927 
928   // Get the list of the values's legal parts.
929   unsigned NumRegs = Regs.size();
930   SmallVector<SDValue, 8> Parts(NumRegs);
931   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
932     unsigned NumParts = RegCount[Value];
933 
934     MVT RegisterVT = isABIMangled()
935                          ? TLI.getRegisterTypeForCallingConv(
936                                *DAG.getContext(), *CallConv, RegVTs[Value])
937                          : RegVTs[Value];
938 
939     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
940       ExtendKind = ISD::ZERO_EXTEND;
941 
942     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
943                    NumParts, RegisterVT, V, CallConv, ExtendKind);
944     Part += NumParts;
945   }
946 
947   // Copy the parts into the registers.
948   SmallVector<SDValue, 8> Chains(NumRegs);
949   for (unsigned i = 0; i != NumRegs; ++i) {
950     SDValue Part;
951     if (!Flag) {
952       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
953     } else {
954       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
955       *Flag = Part.getValue(1);
956     }
957 
958     Chains[i] = Part.getValue(0);
959   }
960 
961   if (NumRegs == 1 || Flag)
962     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
963     // flagged to it. That is the CopyToReg nodes and the user are considered
964     // a single scheduling unit. If we create a TokenFactor and return it as
965     // chain, then the TokenFactor is both a predecessor (operand) of the
966     // user as well as a successor (the TF operands are flagged to the user).
967     // c1, f1 = CopyToReg
968     // c2, f2 = CopyToReg
969     // c3     = TokenFactor c1, c2
970     // ...
971     //        = op c3, ..., f2
972     Chain = Chains[NumRegs-1];
973   else
974     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
975 }
976 
977 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
978                                         unsigned MatchingIdx, const SDLoc &dl,
979                                         SelectionDAG &DAG,
980                                         std::vector<SDValue> &Ops) const {
981   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
982 
983   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
984   if (HasMatching)
985     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
986   else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
987     // Put the register class of the virtual registers in the flag word.  That
988     // way, later passes can recompute register class constraints for inline
989     // assembly as well as normal instructions.
990     // Don't do this for tied operands that can use the regclass information
991     // from the def.
992     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
993     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
994     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
995   }
996 
997   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
998   Ops.push_back(Res);
999 
1000   if (Code == InlineAsm::Kind_Clobber) {
1001     // Clobbers should always have a 1:1 mapping with registers, and may
1002     // reference registers that have illegal (e.g. vector) types. Hence, we
1003     // shouldn't try to apply any sort of splitting logic to them.
1004     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1005            "No 1:1 mapping from clobbers to regs?");
1006     Register SP = TLI.getStackPointerRegisterToSaveRestore();
1007     (void)SP;
1008     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1009       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1010       assert(
1011           (Regs[I] != SP ||
1012            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1013           "If we clobbered the stack pointer, MFI should know about it.");
1014     }
1015     return;
1016   }
1017 
1018   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1019     MVT RegisterVT = RegVTs[Value];
1020     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1021                                            RegisterVT);
1022     for (unsigned i = 0; i != NumRegs; ++i) {
1023       assert(Reg < Regs.size() && "Mismatch in # registers expected");
1024       unsigned TheReg = Regs[Reg++];
1025       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1026     }
1027   }
1028 }
1029 
1030 SmallVector<std::pair<unsigned, TypeSize>, 4>
1031 RegsForValue::getRegsAndSizes() const {
1032   SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
1033   unsigned I = 0;
1034   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1035     unsigned RegCount = std::get<0>(CountAndVT);
1036     MVT RegisterVT = std::get<1>(CountAndVT);
1037     TypeSize RegisterSize = RegisterVT.getSizeInBits();
1038     for (unsigned E = I + RegCount; I != E; ++I)
1039       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1040   }
1041   return OutVec;
1042 }
1043 
1044 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1045                                AssumptionCache *ac,
1046                                const TargetLibraryInfo *li) {
1047   AA = aa;
1048   AC = ac;
1049   GFI = gfi;
1050   LibInfo = li;
1051   Context = DAG.getContext();
1052   LPadToCallSiteMap.clear();
1053   SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1054 }
1055 
1056 void SelectionDAGBuilder::clear() {
1057   NodeMap.clear();
1058   UnusedArgNodeMap.clear();
1059   PendingLoads.clear();
1060   PendingExports.clear();
1061   PendingConstrainedFP.clear();
1062   PendingConstrainedFPStrict.clear();
1063   CurInst = nullptr;
1064   HasTailCall = false;
1065   SDNodeOrder = LowestSDNodeOrder;
1066   StatepointLowering.clear();
1067 }
1068 
1069 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1070   DanglingDebugInfoMap.clear();
1071 }
1072 
1073 // Update DAG root to include dependencies on Pending chains.
1074 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1075   SDValue Root = DAG.getRoot();
1076 
1077   if (Pending.empty())
1078     return Root;
1079 
1080   // Add current root to PendingChains, unless we already indirectly
1081   // depend on it.
1082   if (Root.getOpcode() != ISD::EntryToken) {
1083     unsigned i = 0, e = Pending.size();
1084     for (; i != e; ++i) {
1085       assert(Pending[i].getNode()->getNumOperands() > 1);
1086       if (Pending[i].getNode()->getOperand(0) == Root)
1087         break;  // Don't add the root if we already indirectly depend on it.
1088     }
1089 
1090     if (i == e)
1091       Pending.push_back(Root);
1092   }
1093 
1094   if (Pending.size() == 1)
1095     Root = Pending[0];
1096   else
1097     Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1098 
1099   DAG.setRoot(Root);
1100   Pending.clear();
1101   return Root;
1102 }
1103 
1104 SDValue SelectionDAGBuilder::getMemoryRoot() {
1105   return updateRoot(PendingLoads);
1106 }
1107 
1108 SDValue SelectionDAGBuilder::getRoot() {
1109   // Chain up all pending constrained intrinsics together with all
1110   // pending loads, by simply appending them to PendingLoads and
1111   // then calling getMemoryRoot().
1112   PendingLoads.reserve(PendingLoads.size() +
1113                        PendingConstrainedFP.size() +
1114                        PendingConstrainedFPStrict.size());
1115   PendingLoads.append(PendingConstrainedFP.begin(),
1116                       PendingConstrainedFP.end());
1117   PendingLoads.append(PendingConstrainedFPStrict.begin(),
1118                       PendingConstrainedFPStrict.end());
1119   PendingConstrainedFP.clear();
1120   PendingConstrainedFPStrict.clear();
1121   return getMemoryRoot();
1122 }
1123 
1124 SDValue SelectionDAGBuilder::getControlRoot() {
1125   // We need to emit pending fpexcept.strict constrained intrinsics,
1126   // so append them to the PendingExports list.
1127   PendingExports.append(PendingConstrainedFPStrict.begin(),
1128                         PendingConstrainedFPStrict.end());
1129   PendingConstrainedFPStrict.clear();
1130   return updateRoot(PendingExports);
1131 }
1132 
1133 void SelectionDAGBuilder::visit(const Instruction &I) {
1134   // Set up outgoing PHI node register values before emitting the terminator.
1135   if (I.isTerminator()) {
1136     HandlePHINodesInSuccessorBlocks(I.getParent());
1137   }
1138 
1139   // Add SDDbgValue nodes for any var locs here. Do so before updating
1140   // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1141   if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1142     // Add SDDbgValue nodes for any var locs here. Do so before updating
1143     // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1144     for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1145          It != End; ++It) {
1146       auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1147       dropDanglingDebugInfo(Var, It->Expr);
1148       SmallVector<Value *> Values(It->Values.location_ops());
1149       if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1150                             It->Values.hasArgList()))
1151         addDanglingDebugInfo(It, SDNodeOrder);
1152     }
1153   }
1154 
1155   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1156   if (!isa<DbgInfoIntrinsic>(I))
1157     ++SDNodeOrder;
1158 
1159   CurInst = &I;
1160 
1161   // Set inserted listener only if required.
1162   bool NodeInserted = false;
1163   std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1164   MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1165   if (PCSectionsMD) {
1166     InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1167         DAG, [&](SDNode *) { NodeInserted = true; });
1168   }
1169 
1170   visit(I.getOpcode(), I);
1171 
1172   if (!I.isTerminator() && !HasTailCall &&
1173       !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1174     CopyToExportRegsIfNeeded(&I);
1175 
1176   // Handle metadata.
1177   if (PCSectionsMD) {
1178     auto It = NodeMap.find(&I);
1179     if (It != NodeMap.end()) {
1180       DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1181     } else if (NodeInserted) {
1182       // This should not happen; if it does, don't let it go unnoticed so we can
1183       // fix it. Relevant visit*() function is probably missing a setValue().
1184       errs() << "warning: loosing !pcsections metadata ["
1185              << I.getModule()->getName() << "]\n";
1186       LLVM_DEBUG(I.dump());
1187       assert(false);
1188     }
1189   }
1190 
1191   CurInst = nullptr;
1192 }
1193 
1194 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1195   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1196 }
1197 
1198 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1199   // Note: this doesn't use InstVisitor, because it has to work with
1200   // ConstantExpr's in addition to instructions.
1201   switch (Opcode) {
1202   default: llvm_unreachable("Unknown instruction type encountered!");
1203     // Build the switch statement using the Instruction.def file.
1204 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1205     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1206 #include "llvm/IR/Instruction.def"
1207   }
1208 }
1209 
1210 static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1211                                             DILocalVariable *Variable,
1212                                             DebugLoc DL, unsigned Order,
1213                                             RawLocationWrapper Values,
1214                                             DIExpression *Expression) {
1215   if (!Values.hasArgList())
1216     return false;
1217   // For variadic dbg_values we will now insert an undef.
1218   // FIXME: We can potentially recover these!
1219   SmallVector<SDDbgOperand, 2> Locs;
1220   for (const Value *V : Values.location_ops()) {
1221     auto *Undef = UndefValue::get(V->getType());
1222     Locs.push_back(SDDbgOperand::fromConst(Undef));
1223   }
1224   SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1225                                         /*IsIndirect=*/false, DL, Order,
1226                                         /*IsVariadic=*/true);
1227   DAG.AddDbgValue(SDV, /*isParameter=*/false);
1228   return true;
1229 }
1230 
1231 void SelectionDAGBuilder::addDanglingDebugInfo(const VarLocInfo *VarLoc,
1232                                                unsigned Order) {
1233   if (!handleDanglingVariadicDebugInfo(
1234           DAG,
1235           const_cast<DILocalVariable *>(DAG.getFunctionVarLocs()
1236                                             ->getVariable(VarLoc->VariableID)
1237                                             .getVariable()),
1238           VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) {
1239     DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back(
1240         VarLoc, Order);
1241   }
1242 }
1243 
1244 void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
1245                                                unsigned Order) {
1246   // We treat variadic dbg_values differently at this stage.
1247   if (!handleDanglingVariadicDebugInfo(
1248           DAG, DI->getVariable(), DI->getDebugLoc(), Order,
1249           DI->getWrappedLocation(), DI->getExpression())) {
1250     // TODO: Dangling debug info will eventually either be resolved or produce
1251     // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1252     // between the original dbg.value location and its resolved DBG_VALUE,
1253     // which we should ideally fill with an extra Undef DBG_VALUE.
1254     assert(DI->getNumVariableLocationOps() == 1 &&
1255            "DbgValueInst without an ArgList should have a single location "
1256            "operand.");
1257     DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);
1258   }
1259 }
1260 
1261 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1262                                                 const DIExpression *Expr) {
1263   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1264     DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs());
1265     DIExpression *DanglingExpr = DDI.getExpression();
1266     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1267       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)
1268                         << "\n");
1269       return true;
1270     }
1271     return false;
1272   };
1273 
1274   for (auto &DDIMI : DanglingDebugInfoMap) {
1275     DanglingDebugInfoVector &DDIV = DDIMI.second;
1276 
1277     // If debug info is to be dropped, run it through final checks to see
1278     // whether it can be salvaged.
1279     for (auto &DDI : DDIV)
1280       if (isMatchingDbgValue(DDI))
1281         salvageUnresolvedDbgValue(DDI);
1282 
1283     erase_if(DDIV, isMatchingDbgValue);
1284   }
1285 }
1286 
1287 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1288 // generate the debug data structures now that we've seen its definition.
1289 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1290                                                    SDValue Val) {
1291   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1292   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1293     return;
1294 
1295   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1296   for (auto &DDI : DDIV) {
1297     DebugLoc DL = DDI.getDebugLoc();
1298     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1299     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1300     DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs());
1301     DIExpression *Expr = DDI.getExpression();
1302     assert(Variable->isValidLocationForIntrinsic(DL) &&
1303            "Expected inlined-at fields to agree");
1304     SDDbgValue *SDV;
1305     if (Val.getNode()) {
1306       // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1307       // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1308       // we couldn't resolve it directly when examining the DbgValue intrinsic
1309       // in the first place we should not be more successful here). Unless we
1310       // have some test case that prove this to be correct we should avoid
1311       // calling EmitFuncArgumentDbgValue here.
1312       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1313                                     FuncArgumentDbgValueKind::Value, Val)) {
1314         LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)
1315                           << "\n");
1316         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1317         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1318         // inserted after the definition of Val when emitting the instructions
1319         // after ISel. An alternative could be to teach
1320         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1321         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1322                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1323                    << ValSDNodeOrder << "\n");
1324         SDV = getDbgValue(Val, Variable, Expr, DL,
1325                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1326         DAG.AddDbgValue(SDV, false);
1327       } else
1328         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1329                           << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n");
1330     } else {
1331       LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n");
1332       auto Undef = UndefValue::get(V->getType());
1333       auto SDV =
1334           DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1335       DAG.AddDbgValue(SDV, false);
1336     }
1337   }
1338   DDIV.clear();
1339 }
1340 
1341 void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1342   // TODO: For the variadic implementation, instead of only checking the fail
1343   // state of `handleDebugValue`, we need know specifically which values were
1344   // invalid, so that we attempt to salvage only those values when processing
1345   // a DIArgList.
1346   Value *V = DDI.getVariableLocationOp(0);
1347   Value *OrigV = V;
1348   DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs());
1349   DIExpression *Expr = DDI.getExpression();
1350   DebugLoc DL = DDI.getDebugLoc();
1351   unsigned SDOrder = DDI.getSDNodeOrder();
1352 
1353   // Currently we consider only dbg.value intrinsics -- we tell the salvager
1354   // that DW_OP_stack_value is desired.
1355   bool StackValue = true;
1356 
1357   // Can this Value can be encoded without any further work?
1358   if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1359     return;
1360 
1361   // Attempt to salvage back through as many instructions as possible. Bail if
1362   // a non-instruction is seen, such as a constant expression or global
1363   // variable. FIXME: Further work could recover those too.
1364   while (isa<Instruction>(V)) {
1365     Instruction &VAsInst = *cast<Instruction>(V);
1366     // Temporary "0", awaiting real implementation.
1367     SmallVector<uint64_t, 16> Ops;
1368     SmallVector<Value *, 4> AdditionalValues;
1369     V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1370                              AdditionalValues);
1371     // If we cannot salvage any further, and haven't yet found a suitable debug
1372     // expression, bail out.
1373     if (!V)
1374       break;
1375 
1376     // TODO: If AdditionalValues isn't empty, then the salvage can only be
1377     // represented with a DBG_VALUE_LIST, so we give up. When we have support
1378     // here for variadic dbg_values, remove that condition.
1379     if (!AdditionalValues.empty())
1380       break;
1381 
1382     // New value and expr now represent this debuginfo.
1383     Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1384 
1385     // Some kind of simplification occurred: check whether the operand of the
1386     // salvaged debug expression can be encoded in this DAG.
1387     if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1388       LLVM_DEBUG(
1389           dbgs() << "Salvaged debug location info for:\n  " << *Var << "\n"
1390                  << *OrigV << "\nBy stripping back to:\n  " << *V << "\n");
1391       return;
1392     }
1393   }
1394 
1395   // This was the final opportunity to salvage this debug information, and it
1396   // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1397   // any earlier variable location.
1398   assert(OrigV && "V shouldn't be null");
1399   auto *Undef = UndefValue::get(OrigV->getType());
1400   auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1401   DAG.AddDbgValue(SDV, false);
1402   LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << printDDI(DDI)
1403                     << "\n");
1404 }
1405 
1406 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1407                                            DILocalVariable *Var,
1408                                            DIExpression *Expr, DebugLoc DbgLoc,
1409                                            unsigned Order, bool IsVariadic) {
1410   if (Values.empty())
1411     return true;
1412   SmallVector<SDDbgOperand> LocationOps;
1413   SmallVector<SDNode *> Dependencies;
1414   for (const Value *V : Values) {
1415     // Constant value.
1416     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1417         isa<ConstantPointerNull>(V)) {
1418       LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1419       continue;
1420     }
1421 
1422     // Look through IntToPtr constants.
1423     if (auto *CE = dyn_cast<ConstantExpr>(V))
1424       if (CE->getOpcode() == Instruction::IntToPtr) {
1425         LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1426         continue;
1427       }
1428 
1429     // If the Value is a frame index, we can create a FrameIndex debug value
1430     // without relying on the DAG at all.
1431     if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1432       auto SI = FuncInfo.StaticAllocaMap.find(AI);
1433       if (SI != FuncInfo.StaticAllocaMap.end()) {
1434         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1435         continue;
1436       }
1437     }
1438 
1439     // Do not use getValue() in here; we don't want to generate code at
1440     // this point if it hasn't been done yet.
1441     SDValue N = NodeMap[V];
1442     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1443       N = UnusedArgNodeMap[V];
1444     if (N.getNode()) {
1445       // Only emit func arg dbg value for non-variadic dbg.values for now.
1446       if (!IsVariadic &&
1447           EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1448                                    FuncArgumentDbgValueKind::Value, N))
1449         return true;
1450       if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1451         // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1452         // describe stack slot locations.
1453         //
1454         // Consider "int x = 0; int *px = &x;". There are two kinds of
1455         // interesting debug values here after optimization:
1456         //
1457         //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
1458         //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1459         //
1460         // Both describe the direct values of their associated variables.
1461         Dependencies.push_back(N.getNode());
1462         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1463         continue;
1464       }
1465       LocationOps.emplace_back(
1466           SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1467       continue;
1468     }
1469 
1470     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1471     // Special rules apply for the first dbg.values of parameter variables in a
1472     // function. Identify them by the fact they reference Argument Values, that
1473     // they're parameters, and they are parameters of the current function. We
1474     // need to let them dangle until they get an SDNode.
1475     bool IsParamOfFunc =
1476         isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1477     if (IsParamOfFunc)
1478       return false;
1479 
1480     // The value is not used in this block yet (or it would have an SDNode).
1481     // We still want the value to appear for the user if possible -- if it has
1482     // an associated VReg, we can refer to that instead.
1483     auto VMI = FuncInfo.ValueMap.find(V);
1484     if (VMI != FuncInfo.ValueMap.end()) {
1485       unsigned Reg = VMI->second;
1486       // If this is a PHI node, it may be split up into several MI PHI nodes
1487       // (in FunctionLoweringInfo::set).
1488       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1489                        V->getType(), std::nullopt);
1490       if (RFV.occupiesMultipleRegs()) {
1491         // FIXME: We could potentially support variadic dbg_values here.
1492         if (IsVariadic)
1493           return false;
1494         unsigned Offset = 0;
1495         unsigned BitsToDescribe = 0;
1496         if (auto VarSize = Var->getSizeInBits())
1497           BitsToDescribe = *VarSize;
1498         if (auto Fragment = Expr->getFragmentInfo())
1499           BitsToDescribe = Fragment->SizeInBits;
1500         for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1501           // Bail out if all bits are described already.
1502           if (Offset >= BitsToDescribe)
1503             break;
1504           // TODO: handle scalable vectors.
1505           unsigned RegisterSize = RegAndSize.second;
1506           unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1507                                       ? BitsToDescribe - Offset
1508                                       : RegisterSize;
1509           auto FragmentExpr = DIExpression::createFragmentExpression(
1510               Expr, Offset, FragmentSize);
1511           if (!FragmentExpr)
1512             continue;
1513           SDDbgValue *SDV = DAG.getVRegDbgValue(
1514               Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1515           DAG.AddDbgValue(SDV, false);
1516           Offset += RegisterSize;
1517         }
1518         return true;
1519       }
1520       // We can use simple vreg locations for variadic dbg_values as well.
1521       LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1522       continue;
1523     }
1524     // We failed to create a SDDbgOperand for V.
1525     return false;
1526   }
1527 
1528   // We have created a SDDbgOperand for each Value in Values.
1529   // Should use Order instead of SDNodeOrder?
1530   assert(!LocationOps.empty());
1531   SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1532                                         /*IsIndirect=*/false, DbgLoc,
1533                                         SDNodeOrder, IsVariadic);
1534   DAG.AddDbgValue(SDV, /*isParameter=*/false);
1535   return true;
1536 }
1537 
1538 void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1539   // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1540   for (auto &Pair : DanglingDebugInfoMap)
1541     for (auto &DDI : Pair.second)
1542       salvageUnresolvedDbgValue(DDI);
1543   clearDanglingDebugInfo();
1544 }
1545 
1546 /// getCopyFromRegs - If there was virtual register allocated for the value V
1547 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1548 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1549   DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
1550   SDValue Result;
1551 
1552   if (It != FuncInfo.ValueMap.end()) {
1553     Register InReg = It->second;
1554 
1555     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1556                      DAG.getDataLayout(), InReg, Ty,
1557                      std::nullopt); // This is not an ABI copy.
1558     SDValue Chain = DAG.getEntryNode();
1559     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1560                                  V);
1561     resolveDanglingDebugInfo(V, Result);
1562   }
1563 
1564   return Result;
1565 }
1566 
1567 /// getValue - Return an SDValue for the given Value.
1568 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1569   // If we already have an SDValue for this value, use it. It's important
1570   // to do this first, so that we don't create a CopyFromReg if we already
1571   // have a regular SDValue.
1572   SDValue &N = NodeMap[V];
1573   if (N.getNode()) return N;
1574 
1575   // If there's a virtual register allocated and initialized for this
1576   // value, use it.
1577   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1578     return copyFromReg;
1579 
1580   // Otherwise create a new SDValue and remember it.
1581   SDValue Val = getValueImpl(V);
1582   NodeMap[V] = Val;
1583   resolveDanglingDebugInfo(V, Val);
1584   return Val;
1585 }
1586 
1587 /// getNonRegisterValue - Return an SDValue for the given Value, but
1588 /// don't look in FuncInfo.ValueMap for a virtual register.
1589 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1590   // If we already have an SDValue for this value, use it.
1591   SDValue &N = NodeMap[V];
1592   if (N.getNode()) {
1593     if (isIntOrFPConstant(N)) {
1594       // Remove the debug location from the node as the node is about to be used
1595       // in a location which may differ from the original debug location.  This
1596       // is relevant to Constant and ConstantFP nodes because they can appear
1597       // as constant expressions inside PHI nodes.
1598       N->setDebugLoc(DebugLoc());
1599     }
1600     return N;
1601   }
1602 
1603   // Otherwise create a new SDValue and remember it.
1604   SDValue Val = getValueImpl(V);
1605   NodeMap[V] = Val;
1606   resolveDanglingDebugInfo(V, Val);
1607   return Val;
1608 }
1609 
1610 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1611 /// Create an SDValue for the given value.
1612 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1613   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1614 
1615   if (const Constant *C = dyn_cast<Constant>(V)) {
1616     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1617 
1618     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1619       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1620 
1621     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1622       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1623 
1624     if (isa<ConstantPointerNull>(C)) {
1625       unsigned AS = V->getType()->getPointerAddressSpace();
1626       return DAG.getConstant(0, getCurSDLoc(),
1627                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1628     }
1629 
1630     if (match(C, m_VScale()))
1631       return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1632 
1633     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1634       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1635 
1636     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1637       return DAG.getUNDEF(VT);
1638 
1639     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1640       visit(CE->getOpcode(), *CE);
1641       SDValue N1 = NodeMap[V];
1642       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1643       return N1;
1644     }
1645 
1646     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1647       SmallVector<SDValue, 4> Constants;
1648       for (const Use &U : C->operands()) {
1649         SDNode *Val = getValue(U).getNode();
1650         // If the operand is an empty aggregate, there are no values.
1651         if (!Val) continue;
1652         // Add each leaf value from the operand to the Constants list
1653         // to form a flattened list of all the values.
1654         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1655           Constants.push_back(SDValue(Val, i));
1656       }
1657 
1658       return DAG.getMergeValues(Constants, getCurSDLoc());
1659     }
1660 
1661     if (const ConstantDataSequential *CDS =
1662           dyn_cast<ConstantDataSequential>(C)) {
1663       SmallVector<SDValue, 4> Ops;
1664       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1665         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1666         // Add each leaf value from the operand to the Constants list
1667         // to form a flattened list of all the values.
1668         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1669           Ops.push_back(SDValue(Val, i));
1670       }
1671 
1672       if (isa<ArrayType>(CDS->getType()))
1673         return DAG.getMergeValues(Ops, getCurSDLoc());
1674       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1675     }
1676 
1677     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1678       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1679              "Unknown struct or array constant!");
1680 
1681       SmallVector<EVT, 4> ValueVTs;
1682       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1683       unsigned NumElts = ValueVTs.size();
1684       if (NumElts == 0)
1685         return SDValue(); // empty struct
1686       SmallVector<SDValue, 4> Constants(NumElts);
1687       for (unsigned i = 0; i != NumElts; ++i) {
1688         EVT EltVT = ValueVTs[i];
1689         if (isa<UndefValue>(C))
1690           Constants[i] = DAG.getUNDEF(EltVT);
1691         else if (EltVT.isFloatingPoint())
1692           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1693         else
1694           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1695       }
1696 
1697       return DAG.getMergeValues(Constants, getCurSDLoc());
1698     }
1699 
1700     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1701       return DAG.getBlockAddress(BA, VT);
1702 
1703     if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1704       return getValue(Equiv->getGlobalValue());
1705 
1706     if (const auto *NC = dyn_cast<NoCFIValue>(C))
1707       return getValue(NC->getGlobalValue());
1708 
1709     VectorType *VecTy = cast<VectorType>(V->getType());
1710 
1711     // Now that we know the number and type of the elements, get that number of
1712     // elements into the Ops array based on what kind of constant it is.
1713     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1714       SmallVector<SDValue, 16> Ops;
1715       unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1716       for (unsigned i = 0; i != NumElements; ++i)
1717         Ops.push_back(getValue(CV->getOperand(i)));
1718 
1719       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1720     }
1721 
1722     if (isa<ConstantAggregateZero>(C)) {
1723       EVT EltVT =
1724           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1725 
1726       SDValue Op;
1727       if (EltVT.isFloatingPoint())
1728         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1729       else
1730         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1731 
1732       return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1733     }
1734 
1735     llvm_unreachable("Unknown vector constant");
1736   }
1737 
1738   // If this is a static alloca, generate it as the frameindex instead of
1739   // computation.
1740   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1741     DenseMap<const AllocaInst*, int>::iterator SI =
1742       FuncInfo.StaticAllocaMap.find(AI);
1743     if (SI != FuncInfo.StaticAllocaMap.end())
1744       return DAG.getFrameIndex(
1745           SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1746   }
1747 
1748   // If this is an instruction which fast-isel has deferred, select it now.
1749   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1750     Register InReg = FuncInfo.InitializeRegForValue(Inst);
1751 
1752     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1753                      Inst->getType(), std::nullopt);
1754     SDValue Chain = DAG.getEntryNode();
1755     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1756   }
1757 
1758   if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1759     return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1760 
1761   if (const auto *BB = dyn_cast<BasicBlock>(V))
1762     return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1763 
1764   llvm_unreachable("Can't get register for value!");
1765 }
1766 
1767 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1768   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1769   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1770   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1771   bool IsSEH = isAsynchronousEHPersonality(Pers);
1772   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1773   if (!IsSEH)
1774     CatchPadMBB->setIsEHScopeEntry();
1775   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1776   if (IsMSVCCXX || IsCoreCLR)
1777     CatchPadMBB->setIsEHFuncletEntry();
1778 }
1779 
1780 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1781   // Update machine-CFG edge.
1782   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1783   FuncInfo.MBB->addSuccessor(TargetMBB);
1784   TargetMBB->setIsEHCatchretTarget(true);
1785   DAG.getMachineFunction().setHasEHCatchret(true);
1786 
1787   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1788   bool IsSEH = isAsynchronousEHPersonality(Pers);
1789   if (IsSEH) {
1790     // If this is not a fall-through branch or optimizations are switched off,
1791     // emit the branch.
1792     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1793         TM.getOptLevel() == CodeGenOpt::None)
1794       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1795                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1796     return;
1797   }
1798 
1799   // Figure out the funclet membership for the catchret's successor.
1800   // This will be used by the FuncletLayout pass to determine how to order the
1801   // BB's.
1802   // A 'catchret' returns to the outer scope's color.
1803   Value *ParentPad = I.getCatchSwitchParentPad();
1804   const BasicBlock *SuccessorColor;
1805   if (isa<ConstantTokenNone>(ParentPad))
1806     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1807   else
1808     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1809   assert(SuccessorColor && "No parent funclet for catchret!");
1810   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1811   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1812 
1813   // Create the terminator node.
1814   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1815                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1816                             DAG.getBasicBlock(SuccessorColorMBB));
1817   DAG.setRoot(Ret);
1818 }
1819 
1820 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1821   // Don't emit any special code for the cleanuppad instruction. It just marks
1822   // the start of an EH scope/funclet.
1823   FuncInfo.MBB->setIsEHScopeEntry();
1824   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1825   if (Pers != EHPersonality::Wasm_CXX) {
1826     FuncInfo.MBB->setIsEHFuncletEntry();
1827     FuncInfo.MBB->setIsCleanupFuncletEntry();
1828   }
1829 }
1830 
1831 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1832 // not match, it is OK to add only the first unwind destination catchpad to the
1833 // successors, because there will be at least one invoke instruction within the
1834 // catch scope that points to the next unwind destination, if one exists, so
1835 // CFGSort cannot mess up with BB sorting order.
1836 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1837 // call within them, and catchpads only consisting of 'catch (...)' have a
1838 // '__cxa_end_catch' call within them, both of which generate invokes in case
1839 // the next unwind destination exists, i.e., the next unwind destination is not
1840 // the caller.)
1841 //
1842 // Having at most one EH pad successor is also simpler and helps later
1843 // transformations.
1844 //
1845 // For example,
1846 // current:
1847 //   invoke void @foo to ... unwind label %catch.dispatch
1848 // catch.dispatch:
1849 //   %0 = catchswitch within ... [label %catch.start] unwind label %next
1850 // catch.start:
1851 //   ...
1852 //   ... in this BB or some other child BB dominated by this BB there will be an
1853 //   invoke that points to 'next' BB as an unwind destination
1854 //
1855 // next: ; We don't need to add this to 'current' BB's successor
1856 //   ...
1857 static void findWasmUnwindDestinations(
1858     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1859     BranchProbability Prob,
1860     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1861         &UnwindDests) {
1862   while (EHPadBB) {
1863     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1864     if (isa<CleanupPadInst>(Pad)) {
1865       // Stop on cleanup pads.
1866       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1867       UnwindDests.back().first->setIsEHScopeEntry();
1868       break;
1869     } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1870       // Add the catchpad handlers to the possible destinations. We don't
1871       // continue to the unwind destination of the catchswitch for wasm.
1872       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1873         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1874         UnwindDests.back().first->setIsEHScopeEntry();
1875       }
1876       break;
1877     } else {
1878       continue;
1879     }
1880   }
1881 }
1882 
1883 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1884 /// many places it could ultimately go. In the IR, we have a single unwind
1885 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1886 /// This function skips over imaginary basic blocks that hold catchswitch
1887 /// instructions, and finds all the "real" machine
1888 /// basic block destinations. As those destinations may not be successors of
1889 /// EHPadBB, here we also calculate the edge probability to those destinations.
1890 /// The passed-in Prob is the edge probability to EHPadBB.
1891 static void findUnwindDestinations(
1892     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1893     BranchProbability Prob,
1894     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1895         &UnwindDests) {
1896   EHPersonality Personality =
1897     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1898   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1899   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1900   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1901   bool IsSEH = isAsynchronousEHPersonality(Personality);
1902 
1903   if (IsWasmCXX) {
1904     findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1905     assert(UnwindDests.size() <= 1 &&
1906            "There should be at most one unwind destination for wasm");
1907     return;
1908   }
1909 
1910   while (EHPadBB) {
1911     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1912     BasicBlock *NewEHPadBB = nullptr;
1913     if (isa<LandingPadInst>(Pad)) {
1914       // Stop on landingpads. They are not funclets.
1915       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1916       break;
1917     } else if (isa<CleanupPadInst>(Pad)) {
1918       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1919       // personalities.
1920       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1921       UnwindDests.back().first->setIsEHScopeEntry();
1922       UnwindDests.back().first->setIsEHFuncletEntry();
1923       break;
1924     } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1925       // Add the catchpad handlers to the possible destinations.
1926       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1927         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1928         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1929         if (IsMSVCCXX || IsCoreCLR)
1930           UnwindDests.back().first->setIsEHFuncletEntry();
1931         if (!IsSEH)
1932           UnwindDests.back().first->setIsEHScopeEntry();
1933       }
1934       NewEHPadBB = CatchSwitch->getUnwindDest();
1935     } else {
1936       continue;
1937     }
1938 
1939     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1940     if (BPI && NewEHPadBB)
1941       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1942     EHPadBB = NewEHPadBB;
1943   }
1944 }
1945 
1946 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1947   // Update successor info.
1948   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1949   auto UnwindDest = I.getUnwindDest();
1950   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1951   BranchProbability UnwindDestProb =
1952       (BPI && UnwindDest)
1953           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1954           : BranchProbability::getZero();
1955   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1956   for (auto &UnwindDest : UnwindDests) {
1957     UnwindDest.first->setIsEHPad();
1958     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1959   }
1960   FuncInfo.MBB->normalizeSuccProbs();
1961 
1962   // Create the terminator node.
1963   SDValue Ret =
1964       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1965   DAG.setRoot(Ret);
1966 }
1967 
1968 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1969   report_fatal_error("visitCatchSwitch not yet implemented!");
1970 }
1971 
1972 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1973   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1974   auto &DL = DAG.getDataLayout();
1975   SDValue Chain = getControlRoot();
1976   SmallVector<ISD::OutputArg, 8> Outs;
1977   SmallVector<SDValue, 8> OutVals;
1978 
1979   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1980   // lower
1981   //
1982   //   %val = call <ty> @llvm.experimental.deoptimize()
1983   //   ret <ty> %val
1984   //
1985   // differently.
1986   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1987     LowerDeoptimizingReturn();
1988     return;
1989   }
1990 
1991   if (!FuncInfo.CanLowerReturn) {
1992     unsigned DemoteReg = FuncInfo.DemoteRegister;
1993     const Function *F = I.getParent()->getParent();
1994 
1995     // Emit a store of the return value through the virtual register.
1996     // Leave Outs empty so that LowerReturn won't try to load return
1997     // registers the usual way.
1998     SmallVector<EVT, 1> PtrValueVTs;
1999     ComputeValueVTs(TLI, DL,
2000                     F->getReturnType()->getPointerTo(
2001                         DAG.getDataLayout().getAllocaAddrSpace()),
2002                     PtrValueVTs);
2003 
2004     SDValue RetPtr =
2005         DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2006     SDValue RetOp = getValue(I.getOperand(0));
2007 
2008     SmallVector<EVT, 4> ValueVTs, MemVTs;
2009     SmallVector<uint64_t, 4> Offsets;
2010     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2011                     &Offsets);
2012     unsigned NumValues = ValueVTs.size();
2013 
2014     SmallVector<SDValue, 4> Chains(NumValues);
2015     Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2016     for (unsigned i = 0; i != NumValues; ++i) {
2017       // An aggregate return value cannot wrap around the address space, so
2018       // offsets to its parts don't wrap either.
2019       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
2020                                            TypeSize::Fixed(Offsets[i]));
2021 
2022       SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2023       if (MemVTs[i] != ValueVTs[i])
2024         Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2025       Chains[i] = DAG.getStore(
2026           Chain, getCurSDLoc(), Val,
2027           // FIXME: better loc info would be nice.
2028           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
2029           commonAlignment(BaseAlign, Offsets[i]));
2030     }
2031 
2032     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2033                         MVT::Other, Chains);
2034   } else if (I.getNumOperands() != 0) {
2035     SmallVector<EVT, 4> ValueVTs;
2036     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2037     unsigned NumValues = ValueVTs.size();
2038     if (NumValues) {
2039       SDValue RetOp = getValue(I.getOperand(0));
2040 
2041       const Function *F = I.getParent()->getParent();
2042 
2043       bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2044           I.getOperand(0)->getType(), F->getCallingConv(),
2045           /*IsVarArg*/ false, DL);
2046 
2047       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2048       if (F->getAttributes().hasRetAttr(Attribute::SExt))
2049         ExtendKind = ISD::SIGN_EXTEND;
2050       else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2051         ExtendKind = ISD::ZERO_EXTEND;
2052 
2053       LLVMContext &Context = F->getContext();
2054       bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2055 
2056       for (unsigned j = 0; j != NumValues; ++j) {
2057         EVT VT = ValueVTs[j];
2058 
2059         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2060           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2061 
2062         CallingConv::ID CC = F->getCallingConv();
2063 
2064         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2065         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2066         SmallVector<SDValue, 4> Parts(NumParts);
2067         getCopyToParts(DAG, getCurSDLoc(),
2068                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2069                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2070 
2071         // 'inreg' on function refers to return value
2072         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2073         if (RetInReg)
2074           Flags.setInReg();
2075 
2076         if (I.getOperand(0)->getType()->isPointerTy()) {
2077           Flags.setPointer();
2078           Flags.setPointerAddrSpace(
2079               cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2080         }
2081 
2082         if (NeedsRegBlock) {
2083           Flags.setInConsecutiveRegs();
2084           if (j == NumValues - 1)
2085             Flags.setInConsecutiveRegsLast();
2086         }
2087 
2088         // Propagate extension type if any
2089         if (ExtendKind == ISD::SIGN_EXTEND)
2090           Flags.setSExt();
2091         else if (ExtendKind == ISD::ZERO_EXTEND)
2092           Flags.setZExt();
2093 
2094         for (unsigned i = 0; i < NumParts; ++i) {
2095           Outs.push_back(ISD::OutputArg(Flags,
2096                                         Parts[i].getValueType().getSimpleVT(),
2097                                         VT, /*isfixed=*/true, 0, 0));
2098           OutVals.push_back(Parts[i]);
2099         }
2100       }
2101     }
2102   }
2103 
2104   // Push in swifterror virtual register as the last element of Outs. This makes
2105   // sure swifterror virtual register will be returned in the swifterror
2106   // physical register.
2107   const Function *F = I.getParent()->getParent();
2108   if (TLI.supportSwiftError() &&
2109       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2110     assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2111     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2112     Flags.setSwiftError();
2113     Outs.push_back(ISD::OutputArg(
2114         Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2115         /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2116     // Create SDNode for the swifterror virtual register.
2117     OutVals.push_back(
2118         DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2119                             &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2120                         EVT(TLI.getPointerTy(DL))));
2121   }
2122 
2123   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2124   CallingConv::ID CallConv =
2125     DAG.getMachineFunction().getFunction().getCallingConv();
2126   Chain = DAG.getTargetLoweringInfo().LowerReturn(
2127       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2128 
2129   // Verify that the target's LowerReturn behaved as expected.
2130   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2131          "LowerReturn didn't return a valid chain!");
2132 
2133   // Update the DAG with the new chain value resulting from return lowering.
2134   DAG.setRoot(Chain);
2135 }
2136 
2137 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2138 /// created for it, emit nodes to copy the value into the virtual
2139 /// registers.
2140 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2141   // Skip empty types
2142   if (V->getType()->isEmptyTy())
2143     return;
2144 
2145   DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
2146   if (VMI != FuncInfo.ValueMap.end()) {
2147     assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2148            "Unused value assigned virtual registers!");
2149     CopyValueToVirtualRegister(V, VMI->second);
2150   }
2151 }
2152 
2153 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2154 /// the current basic block, add it to ValueMap now so that we'll get a
2155 /// CopyTo/FromReg.
2156 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2157   // No need to export constants.
2158   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2159 
2160   // Already exported?
2161   if (FuncInfo.isExportedInst(V)) return;
2162 
2163   Register Reg = FuncInfo.InitializeRegForValue(V);
2164   CopyValueToVirtualRegister(V, Reg);
2165 }
2166 
2167 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2168                                                      const BasicBlock *FromBB) {
2169   // The operands of the setcc have to be in this block.  We don't know
2170   // how to export them from some other block.
2171   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2172     // Can export from current BB.
2173     if (VI->getParent() == FromBB)
2174       return true;
2175 
2176     // Is already exported, noop.
2177     return FuncInfo.isExportedInst(V);
2178   }
2179 
2180   // If this is an argument, we can export it if the BB is the entry block or
2181   // if it is already exported.
2182   if (isa<Argument>(V)) {
2183     if (FromBB->isEntryBlock())
2184       return true;
2185 
2186     // Otherwise, can only export this if it is already exported.
2187     return FuncInfo.isExportedInst(V);
2188   }
2189 
2190   // Otherwise, constants can always be exported.
2191   return true;
2192 }
2193 
2194 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2195 BranchProbability
2196 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2197                                         const MachineBasicBlock *Dst) const {
2198   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2199   const BasicBlock *SrcBB = Src->getBasicBlock();
2200   const BasicBlock *DstBB = Dst->getBasicBlock();
2201   if (!BPI) {
2202     // If BPI is not available, set the default probability as 1 / N, where N is
2203     // the number of successors.
2204     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2205     return BranchProbability(1, SuccSize);
2206   }
2207   return BPI->getEdgeProbability(SrcBB, DstBB);
2208 }
2209 
2210 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2211                                                MachineBasicBlock *Dst,
2212                                                BranchProbability Prob) {
2213   if (!FuncInfo.BPI)
2214     Src->addSuccessorWithoutProb(Dst);
2215   else {
2216     if (Prob.isUnknown())
2217       Prob = getEdgeProbability(Src, Dst);
2218     Src->addSuccessor(Dst, Prob);
2219   }
2220 }
2221 
2222 static bool InBlock(const Value *V, const BasicBlock *BB) {
2223   if (const Instruction *I = dyn_cast<Instruction>(V))
2224     return I->getParent() == BB;
2225   return true;
2226 }
2227 
2228 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2229 /// This function emits a branch and is used at the leaves of an OR or an
2230 /// AND operator tree.
2231 void
2232 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2233                                                   MachineBasicBlock *TBB,
2234                                                   MachineBasicBlock *FBB,
2235                                                   MachineBasicBlock *CurBB,
2236                                                   MachineBasicBlock *SwitchBB,
2237                                                   BranchProbability TProb,
2238                                                   BranchProbability FProb,
2239                                                   bool InvertCond) {
2240   const BasicBlock *BB = CurBB->getBasicBlock();
2241 
2242   // If the leaf of the tree is a comparison, merge the condition into
2243   // the caseblock.
2244   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2245     // The operands of the cmp have to be in this block.  We don't know
2246     // how to export them from some other block.  If this is the first block
2247     // of the sequence, no exporting is needed.
2248     if (CurBB == SwitchBB ||
2249         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2250          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2251       ISD::CondCode Condition;
2252       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2253         ICmpInst::Predicate Pred =
2254             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2255         Condition = getICmpCondCode(Pred);
2256       } else {
2257         const FCmpInst *FC = cast<FCmpInst>(Cond);
2258         FCmpInst::Predicate Pred =
2259             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2260         Condition = getFCmpCondCode(Pred);
2261         if (TM.Options.NoNaNsFPMath)
2262           Condition = getFCmpCodeWithoutNaN(Condition);
2263       }
2264 
2265       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2266                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2267       SL->SwitchCases.push_back(CB);
2268       return;
2269     }
2270   }
2271 
2272   // Create a CaseBlock record representing this branch.
2273   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2274   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2275                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2276   SL->SwitchCases.push_back(CB);
2277 }
2278 
2279 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2280                                                MachineBasicBlock *TBB,
2281                                                MachineBasicBlock *FBB,
2282                                                MachineBasicBlock *CurBB,
2283                                                MachineBasicBlock *SwitchBB,
2284                                                Instruction::BinaryOps Opc,
2285                                                BranchProbability TProb,
2286                                                BranchProbability FProb,
2287                                                bool InvertCond) {
2288   // Skip over not part of the tree and remember to invert op and operands at
2289   // next level.
2290   Value *NotCond;
2291   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2292       InBlock(NotCond, CurBB->getBasicBlock())) {
2293     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2294                          !InvertCond);
2295     return;
2296   }
2297 
2298   const Instruction *BOp = dyn_cast<Instruction>(Cond);
2299   const Value *BOpOp0, *BOpOp1;
2300   // Compute the effective opcode for Cond, taking into account whether it needs
2301   // to be inverted, e.g.
2302   //   and (not (or A, B)), C
2303   // gets lowered as
2304   //   and (and (not A, not B), C)
2305   Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2306   if (BOp) {
2307     BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2308                ? Instruction::And
2309                : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2310                       ? Instruction::Or
2311                       : (Instruction::BinaryOps)0);
2312     if (InvertCond) {
2313       if (BOpc == Instruction::And)
2314         BOpc = Instruction::Or;
2315       else if (BOpc == Instruction::Or)
2316         BOpc = Instruction::And;
2317     }
2318   }
2319 
2320   // If this node is not part of the or/and tree, emit it as a branch.
2321   // Note that all nodes in the tree should have same opcode.
2322   bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2323   if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2324       !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2325       !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2326     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2327                                  TProb, FProb, InvertCond);
2328     return;
2329   }
2330 
2331   //  Create TmpBB after CurBB.
2332   MachineFunction::iterator BBI(CurBB);
2333   MachineFunction &MF = DAG.getMachineFunction();
2334   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2335   CurBB->getParent()->insert(++BBI, TmpBB);
2336 
2337   if (Opc == Instruction::Or) {
2338     // Codegen X | Y as:
2339     // BB1:
2340     //   jmp_if_X TBB
2341     //   jmp TmpBB
2342     // TmpBB:
2343     //   jmp_if_Y TBB
2344     //   jmp FBB
2345     //
2346 
2347     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2348     // The requirement is that
2349     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2350     //     = TrueProb for original BB.
2351     // Assuming the original probabilities are A and B, one choice is to set
2352     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2353     // A/(1+B) and 2B/(1+B). This choice assumes that
2354     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2355     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2356     // TmpBB, but the math is more complicated.
2357 
2358     auto NewTrueProb = TProb / 2;
2359     auto NewFalseProb = TProb / 2 + FProb;
2360     // Emit the LHS condition.
2361     FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2362                          NewFalseProb, InvertCond);
2363 
2364     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2365     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2366     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2367     // Emit the RHS condition into TmpBB.
2368     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2369                          Probs[1], InvertCond);
2370   } else {
2371     assert(Opc == Instruction::And && "Unknown merge op!");
2372     // Codegen X & Y as:
2373     // BB1:
2374     //   jmp_if_X TmpBB
2375     //   jmp FBB
2376     // TmpBB:
2377     //   jmp_if_Y TBB
2378     //   jmp FBB
2379     //
2380     //  This requires creation of TmpBB after CurBB.
2381 
2382     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2383     // The requirement is that
2384     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2385     //     = FalseProb for original BB.
2386     // Assuming the original probabilities are A and B, one choice is to set
2387     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2388     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2389     // TrueProb for BB1 * FalseProb for TmpBB.
2390 
2391     auto NewTrueProb = TProb + FProb / 2;
2392     auto NewFalseProb = FProb / 2;
2393     // Emit the LHS condition.
2394     FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2395                          NewFalseProb, InvertCond);
2396 
2397     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2398     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2399     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2400     // Emit the RHS condition into TmpBB.
2401     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2402                          Probs[1], InvertCond);
2403   }
2404 }
2405 
2406 /// If the set of cases should be emitted as a series of branches, return true.
2407 /// If we should emit this as a bunch of and/or'd together conditions, return
2408 /// false.
2409 bool
2410 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2411   if (Cases.size() != 2) return true;
2412 
2413   // If this is two comparisons of the same values or'd or and'd together, they
2414   // will get folded into a single comparison, so don't emit two blocks.
2415   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2416        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2417       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2418        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2419     return false;
2420   }
2421 
2422   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2423   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2424   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2425       Cases[0].CC == Cases[1].CC &&
2426       isa<Constant>(Cases[0].CmpRHS) &&
2427       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2428     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2429       return false;
2430     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2431       return false;
2432   }
2433 
2434   return true;
2435 }
2436 
2437 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2438   MachineBasicBlock *BrMBB = FuncInfo.MBB;
2439 
2440   // Update machine-CFG edges.
2441   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2442 
2443   if (I.isUnconditional()) {
2444     // Update machine-CFG edges.
2445     BrMBB->addSuccessor(Succ0MBB);
2446 
2447     // If this is not a fall-through branch or optimizations are switched off,
2448     // emit the branch.
2449     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2450       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2451                               MVT::Other, getControlRoot(),
2452                               DAG.getBasicBlock(Succ0MBB)));
2453 
2454     return;
2455   }
2456 
2457   // If this condition is one of the special cases we handle, do special stuff
2458   // now.
2459   const Value *CondVal = I.getCondition();
2460   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2461 
2462   // If this is a series of conditions that are or'd or and'd together, emit
2463   // this as a sequence of branches instead of setcc's with and/or operations.
2464   // As long as jumps are not expensive (exceptions for multi-use logic ops,
2465   // unpredictable branches, and vector extracts because those jumps are likely
2466   // expensive for any target), this should improve performance.
2467   // For example, instead of something like:
2468   //     cmp A, B
2469   //     C = seteq
2470   //     cmp D, E
2471   //     F = setle
2472   //     or C, F
2473   //     jnz foo
2474   // Emit:
2475   //     cmp A, B
2476   //     je foo
2477   //     cmp D, E
2478   //     jle foo
2479   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2480   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2481       BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2482     Value *Vec;
2483     const Value *BOp0, *BOp1;
2484     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2485     if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2486       Opcode = Instruction::And;
2487     else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2488       Opcode = Instruction::Or;
2489 
2490     if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2491                     match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2492       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2493                            getEdgeProbability(BrMBB, Succ0MBB),
2494                            getEdgeProbability(BrMBB, Succ1MBB),
2495                            /*InvertCond=*/false);
2496       // If the compares in later blocks need to use values not currently
2497       // exported from this block, export them now.  This block should always
2498       // be the first entry.
2499       assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2500 
2501       // Allow some cases to be rejected.
2502       if (ShouldEmitAsBranches(SL->SwitchCases)) {
2503         for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2504           ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2505           ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2506         }
2507 
2508         // Emit the branch for this block.
2509         visitSwitchCase(SL->SwitchCases[0], BrMBB);
2510         SL->SwitchCases.erase(SL->SwitchCases.begin());
2511         return;
2512       }
2513 
2514       // Okay, we decided not to do this, remove any inserted MBB's and clear
2515       // SwitchCases.
2516       for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2517         FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2518 
2519       SL->SwitchCases.clear();
2520     }
2521   }
2522 
2523   // Create a CaseBlock record representing this branch.
2524   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2525                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2526 
2527   // Use visitSwitchCase to actually insert the fast branch sequence for this
2528   // cond branch.
2529   visitSwitchCase(CB, BrMBB);
2530 }
2531 
2532 /// visitSwitchCase - Emits the necessary code to represent a single node in
2533 /// the binary search tree resulting from lowering a switch instruction.
2534 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2535                                           MachineBasicBlock *SwitchBB) {
2536   SDValue Cond;
2537   SDValue CondLHS = getValue(CB.CmpLHS);
2538   SDLoc dl = CB.DL;
2539 
2540   if (CB.CC == ISD::SETTRUE) {
2541     // Branch or fall through to TrueBB.
2542     addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2543     SwitchBB->normalizeSuccProbs();
2544     if (CB.TrueBB != NextBlock(SwitchBB)) {
2545       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2546                               DAG.getBasicBlock(CB.TrueBB)));
2547     }
2548     return;
2549   }
2550 
2551   auto &TLI = DAG.getTargetLoweringInfo();
2552   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2553 
2554   // Build the setcc now.
2555   if (!CB.CmpMHS) {
2556     // Fold "(X == true)" to X and "(X == false)" to !X to
2557     // handle common cases produced by branch lowering.
2558     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2559         CB.CC == ISD::SETEQ)
2560       Cond = CondLHS;
2561     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2562              CB.CC == ISD::SETEQ) {
2563       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2564       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2565     } else {
2566       SDValue CondRHS = getValue(CB.CmpRHS);
2567 
2568       // If a pointer's DAG type is larger than its memory type then the DAG
2569       // values are zero-extended. This breaks signed comparisons so truncate
2570       // back to the underlying type before doing the compare.
2571       if (CondLHS.getValueType() != MemVT) {
2572         CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2573         CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2574       }
2575       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2576     }
2577   } else {
2578     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2579 
2580     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2581     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2582 
2583     SDValue CmpOp = getValue(CB.CmpMHS);
2584     EVT VT = CmpOp.getValueType();
2585 
2586     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2587       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2588                           ISD::SETLE);
2589     } else {
2590       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2591                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2592       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2593                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2594     }
2595   }
2596 
2597   // Update successor info
2598   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2599   // TrueBB and FalseBB are always different unless the incoming IR is
2600   // degenerate. This only happens when running llc on weird IR.
2601   if (CB.TrueBB != CB.FalseBB)
2602     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2603   SwitchBB->normalizeSuccProbs();
2604 
2605   // If the lhs block is the next block, invert the condition so that we can
2606   // fall through to the lhs instead of the rhs block.
2607   if (CB.TrueBB == NextBlock(SwitchBB)) {
2608     std::swap(CB.TrueBB, CB.FalseBB);
2609     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2610     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2611   }
2612 
2613   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2614                                MVT::Other, getControlRoot(), Cond,
2615                                DAG.getBasicBlock(CB.TrueBB));
2616 
2617   setValue(CurInst, BrCond);
2618 
2619   // Insert the false branch. Do this even if it's a fall through branch,
2620   // this makes it easier to do DAG optimizations which require inverting
2621   // the branch condition.
2622   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2623                        DAG.getBasicBlock(CB.FalseBB));
2624 
2625   DAG.setRoot(BrCond);
2626 }
2627 
2628 /// visitJumpTable - Emit JumpTable node in the current MBB
2629 void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2630   // Emit the code for the jump table
2631   assert(JT.Reg != -1U && "Should lower JT Header first!");
2632   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2633   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2634                                      JT.Reg, PTy);
2635   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2636   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2637                                     MVT::Other, Index.getValue(1),
2638                                     Table, Index);
2639   DAG.setRoot(BrJumpTable);
2640 }
2641 
2642 /// visitJumpTableHeader - This function emits necessary code to produce index
2643 /// in the JumpTable from switch case.
2644 void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2645                                                JumpTableHeader &JTH,
2646                                                MachineBasicBlock *SwitchBB) {
2647   SDLoc dl = getCurSDLoc();
2648 
2649   // Subtract the lowest switch case value from the value being switched on.
2650   SDValue SwitchOp = getValue(JTH.SValue);
2651   EVT VT = SwitchOp.getValueType();
2652   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2653                             DAG.getConstant(JTH.First, dl, VT));
2654 
2655   // The SDNode we just created, which holds the value being switched on minus
2656   // the smallest case value, needs to be copied to a virtual register so it
2657   // can be used as an index into the jump table in a subsequent basic block.
2658   // This value may be smaller or larger than the target's pointer type, and
2659   // therefore require extension or truncating.
2660   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2661   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2662 
2663   unsigned JumpTableReg =
2664       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2665   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2666                                     JumpTableReg, SwitchOp);
2667   JT.Reg = JumpTableReg;
2668 
2669   if (!JTH.FallthroughUnreachable) {
2670     // Emit the range check for the jump table, and branch to the default block
2671     // for the switch statement if the value being switched on exceeds the
2672     // largest case in the switch.
2673     SDValue CMP = DAG.getSetCC(
2674         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2675                                    Sub.getValueType()),
2676         Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2677 
2678     SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2679                                  MVT::Other, CopyTo, CMP,
2680                                  DAG.getBasicBlock(JT.Default));
2681 
2682     // Avoid emitting unnecessary branches to the next block.
2683     if (JT.MBB != NextBlock(SwitchBB))
2684       BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2685                            DAG.getBasicBlock(JT.MBB));
2686 
2687     DAG.setRoot(BrCond);
2688   } else {
2689     // Avoid emitting unnecessary branches to the next block.
2690     if (JT.MBB != NextBlock(SwitchBB))
2691       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2692                               DAG.getBasicBlock(JT.MBB)));
2693     else
2694       DAG.setRoot(CopyTo);
2695   }
2696 }
2697 
2698 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2699 /// variable if there exists one.
2700 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2701                                  SDValue &Chain) {
2702   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2703   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2704   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2705   MachineFunction &MF = DAG.getMachineFunction();
2706   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2707   MachineSDNode *Node =
2708       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2709   if (Global) {
2710     MachinePointerInfo MPInfo(Global);
2711     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2712                  MachineMemOperand::MODereferenceable;
2713     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2714         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2715     DAG.setNodeMemRefs(Node, {MemRef});
2716   }
2717   if (PtrTy != PtrMemTy)
2718     return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2719   return SDValue(Node, 0);
2720 }
2721 
2722 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2723 /// tail spliced into a stack protector check success bb.
2724 ///
2725 /// For a high level explanation of how this fits into the stack protector
2726 /// generation see the comment on the declaration of class
2727 /// StackProtectorDescriptor.
2728 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2729                                                   MachineBasicBlock *ParentBB) {
2730 
2731   // First create the loads to the guard/stack slot for the comparison.
2732   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2733   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2734   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2735 
2736   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2737   int FI = MFI.getStackProtectorIndex();
2738 
2739   SDValue Guard;
2740   SDLoc dl = getCurSDLoc();
2741   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2742   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2743   Align Align =
2744       DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2745 
2746   // Generate code to load the content of the guard slot.
2747   SDValue GuardVal = DAG.getLoad(
2748       PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2749       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2750       MachineMemOperand::MOVolatile);
2751 
2752   if (TLI.useStackGuardXorFP())
2753     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2754 
2755   // Retrieve guard check function, nullptr if instrumentation is inlined.
2756   if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2757     // The target provides a guard check function to validate the guard value.
2758     // Generate a call to that function with the content of the guard slot as
2759     // argument.
2760     FunctionType *FnTy = GuardCheckFn->getFunctionType();
2761     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2762 
2763     TargetLowering::ArgListTy Args;
2764     TargetLowering::ArgListEntry Entry;
2765     Entry.Node = GuardVal;
2766     Entry.Ty = FnTy->getParamType(0);
2767     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2768       Entry.IsInReg = true;
2769     Args.push_back(Entry);
2770 
2771     TargetLowering::CallLoweringInfo CLI(DAG);
2772     CLI.setDebugLoc(getCurSDLoc())
2773         .setChain(DAG.getEntryNode())
2774         .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2775                    getValue(GuardCheckFn), std::move(Args));
2776 
2777     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2778     DAG.setRoot(Result.second);
2779     return;
2780   }
2781 
2782   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2783   // Otherwise, emit a volatile load to retrieve the stack guard value.
2784   SDValue Chain = DAG.getEntryNode();
2785   if (TLI.useLoadStackGuardNode()) {
2786     Guard = getLoadStackGuard(DAG, dl, Chain);
2787   } else {
2788     const Value *IRGuard = TLI.getSDagStackGuard(M);
2789     SDValue GuardPtr = getValue(IRGuard);
2790 
2791     Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2792                         MachinePointerInfo(IRGuard, 0), Align,
2793                         MachineMemOperand::MOVolatile);
2794   }
2795 
2796   // Perform the comparison via a getsetcc.
2797   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2798                                                         *DAG.getContext(),
2799                                                         Guard.getValueType()),
2800                              Guard, GuardVal, ISD::SETNE);
2801 
2802   // If the guard/stackslot do not equal, branch to failure MBB.
2803   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2804                                MVT::Other, GuardVal.getOperand(0),
2805                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2806   // Otherwise branch to success MBB.
2807   SDValue Br = DAG.getNode(ISD::BR, dl,
2808                            MVT::Other, BrCond,
2809                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2810 
2811   DAG.setRoot(Br);
2812 }
2813 
2814 /// Codegen the failure basic block for a stack protector check.
2815 ///
2816 /// A failure stack protector machine basic block consists simply of a call to
2817 /// __stack_chk_fail().
2818 ///
2819 /// For a high level explanation of how this fits into the stack protector
2820 /// generation see the comment on the declaration of class
2821 /// StackProtectorDescriptor.
2822 void
2823 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2824   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2825   TargetLowering::MakeLibCallOptions CallOptions;
2826   CallOptions.setDiscardResult(true);
2827   SDValue Chain =
2828       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2829                       std::nullopt, CallOptions, getCurSDLoc())
2830           .second;
2831   // On PS4/PS5, the "return address" must still be within the calling
2832   // function, even if it's at the very end, so emit an explicit TRAP here.
2833   // Passing 'true' for doesNotReturn above won't generate the trap for us.
2834   if (TM.getTargetTriple().isPS())
2835     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2836   // WebAssembly needs an unreachable instruction after a non-returning call,
2837   // because the function return type can be different from __stack_chk_fail's
2838   // return type (void).
2839   if (TM.getTargetTriple().isWasm())
2840     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2841 
2842   DAG.setRoot(Chain);
2843 }
2844 
2845 /// visitBitTestHeader - This function emits necessary code to produce value
2846 /// suitable for "bit tests"
2847 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2848                                              MachineBasicBlock *SwitchBB) {
2849   SDLoc dl = getCurSDLoc();
2850 
2851   // Subtract the minimum value.
2852   SDValue SwitchOp = getValue(B.SValue);
2853   EVT VT = SwitchOp.getValueType();
2854   SDValue RangeSub =
2855       DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2856 
2857   // Determine the type of the test operands.
2858   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2859   bool UsePtrType = false;
2860   if (!TLI.isTypeLegal(VT)) {
2861     UsePtrType = true;
2862   } else {
2863     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2864       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2865         // Switch table case range are encoded into series of masks.
2866         // Just use pointer type, it's guaranteed to fit.
2867         UsePtrType = true;
2868         break;
2869       }
2870   }
2871   SDValue Sub = RangeSub;
2872   if (UsePtrType) {
2873     VT = TLI.getPointerTy(DAG.getDataLayout());
2874     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2875   }
2876 
2877   B.RegVT = VT.getSimpleVT();
2878   B.Reg = FuncInfo.CreateReg(B.RegVT);
2879   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2880 
2881   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2882 
2883   if (!B.FallthroughUnreachable)
2884     addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2885   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2886   SwitchBB->normalizeSuccProbs();
2887 
2888   SDValue Root = CopyTo;
2889   if (!B.FallthroughUnreachable) {
2890     // Conditional branch to the default block.
2891     SDValue RangeCmp = DAG.getSetCC(dl,
2892         TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2893                                RangeSub.getValueType()),
2894         RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2895         ISD::SETUGT);
2896 
2897     Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2898                        DAG.getBasicBlock(B.Default));
2899   }
2900 
2901   // Avoid emitting unnecessary branches to the next block.
2902   if (MBB != NextBlock(SwitchBB))
2903     Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2904 
2905   DAG.setRoot(Root);
2906 }
2907 
2908 /// visitBitTestCase - this function produces one "bit test"
2909 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2910                                            MachineBasicBlock* NextMBB,
2911                                            BranchProbability BranchProbToNext,
2912                                            unsigned Reg,
2913                                            BitTestCase &B,
2914                                            MachineBasicBlock *SwitchBB) {
2915   SDLoc dl = getCurSDLoc();
2916   MVT VT = BB.RegVT;
2917   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2918   SDValue Cmp;
2919   unsigned PopCount = llvm::popcount(B.Mask);
2920   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2921   if (PopCount == 1) {
2922     // Testing for a single bit; just compare the shift count with what it
2923     // would need to be to shift a 1 bit in that position.
2924     Cmp = DAG.getSetCC(
2925         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2926         ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
2927         ISD::SETEQ);
2928   } else if (PopCount == BB.Range) {
2929     // There is only one zero bit in the range, test for it directly.
2930     Cmp = DAG.getSetCC(
2931         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2932         ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
2933   } else {
2934     // Make desired shift
2935     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2936                                     DAG.getConstant(1, dl, VT), ShiftOp);
2937 
2938     // Emit bit tests and jumps
2939     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2940                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2941     Cmp = DAG.getSetCC(
2942         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2943         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2944   }
2945 
2946   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2947   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2948   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2949   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2950   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2951   // one as they are relative probabilities (and thus work more like weights),
2952   // and hence we need to normalize them to let the sum of them become one.
2953   SwitchBB->normalizeSuccProbs();
2954 
2955   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2956                               MVT::Other, getControlRoot(),
2957                               Cmp, DAG.getBasicBlock(B.TargetBB));
2958 
2959   // Avoid emitting unnecessary branches to the next block.
2960   if (NextMBB != NextBlock(SwitchBB))
2961     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2962                         DAG.getBasicBlock(NextMBB));
2963 
2964   DAG.setRoot(BrAnd);
2965 }
2966 
2967 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2968   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2969 
2970   // Retrieve successors. Look through artificial IR level blocks like
2971   // catchswitch for successors.
2972   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2973   const BasicBlock *EHPadBB = I.getSuccessor(1);
2974   MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
2975 
2976   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2977   // have to do anything here to lower funclet bundles.
2978   assert(!I.hasOperandBundlesOtherThan(
2979              {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2980               LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2981               LLVMContext::OB_cfguardtarget,
2982               LLVMContext::OB_clang_arc_attachedcall}) &&
2983          "Cannot lower invokes with arbitrary operand bundles yet!");
2984 
2985   const Value *Callee(I.getCalledOperand());
2986   const Function *Fn = dyn_cast<Function>(Callee);
2987   if (isa<InlineAsm>(Callee))
2988     visitInlineAsm(I, EHPadBB);
2989   else if (Fn && Fn->isIntrinsic()) {
2990     switch (Fn->getIntrinsicID()) {
2991     default:
2992       llvm_unreachable("Cannot invoke this intrinsic");
2993     case Intrinsic::donothing:
2994       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2995     case Intrinsic::seh_try_begin:
2996     case Intrinsic::seh_scope_begin:
2997     case Intrinsic::seh_try_end:
2998     case Intrinsic::seh_scope_end:
2999       if (EHPadMBB)
3000           // a block referenced by EH table
3001           // so dtor-funclet not removed by opts
3002           EHPadMBB->setMachineBlockAddressTaken();
3003       break;
3004     case Intrinsic::experimental_patchpoint_void:
3005     case Intrinsic::experimental_patchpoint_i64:
3006       visitPatchpoint(I, EHPadBB);
3007       break;
3008     case Intrinsic::experimental_gc_statepoint:
3009       LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3010       break;
3011     case Intrinsic::wasm_rethrow: {
3012       // This is usually done in visitTargetIntrinsic, but this intrinsic is
3013       // special because it can be invoked, so we manually lower it to a DAG
3014       // node here.
3015       SmallVector<SDValue, 8> Ops;
3016       Ops.push_back(getRoot()); // inchain
3017       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3018       Ops.push_back(
3019           DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3020                                 TLI.getPointerTy(DAG.getDataLayout())));
3021       SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3022       DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3023       break;
3024     }
3025     }
3026   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3027     // Currently we do not lower any intrinsic calls with deopt operand bundles.
3028     // Eventually we will support lowering the @llvm.experimental.deoptimize
3029     // intrinsic, and right now there are no plans to support other intrinsics
3030     // with deopt state.
3031     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3032   } else {
3033     LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3034   }
3035 
3036   // If the value of the invoke is used outside of its defining block, make it
3037   // available as a virtual register.
3038   // We already took care of the exported value for the statepoint instruction
3039   // during call to the LowerStatepoint.
3040   if (!isa<GCStatepointInst>(I)) {
3041     CopyToExportRegsIfNeeded(&I);
3042   }
3043 
3044   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3045   BranchProbabilityInfo *BPI = FuncInfo.BPI;
3046   BranchProbability EHPadBBProb =
3047       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3048           : BranchProbability::getZero();
3049   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3050 
3051   // Update successor info.
3052   addSuccessorWithProb(InvokeMBB, Return);
3053   for (auto &UnwindDest : UnwindDests) {
3054     UnwindDest.first->setIsEHPad();
3055     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3056   }
3057   InvokeMBB->normalizeSuccProbs();
3058 
3059   // Drop into normal successor.
3060   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3061                           DAG.getBasicBlock(Return)));
3062 }
3063 
3064 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3065   MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3066 
3067   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3068   // have to do anything here to lower funclet bundles.
3069   assert(!I.hasOperandBundlesOtherThan(
3070              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3071          "Cannot lower callbrs with arbitrary operand bundles yet!");
3072 
3073   assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3074   visitInlineAsm(I);
3075   CopyToExportRegsIfNeeded(&I);
3076 
3077   // Retrieve successors.
3078   SmallPtrSet<BasicBlock *, 8> Dests;
3079   Dests.insert(I.getDefaultDest());
3080   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3081 
3082   // Update successor info.
3083   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3084   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3085     BasicBlock *Dest = I.getIndirectDest(i);
3086     MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];
3087     Target->setIsInlineAsmBrIndirectTarget();
3088     Target->setMachineBlockAddressTaken();
3089     Target->setLabelMustBeEmitted();
3090     // Don't add duplicate machine successors.
3091     if (Dests.insert(Dest).second)
3092       addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3093   }
3094   CallBrMBB->normalizeSuccProbs();
3095 
3096   // Drop into default successor.
3097   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3098                           MVT::Other, getControlRoot(),
3099                           DAG.getBasicBlock(Return)));
3100 }
3101 
3102 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3103   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3104 }
3105 
3106 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3107   assert(FuncInfo.MBB->isEHPad() &&
3108          "Call to landingpad not in landing pad!");
3109 
3110   // If there aren't registers to copy the values into (e.g., during SjLj
3111   // exceptions), then don't bother to create these DAG nodes.
3112   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3113   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3114   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3115       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3116     return;
3117 
3118   // If landingpad's return type is token type, we don't create DAG nodes
3119   // for its exception pointer and selector value. The extraction of exception
3120   // pointer or selector value from token type landingpads is not currently
3121   // supported.
3122   if (LP.getType()->isTokenTy())
3123     return;
3124 
3125   SmallVector<EVT, 2> ValueVTs;
3126   SDLoc dl = getCurSDLoc();
3127   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3128   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3129 
3130   // Get the two live-in registers as SDValues. The physregs have already been
3131   // copied into virtual registers.
3132   SDValue Ops[2];
3133   if (FuncInfo.ExceptionPointerVirtReg) {
3134     Ops[0] = DAG.getZExtOrTrunc(
3135         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3136                            FuncInfo.ExceptionPointerVirtReg,
3137                            TLI.getPointerTy(DAG.getDataLayout())),
3138         dl, ValueVTs[0]);
3139   } else {
3140     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3141   }
3142   Ops[1] = DAG.getZExtOrTrunc(
3143       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3144                          FuncInfo.ExceptionSelectorVirtReg,
3145                          TLI.getPointerTy(DAG.getDataLayout())),
3146       dl, ValueVTs[1]);
3147 
3148   // Merge into one.
3149   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3150                             DAG.getVTList(ValueVTs), Ops);
3151   setValue(&LP, Res);
3152 }
3153 
3154 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3155                                            MachineBasicBlock *Last) {
3156   // Update JTCases.
3157   for (JumpTableBlock &JTB : SL->JTCases)
3158     if (JTB.first.HeaderBB == First)
3159       JTB.first.HeaderBB = Last;
3160 
3161   // Update BitTestCases.
3162   for (BitTestBlock &BTB : SL->BitTestCases)
3163     if (BTB.Parent == First)
3164       BTB.Parent = Last;
3165 }
3166 
3167 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3168   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3169 
3170   // Update machine-CFG edges with unique successors.
3171   SmallSet<BasicBlock*, 32> Done;
3172   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3173     BasicBlock *BB = I.getSuccessor(i);
3174     bool Inserted = Done.insert(BB).second;
3175     if (!Inserted)
3176         continue;
3177 
3178     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3179     addSuccessorWithProb(IndirectBrMBB, Succ);
3180   }
3181   IndirectBrMBB->normalizeSuccProbs();
3182 
3183   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3184                           MVT::Other, getControlRoot(),
3185                           getValue(I.getAddress())));
3186 }
3187 
3188 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3189   if (!DAG.getTarget().Options.TrapUnreachable)
3190     return;
3191 
3192   // We may be able to ignore unreachable behind a noreturn call.
3193   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
3194     const BasicBlock &BB = *I.getParent();
3195     if (&I != &BB.front()) {
3196       BasicBlock::const_iterator PredI =
3197         std::prev(BasicBlock::const_iterator(&I));
3198       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3199         if (Call->doesNotReturn())
3200           return;
3201       }
3202     }
3203   }
3204 
3205   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3206 }
3207 
3208 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3209   SDNodeFlags Flags;
3210   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3211     Flags.copyFMF(*FPOp);
3212 
3213   SDValue Op = getValue(I.getOperand(0));
3214   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3215                                     Op, Flags);
3216   setValue(&I, UnNodeValue);
3217 }
3218 
3219 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3220   SDNodeFlags Flags;
3221   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3222     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3223     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3224   }
3225   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3226     Flags.setExact(ExactOp->isExact());
3227   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3228     Flags.copyFMF(*FPOp);
3229 
3230   SDValue Op1 = getValue(I.getOperand(0));
3231   SDValue Op2 = getValue(I.getOperand(1));
3232   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3233                                      Op1, Op2, Flags);
3234   setValue(&I, BinNodeValue);
3235 }
3236 
3237 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3238   SDValue Op1 = getValue(I.getOperand(0));
3239   SDValue Op2 = getValue(I.getOperand(1));
3240 
3241   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3242       Op1.getValueType(), DAG.getDataLayout());
3243 
3244   // Coerce the shift amount to the right type if we can. This exposes the
3245   // truncate or zext to optimization early.
3246   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3247     assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3248            "Unexpected shift type");
3249     Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3250   }
3251 
3252   bool nuw = false;
3253   bool nsw = false;
3254   bool exact = false;
3255 
3256   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3257 
3258     if (const OverflowingBinaryOperator *OFBinOp =
3259             dyn_cast<const OverflowingBinaryOperator>(&I)) {
3260       nuw = OFBinOp->hasNoUnsignedWrap();
3261       nsw = OFBinOp->hasNoSignedWrap();
3262     }
3263     if (const PossiblyExactOperator *ExactOp =
3264             dyn_cast<const PossiblyExactOperator>(&I))
3265       exact = ExactOp->isExact();
3266   }
3267   SDNodeFlags Flags;
3268   Flags.setExact(exact);
3269   Flags.setNoSignedWrap(nsw);
3270   Flags.setNoUnsignedWrap(nuw);
3271   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3272                             Flags);
3273   setValue(&I, Res);
3274 }
3275 
3276 void SelectionDAGBuilder::visitSDiv(const User &I) {
3277   SDValue Op1 = getValue(I.getOperand(0));
3278   SDValue Op2 = getValue(I.getOperand(1));
3279 
3280   SDNodeFlags Flags;
3281   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3282                  cast<PossiblyExactOperator>(&I)->isExact());
3283   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3284                            Op2, Flags));
3285 }
3286 
3287 void SelectionDAGBuilder::visitICmp(const User &I) {
3288   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3289   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3290     predicate = IC->getPredicate();
3291   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3292     predicate = ICmpInst::Predicate(IC->getPredicate());
3293   SDValue Op1 = getValue(I.getOperand(0));
3294   SDValue Op2 = getValue(I.getOperand(1));
3295   ISD::CondCode Opcode = getICmpCondCode(predicate);
3296 
3297   auto &TLI = DAG.getTargetLoweringInfo();
3298   EVT MemVT =
3299       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3300 
3301   // If a pointer's DAG type is larger than its memory type then the DAG values
3302   // are zero-extended. This breaks signed comparisons so truncate back to the
3303   // underlying type before doing the compare.
3304   if (Op1.getValueType() != MemVT) {
3305     Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3306     Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3307   }
3308 
3309   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3310                                                         I.getType());
3311   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3312 }
3313 
3314 void SelectionDAGBuilder::visitFCmp(const User &I) {
3315   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3316   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3317     predicate = FC->getPredicate();
3318   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3319     predicate = FCmpInst::Predicate(FC->getPredicate());
3320   SDValue Op1 = getValue(I.getOperand(0));
3321   SDValue Op2 = getValue(I.getOperand(1));
3322 
3323   ISD::CondCode Condition = getFCmpCondCode(predicate);
3324   auto *FPMO = cast<FPMathOperator>(&I);
3325   if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3326     Condition = getFCmpCodeWithoutNaN(Condition);
3327 
3328   SDNodeFlags Flags;
3329   Flags.copyFMF(*FPMO);
3330   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3331 
3332   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3333                                                         I.getType());
3334   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3335 }
3336 
3337 // Check if the condition of the select has one use or two users that are both
3338 // selects with the same condition.
3339 static bool hasOnlySelectUsers(const Value *Cond) {
3340   return llvm::all_of(Cond->users(), [](const Value *V) {
3341     return isa<SelectInst>(V);
3342   });
3343 }
3344 
3345 void SelectionDAGBuilder::visitSelect(const User &I) {
3346   SmallVector<EVT, 4> ValueVTs;
3347   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3348                   ValueVTs);
3349   unsigned NumValues = ValueVTs.size();
3350   if (NumValues == 0) return;
3351 
3352   SmallVector<SDValue, 4> Values(NumValues);
3353   SDValue Cond     = getValue(I.getOperand(0));
3354   SDValue LHSVal   = getValue(I.getOperand(1));
3355   SDValue RHSVal   = getValue(I.getOperand(2));
3356   SmallVector<SDValue, 1> BaseOps(1, Cond);
3357   ISD::NodeType OpCode =
3358       Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3359 
3360   bool IsUnaryAbs = false;
3361   bool Negate = false;
3362 
3363   SDNodeFlags Flags;
3364   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3365     Flags.copyFMF(*FPOp);
3366 
3367   // Min/max matching is only viable if all output VTs are the same.
3368   if (all_equal(ValueVTs)) {
3369     EVT VT = ValueVTs[0];
3370     LLVMContext &Ctx = *DAG.getContext();
3371     auto &TLI = DAG.getTargetLoweringInfo();
3372 
3373     // We care about the legality of the operation after it has been type
3374     // legalized.
3375     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3376       VT = TLI.getTypeToTransformTo(Ctx, VT);
3377 
3378     // If the vselect is legal, assume we want to leave this as a vector setcc +
3379     // vselect. Otherwise, if this is going to be scalarized, we want to see if
3380     // min/max is legal on the scalar type.
3381     bool UseScalarMinMax = VT.isVector() &&
3382       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3383 
3384     // ValueTracking's select pattern matching does not account for -0.0,
3385     // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3386     // -0.0 is less than +0.0.
3387     Value *LHS, *RHS;
3388     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3389     ISD::NodeType Opc = ISD::DELETED_NODE;
3390     switch (SPR.Flavor) {
3391     case SPF_UMAX:    Opc = ISD::UMAX; break;
3392     case SPF_UMIN:    Opc = ISD::UMIN; break;
3393     case SPF_SMAX:    Opc = ISD::SMAX; break;
3394     case SPF_SMIN:    Opc = ISD::SMIN; break;
3395     case SPF_FMINNUM:
3396       switch (SPR.NaNBehavior) {
3397       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3398       case SPNB_RETURNS_NAN: break;
3399       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3400       case SPNB_RETURNS_ANY:
3401         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ||
3402             (UseScalarMinMax &&
3403              TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType())))
3404           Opc = ISD::FMINNUM;
3405         break;
3406       }
3407       break;
3408     case SPF_FMAXNUM:
3409       switch (SPR.NaNBehavior) {
3410       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3411       case SPNB_RETURNS_NAN: break;
3412       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3413       case SPNB_RETURNS_ANY:
3414         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ||
3415             (UseScalarMinMax &&
3416              TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType())))
3417           Opc = ISD::FMAXNUM;
3418         break;
3419       }
3420       break;
3421     case SPF_NABS:
3422       Negate = true;
3423       [[fallthrough]];
3424     case SPF_ABS:
3425       IsUnaryAbs = true;
3426       Opc = ISD::ABS;
3427       break;
3428     default: break;
3429     }
3430 
3431     if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3432         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3433          (UseScalarMinMax &&
3434           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3435         // If the underlying comparison instruction is used by any other
3436         // instruction, the consumed instructions won't be destroyed, so it is
3437         // not profitable to convert to a min/max.
3438         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3439       OpCode = Opc;
3440       LHSVal = getValue(LHS);
3441       RHSVal = getValue(RHS);
3442       BaseOps.clear();
3443     }
3444 
3445     if (IsUnaryAbs) {
3446       OpCode = Opc;
3447       LHSVal = getValue(LHS);
3448       BaseOps.clear();
3449     }
3450   }
3451 
3452   if (IsUnaryAbs) {
3453     for (unsigned i = 0; i != NumValues; ++i) {
3454       SDLoc dl = getCurSDLoc();
3455       EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3456       Values[i] =
3457           DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3458       if (Negate)
3459         Values[i] = DAG.getNegative(Values[i], dl, VT);
3460     }
3461   } else {
3462     for (unsigned i = 0; i != NumValues; ++i) {
3463       SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3464       Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3465       Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3466       Values[i] = DAG.getNode(
3467           OpCode, getCurSDLoc(),
3468           LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3469     }
3470   }
3471 
3472   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3473                            DAG.getVTList(ValueVTs), Values));
3474 }
3475 
3476 void SelectionDAGBuilder::visitTrunc(const User &I) {
3477   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3478   SDValue N = getValue(I.getOperand(0));
3479   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3480                                                         I.getType());
3481   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3482 }
3483 
3484 void SelectionDAGBuilder::visitZExt(const User &I) {
3485   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3486   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3487   SDValue N = getValue(I.getOperand(0));
3488   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3489                                                         I.getType());
3490   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3491 }
3492 
3493 void SelectionDAGBuilder::visitSExt(const User &I) {
3494   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3495   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3496   SDValue N = getValue(I.getOperand(0));
3497   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3498                                                         I.getType());
3499   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3500 }
3501 
3502 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3503   // FPTrunc is never a no-op cast, no need to check
3504   SDValue N = getValue(I.getOperand(0));
3505   SDLoc dl = getCurSDLoc();
3506   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3507   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3508   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3509                            DAG.getTargetConstant(
3510                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3511 }
3512 
3513 void SelectionDAGBuilder::visitFPExt(const User &I) {
3514   // FPExt is never a no-op cast, no need to check
3515   SDValue N = getValue(I.getOperand(0));
3516   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3517                                                         I.getType());
3518   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3519 }
3520 
3521 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3522   // FPToUI is never a no-op cast, no need to check
3523   SDValue N = getValue(I.getOperand(0));
3524   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3525                                                         I.getType());
3526   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3527 }
3528 
3529 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3530   // FPToSI is never a no-op cast, no need to check
3531   SDValue N = getValue(I.getOperand(0));
3532   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3533                                                         I.getType());
3534   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3535 }
3536 
3537 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3538   // UIToFP is never a no-op cast, no need to check
3539   SDValue N = getValue(I.getOperand(0));
3540   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3541                                                         I.getType());
3542   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3543 }
3544 
3545 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3546   // SIToFP is never a no-op cast, no need to check
3547   SDValue N = getValue(I.getOperand(0));
3548   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3549                                                         I.getType());
3550   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3551 }
3552 
3553 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3554   // What to do depends on the size of the integer and the size of the pointer.
3555   // We can either truncate, zero extend, or no-op, accordingly.
3556   SDValue N = getValue(I.getOperand(0));
3557   auto &TLI = DAG.getTargetLoweringInfo();
3558   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3559                                                         I.getType());
3560   EVT PtrMemVT =
3561       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3562   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3563   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3564   setValue(&I, N);
3565 }
3566 
3567 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3568   // What to do depends on the size of the integer and the size of the pointer.
3569   // We can either truncate, zero extend, or no-op, accordingly.
3570   SDValue N = getValue(I.getOperand(0));
3571   auto &TLI = DAG.getTargetLoweringInfo();
3572   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3573   EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3574   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3575   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3576   setValue(&I, N);
3577 }
3578 
3579 void SelectionDAGBuilder::visitBitCast(const User &I) {
3580   SDValue N = getValue(I.getOperand(0));
3581   SDLoc dl = getCurSDLoc();
3582   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3583                                                         I.getType());
3584 
3585   // BitCast assures us that source and destination are the same size so this is
3586   // either a BITCAST or a no-op.
3587   if (DestVT != N.getValueType())
3588     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3589                              DestVT, N)); // convert types.
3590   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3591   // might fold any kind of constant expression to an integer constant and that
3592   // is not what we are looking for. Only recognize a bitcast of a genuine
3593   // constant integer as an opaque constant.
3594   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3595     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3596                                  /*isOpaque*/true));
3597   else
3598     setValue(&I, N);            // noop cast.
3599 }
3600 
3601 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3602   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3603   const Value *SV = I.getOperand(0);
3604   SDValue N = getValue(SV);
3605   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3606 
3607   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3608   unsigned DestAS = I.getType()->getPointerAddressSpace();
3609 
3610   if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3611     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3612 
3613   setValue(&I, N);
3614 }
3615 
3616 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3617   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3618   SDValue InVec = getValue(I.getOperand(0));
3619   SDValue InVal = getValue(I.getOperand(1));
3620   SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3621                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3622   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3623                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3624                            InVec, InVal, InIdx));
3625 }
3626 
3627 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3628   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3629   SDValue InVec = getValue(I.getOperand(0));
3630   SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3631                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3632   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3633                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3634                            InVec, InIdx));
3635 }
3636 
3637 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3638   SDValue Src1 = getValue(I.getOperand(0));
3639   SDValue Src2 = getValue(I.getOperand(1));
3640   ArrayRef<int> Mask;
3641   if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3642     Mask = SVI->getShuffleMask();
3643   else
3644     Mask = cast<ConstantExpr>(I).getShuffleMask();
3645   SDLoc DL = getCurSDLoc();
3646   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3647   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3648   EVT SrcVT = Src1.getValueType();
3649 
3650   if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3651       VT.isScalableVector()) {
3652     // Canonical splat form of first element of first input vector.
3653     SDValue FirstElt =
3654         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
3655                     DAG.getVectorIdxConstant(0, DL));
3656     setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3657     return;
3658   }
3659 
3660   // For now, we only handle splats for scalable vectors.
3661   // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3662   // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3663   assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3664 
3665   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3666   unsigned MaskNumElts = Mask.size();
3667 
3668   if (SrcNumElts == MaskNumElts) {
3669     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3670     return;
3671   }
3672 
3673   // Normalize the shuffle vector since mask and vector length don't match.
3674   if (SrcNumElts < MaskNumElts) {
3675     // Mask is longer than the source vectors. We can use concatenate vector to
3676     // make the mask and vectors lengths match.
3677 
3678     if (MaskNumElts % SrcNumElts == 0) {
3679       // Mask length is a multiple of the source vector length.
3680       // Check if the shuffle is some kind of concatenation of the input
3681       // vectors.
3682       unsigned NumConcat = MaskNumElts / SrcNumElts;
3683       bool IsConcat = true;
3684       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3685       for (unsigned i = 0; i != MaskNumElts; ++i) {
3686         int Idx = Mask[i];
3687         if (Idx < 0)
3688           continue;
3689         // Ensure the indices in each SrcVT sized piece are sequential and that
3690         // the same source is used for the whole piece.
3691         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3692             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3693              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3694           IsConcat = false;
3695           break;
3696         }
3697         // Remember which source this index came from.
3698         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3699       }
3700 
3701       // The shuffle is concatenating multiple vectors together. Just emit
3702       // a CONCAT_VECTORS operation.
3703       if (IsConcat) {
3704         SmallVector<SDValue, 8> ConcatOps;
3705         for (auto Src : ConcatSrcs) {
3706           if (Src < 0)
3707             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3708           else if (Src == 0)
3709             ConcatOps.push_back(Src1);
3710           else
3711             ConcatOps.push_back(Src2);
3712         }
3713         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3714         return;
3715       }
3716     }
3717 
3718     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3719     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3720     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3721                                     PaddedMaskNumElts);
3722 
3723     // Pad both vectors with undefs to make them the same length as the mask.
3724     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3725 
3726     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3727     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3728     MOps1[0] = Src1;
3729     MOps2[0] = Src2;
3730 
3731     Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3732     Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3733 
3734     // Readjust mask for new input vector length.
3735     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3736     for (unsigned i = 0; i != MaskNumElts; ++i) {
3737       int Idx = Mask[i];
3738       if (Idx >= (int)SrcNumElts)
3739         Idx -= SrcNumElts - PaddedMaskNumElts;
3740       MappedOps[i] = Idx;
3741     }
3742 
3743     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3744 
3745     // If the concatenated vector was padded, extract a subvector with the
3746     // correct number of elements.
3747     if (MaskNumElts != PaddedMaskNumElts)
3748       Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3749                            DAG.getVectorIdxConstant(0, DL));
3750 
3751     setValue(&I, Result);
3752     return;
3753   }
3754 
3755   if (SrcNumElts > MaskNumElts) {
3756     // Analyze the access pattern of the vector to see if we can extract
3757     // two subvectors and do the shuffle.
3758     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3759     bool CanExtract = true;
3760     for (int Idx : Mask) {
3761       unsigned Input = 0;
3762       if (Idx < 0)
3763         continue;
3764 
3765       if (Idx >= (int)SrcNumElts) {
3766         Input = 1;
3767         Idx -= SrcNumElts;
3768       }
3769 
3770       // If all the indices come from the same MaskNumElts sized portion of
3771       // the sources we can use extract. Also make sure the extract wouldn't
3772       // extract past the end of the source.
3773       int NewStartIdx = alignDown(Idx, MaskNumElts);
3774       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3775           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3776         CanExtract = false;
3777       // Make sure we always update StartIdx as we use it to track if all
3778       // elements are undef.
3779       StartIdx[Input] = NewStartIdx;
3780     }
3781 
3782     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3783       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3784       return;
3785     }
3786     if (CanExtract) {
3787       // Extract appropriate subvector and generate a vector shuffle
3788       for (unsigned Input = 0; Input < 2; ++Input) {
3789         SDValue &Src = Input == 0 ? Src1 : Src2;
3790         if (StartIdx[Input] < 0)
3791           Src = DAG.getUNDEF(VT);
3792         else {
3793           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3794                             DAG.getVectorIdxConstant(StartIdx[Input], DL));
3795         }
3796       }
3797 
3798       // Calculate new mask.
3799       SmallVector<int, 8> MappedOps(Mask);
3800       for (int &Idx : MappedOps) {
3801         if (Idx >= (int)SrcNumElts)
3802           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3803         else if (Idx >= 0)
3804           Idx -= StartIdx[0];
3805       }
3806 
3807       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3808       return;
3809     }
3810   }
3811 
3812   // We can't use either concat vectors or extract subvectors so fall back to
3813   // replacing the shuffle with extract and build vector.
3814   // to insert and build vector.
3815   EVT EltVT = VT.getVectorElementType();
3816   SmallVector<SDValue,8> Ops;
3817   for (int Idx : Mask) {
3818     SDValue Res;
3819 
3820     if (Idx < 0) {
3821       Res = DAG.getUNDEF(EltVT);
3822     } else {
3823       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3824       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3825 
3826       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3827                         DAG.getVectorIdxConstant(Idx, DL));
3828     }
3829 
3830     Ops.push_back(Res);
3831   }
3832 
3833   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3834 }
3835 
3836 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3837   ArrayRef<unsigned> Indices = I.getIndices();
3838   const Value *Op0 = I.getOperand(0);
3839   const Value *Op1 = I.getOperand(1);
3840   Type *AggTy = I.getType();
3841   Type *ValTy = Op1->getType();
3842   bool IntoUndef = isa<UndefValue>(Op0);
3843   bool FromUndef = isa<UndefValue>(Op1);
3844 
3845   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3846 
3847   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3848   SmallVector<EVT, 4> AggValueVTs;
3849   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3850   SmallVector<EVT, 4> ValValueVTs;
3851   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3852 
3853   unsigned NumAggValues = AggValueVTs.size();
3854   unsigned NumValValues = ValValueVTs.size();
3855   SmallVector<SDValue, 4> Values(NumAggValues);
3856 
3857   // Ignore an insertvalue that produces an empty object
3858   if (!NumAggValues) {
3859     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3860     return;
3861   }
3862 
3863   SDValue Agg = getValue(Op0);
3864   unsigned i = 0;
3865   // Copy the beginning value(s) from the original aggregate.
3866   for (; i != LinearIndex; ++i)
3867     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3868                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3869   // Copy values from the inserted value(s).
3870   if (NumValValues) {
3871     SDValue Val = getValue(Op1);
3872     for (; i != LinearIndex + NumValValues; ++i)
3873       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3874                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3875   }
3876   // Copy remaining value(s) from the original aggregate.
3877   for (; i != NumAggValues; ++i)
3878     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3879                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3880 
3881   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3882                            DAG.getVTList(AggValueVTs), Values));
3883 }
3884 
3885 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3886   ArrayRef<unsigned> Indices = I.getIndices();
3887   const Value *Op0 = I.getOperand(0);
3888   Type *AggTy = Op0->getType();
3889   Type *ValTy = I.getType();
3890   bool OutOfUndef = isa<UndefValue>(Op0);
3891 
3892   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3893 
3894   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3895   SmallVector<EVT, 4> ValValueVTs;
3896   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3897 
3898   unsigned NumValValues = ValValueVTs.size();
3899 
3900   // Ignore a extractvalue that produces an empty object
3901   if (!NumValValues) {
3902     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3903     return;
3904   }
3905 
3906   SmallVector<SDValue, 4> Values(NumValValues);
3907 
3908   SDValue Agg = getValue(Op0);
3909   // Copy out the selected value(s).
3910   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3911     Values[i - LinearIndex] =
3912       OutOfUndef ?
3913         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3914         SDValue(Agg.getNode(), Agg.getResNo() + i);
3915 
3916   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3917                            DAG.getVTList(ValValueVTs), Values));
3918 }
3919 
3920 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3921   Value *Op0 = I.getOperand(0);
3922   // Note that the pointer operand may be a vector of pointers. Take the scalar
3923   // element which holds a pointer.
3924   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3925   SDValue N = getValue(Op0);
3926   SDLoc dl = getCurSDLoc();
3927   auto &TLI = DAG.getTargetLoweringInfo();
3928 
3929   // Normalize Vector GEP - all scalar operands should be converted to the
3930   // splat vector.
3931   bool IsVectorGEP = I.getType()->isVectorTy();
3932   ElementCount VectorElementCount =
3933       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3934                   : ElementCount::getFixed(0);
3935 
3936   if (IsVectorGEP && !N.getValueType().isVector()) {
3937     LLVMContext &Context = *DAG.getContext();
3938     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3939     N = DAG.getSplat(VT, dl, N);
3940   }
3941 
3942   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3943        GTI != E; ++GTI) {
3944     const Value *Idx = GTI.getOperand();
3945     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3946       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3947       if (Field) {
3948         // N = N + Offset
3949         uint64_t Offset =
3950             DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
3951 
3952         // In an inbounds GEP with an offset that is nonnegative even when
3953         // interpreted as signed, assume there is no unsigned overflow.
3954         SDNodeFlags Flags;
3955         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3956           Flags.setNoUnsignedWrap(true);
3957 
3958         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3959                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3960       }
3961     } else {
3962       // IdxSize is the width of the arithmetic according to IR semantics.
3963       // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3964       // (and fix up the result later).
3965       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3966       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3967       TypeSize ElementSize =
3968           DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3969       // We intentionally mask away the high bits here; ElementSize may not
3970       // fit in IdxTy.
3971       APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
3972       bool ElementScalable = ElementSize.isScalable();
3973 
3974       // If this is a scalar constant or a splat vector of constants,
3975       // handle it quickly.
3976       const auto *C = dyn_cast<Constant>(Idx);
3977       if (C && isa<VectorType>(C->getType()))
3978         C = C->getSplatValue();
3979 
3980       const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3981       if (CI && CI->isZero())
3982         continue;
3983       if (CI && !ElementScalable) {
3984         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3985         LLVMContext &Context = *DAG.getContext();
3986         SDValue OffsVal;
3987         if (IsVectorGEP)
3988           OffsVal = DAG.getConstant(
3989               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3990         else
3991           OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3992 
3993         // In an inbounds GEP with an offset that is nonnegative even when
3994         // interpreted as signed, assume there is no unsigned overflow.
3995         SDNodeFlags Flags;
3996         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3997           Flags.setNoUnsignedWrap(true);
3998 
3999         OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4000 
4001         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4002         continue;
4003       }
4004 
4005       // N = N + Idx * ElementMul;
4006       SDValue IdxN = getValue(Idx);
4007 
4008       if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4009         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
4010                                   VectorElementCount);
4011         IdxN = DAG.getSplat(VT, dl, IdxN);
4012       }
4013 
4014       // If the index is smaller or larger than intptr_t, truncate or extend
4015       // it.
4016       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4017 
4018       if (ElementScalable) {
4019         EVT VScaleTy = N.getValueType().getScalarType();
4020         SDValue VScale = DAG.getNode(
4021             ISD::VSCALE, dl, VScaleTy,
4022             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4023         if (IsVectorGEP)
4024           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4025         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4026       } else {
4027         // If this is a multiply by a power of two, turn it into a shl
4028         // immediately.  This is a very common case.
4029         if (ElementMul != 1) {
4030           if (ElementMul.isPowerOf2()) {
4031             unsigned Amt = ElementMul.logBase2();
4032             IdxN = DAG.getNode(ISD::SHL, dl,
4033                                N.getValueType(), IdxN,
4034                                DAG.getConstant(Amt, dl, IdxN.getValueType()));
4035           } else {
4036             SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4037                                             IdxN.getValueType());
4038             IdxN = DAG.getNode(ISD::MUL, dl,
4039                                N.getValueType(), IdxN, Scale);
4040           }
4041         }
4042       }
4043 
4044       N = DAG.getNode(ISD::ADD, dl,
4045                       N.getValueType(), N, IdxN);
4046     }
4047   }
4048 
4049   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4050   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4051   if (IsVectorGEP) {
4052     PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4053     PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4054   }
4055 
4056   if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4057     N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4058 
4059   setValue(&I, N);
4060 }
4061 
4062 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4063   // If this is a fixed sized alloca in the entry block of the function,
4064   // allocate it statically on the stack.
4065   if (FuncInfo.StaticAllocaMap.count(&I))
4066     return;   // getValue will auto-populate this.
4067 
4068   SDLoc dl = getCurSDLoc();
4069   Type *Ty = I.getAllocatedType();
4070   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4071   auto &DL = DAG.getDataLayout();
4072   TypeSize TySize = DL.getTypeAllocSize(Ty);
4073   MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4074 
4075   SDValue AllocSize = getValue(I.getArraySize());
4076 
4077   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());
4078   if (AllocSize.getValueType() != IntPtr)
4079     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4080 
4081   if (TySize.isScalable())
4082     AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4083                             DAG.getVScale(dl, IntPtr,
4084                                           APInt(IntPtr.getScalarSizeInBits(),
4085                                                 TySize.getKnownMinValue())));
4086   else
4087     AllocSize =
4088         DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4089                     DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4090 
4091   // Handle alignment.  If the requested alignment is less than or equal to
4092   // the stack alignment, ignore it.  If the size is greater than or equal to
4093   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4094   Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4095   if (*Alignment <= StackAlign)
4096     Alignment = std::nullopt;
4097 
4098   const uint64_t StackAlignMask = StackAlign.value() - 1U;
4099   // Round the size of the allocation up to the stack alignment size
4100   // by add SA-1 to the size. This doesn't overflow because we're computing
4101   // an address inside an alloca.
4102   SDNodeFlags Flags;
4103   Flags.setNoUnsignedWrap(true);
4104   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4105                           DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4106 
4107   // Mask out the low bits for alignment purposes.
4108   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4109                           DAG.getConstant(~StackAlignMask, dl, IntPtr));
4110 
4111   SDValue Ops[] = {
4112       getRoot(), AllocSize,
4113       DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4114   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4115   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4116   setValue(&I, DSA);
4117   DAG.setRoot(DSA.getValue(1));
4118 
4119   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4120 }
4121 
4122 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4123   if (I.isAtomic())
4124     return visitAtomicLoad(I);
4125 
4126   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4127   const Value *SV = I.getOperand(0);
4128   if (TLI.supportSwiftError()) {
4129     // Swifterror values can come from either a function parameter with
4130     // swifterror attribute or an alloca with swifterror attribute.
4131     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4132       if (Arg->hasSwiftErrorAttr())
4133         return visitLoadFromSwiftError(I);
4134     }
4135 
4136     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4137       if (Alloca->isSwiftError())
4138         return visitLoadFromSwiftError(I);
4139     }
4140   }
4141 
4142   SDValue Ptr = getValue(SV);
4143 
4144   Type *Ty = I.getType();
4145   SmallVector<EVT, 4> ValueVTs, MemVTs;
4146   SmallVector<uint64_t, 4> Offsets;
4147   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4148   unsigned NumValues = ValueVTs.size();
4149   if (NumValues == 0)
4150     return;
4151 
4152   Align Alignment = I.getAlign();
4153   AAMDNodes AAInfo = I.getAAMetadata();
4154   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4155   bool isVolatile = I.isVolatile();
4156   MachineMemOperand::Flags MMOFlags =
4157       TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4158 
4159   SDValue Root;
4160   bool ConstantMemory = false;
4161   if (isVolatile)
4162     // Serialize volatile loads with other side effects.
4163     Root = getRoot();
4164   else if (NumValues > MaxParallelChains)
4165     Root = getMemoryRoot();
4166   else if (AA &&
4167            AA->pointsToConstantMemory(MemoryLocation(
4168                SV,
4169                LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4170                AAInfo))) {
4171     // Do not serialize (non-volatile) loads of constant memory with anything.
4172     Root = DAG.getEntryNode();
4173     ConstantMemory = true;
4174     MMOFlags |= MachineMemOperand::MOInvariant;
4175   } else {
4176     // Do not serialize non-volatile loads against each other.
4177     Root = DAG.getRoot();
4178   }
4179 
4180   SDLoc dl = getCurSDLoc();
4181 
4182   if (isVolatile)
4183     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4184 
4185   // An aggregate load cannot wrap around the address space, so offsets to its
4186   // parts don't wrap either.
4187   SDNodeFlags Flags;
4188   Flags.setNoUnsignedWrap(true);
4189 
4190   SmallVector<SDValue, 4> Values(NumValues);
4191   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4192   EVT PtrVT = Ptr.getValueType();
4193 
4194   unsigned ChainI = 0;
4195   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4196     // Serializing loads here may result in excessive register pressure, and
4197     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4198     // could recover a bit by hoisting nodes upward in the chain by recognizing
4199     // they are side-effect free or do not alias. The optimizer should really
4200     // avoid this case by converting large object/array copies to llvm.memcpy
4201     // (MaxParallelChains should always remain as failsafe).
4202     if (ChainI == MaxParallelChains) {
4203       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4204       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4205                                   ArrayRef(Chains.data(), ChainI));
4206       Root = Chain;
4207       ChainI = 0;
4208     }
4209     SDValue A = DAG.getNode(ISD::ADD, dl,
4210                             PtrVT, Ptr,
4211                             DAG.getConstant(Offsets[i], dl, PtrVT),
4212                             Flags);
4213 
4214     SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4215                             MachinePointerInfo(SV, Offsets[i]), Alignment,
4216                             MMOFlags, AAInfo, Ranges);
4217     Chains[ChainI] = L.getValue(1);
4218 
4219     if (MemVTs[i] != ValueVTs[i])
4220       L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4221 
4222     Values[i] = L;
4223   }
4224 
4225   if (!ConstantMemory) {
4226     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4227                                 ArrayRef(Chains.data(), ChainI));
4228     if (isVolatile)
4229       DAG.setRoot(Chain);
4230     else
4231       PendingLoads.push_back(Chain);
4232   }
4233 
4234   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4235                            DAG.getVTList(ValueVTs), Values));
4236 }
4237 
4238 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4239   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4240          "call visitStoreToSwiftError when backend supports swifterror");
4241 
4242   SmallVector<EVT, 4> ValueVTs;
4243   SmallVector<uint64_t, 4> Offsets;
4244   const Value *SrcV = I.getOperand(0);
4245   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4246                   SrcV->getType(), ValueVTs, &Offsets);
4247   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4248          "expect a single EVT for swifterror");
4249 
4250   SDValue Src = getValue(SrcV);
4251   // Create a virtual register, then update the virtual register.
4252   Register VReg =
4253       SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4254   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4255   // Chain can be getRoot or getControlRoot.
4256   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4257                                       SDValue(Src.getNode(), Src.getResNo()));
4258   DAG.setRoot(CopyNode);
4259 }
4260 
4261 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4262   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4263          "call visitLoadFromSwiftError when backend supports swifterror");
4264 
4265   assert(!I.isVolatile() &&
4266          !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4267          !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4268          "Support volatile, non temporal, invariant for load_from_swift_error");
4269 
4270   const Value *SV = I.getOperand(0);
4271   Type *Ty = I.getType();
4272   assert(
4273       (!AA ||
4274        !AA->pointsToConstantMemory(MemoryLocation(
4275            SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4276            I.getAAMetadata()))) &&
4277       "load_from_swift_error should not be constant memory");
4278 
4279   SmallVector<EVT, 4> ValueVTs;
4280   SmallVector<uint64_t, 4> Offsets;
4281   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4282                   ValueVTs, &Offsets);
4283   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4284          "expect a single EVT for swifterror");
4285 
4286   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4287   SDValue L = DAG.getCopyFromReg(
4288       getRoot(), getCurSDLoc(),
4289       SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4290 
4291   setValue(&I, L);
4292 }
4293 
4294 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4295   if (I.isAtomic())
4296     return visitAtomicStore(I);
4297 
4298   const Value *SrcV = I.getOperand(0);
4299   const Value *PtrV = I.getOperand(1);
4300 
4301   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4302   if (TLI.supportSwiftError()) {
4303     // Swifterror values can come from either a function parameter with
4304     // swifterror attribute or an alloca with swifterror attribute.
4305     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4306       if (Arg->hasSwiftErrorAttr())
4307         return visitStoreToSwiftError(I);
4308     }
4309 
4310     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4311       if (Alloca->isSwiftError())
4312         return visitStoreToSwiftError(I);
4313     }
4314   }
4315 
4316   SmallVector<EVT, 4> ValueVTs, MemVTs;
4317   SmallVector<uint64_t, 4> Offsets;
4318   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4319                   SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4320   unsigned NumValues = ValueVTs.size();
4321   if (NumValues == 0)
4322     return;
4323 
4324   // Get the lowered operands. Note that we do this after
4325   // checking if NumResults is zero, because with zero results
4326   // the operands won't have values in the map.
4327   SDValue Src = getValue(SrcV);
4328   SDValue Ptr = getValue(PtrV);
4329 
4330   SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4331   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4332   SDLoc dl = getCurSDLoc();
4333   Align Alignment = I.getAlign();
4334   AAMDNodes AAInfo = I.getAAMetadata();
4335 
4336   auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4337 
4338   // An aggregate load cannot wrap around the address space, so offsets to its
4339   // parts don't wrap either.
4340   SDNodeFlags Flags;
4341   Flags.setNoUnsignedWrap(true);
4342 
4343   unsigned ChainI = 0;
4344   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4345     // See visitLoad comments.
4346     if (ChainI == MaxParallelChains) {
4347       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4348                                   ArrayRef(Chains.data(), ChainI));
4349       Root = Chain;
4350       ChainI = 0;
4351     }
4352     SDValue Add =
4353         DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4354     SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4355     if (MemVTs[i] != ValueVTs[i])
4356       Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4357     SDValue St =
4358         DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4359                      Alignment, MMOFlags, AAInfo);
4360     Chains[ChainI] = St;
4361   }
4362 
4363   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4364                                   ArrayRef(Chains.data(), ChainI));
4365   setValue(&I, StoreNode);
4366   DAG.setRoot(StoreNode);
4367 }
4368 
4369 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4370                                            bool IsCompressing) {
4371   SDLoc sdl = getCurSDLoc();
4372 
4373   auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4374                                MaybeAlign &Alignment) {
4375     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4376     Src0 = I.getArgOperand(0);
4377     Ptr = I.getArgOperand(1);
4378     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4379     Mask = I.getArgOperand(3);
4380   };
4381   auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4382                                     MaybeAlign &Alignment) {
4383     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4384     Src0 = I.getArgOperand(0);
4385     Ptr = I.getArgOperand(1);
4386     Mask = I.getArgOperand(2);
4387     Alignment = std::nullopt;
4388   };
4389 
4390   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4391   MaybeAlign Alignment;
4392   if (IsCompressing)
4393     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4394   else
4395     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4396 
4397   SDValue Ptr = getValue(PtrOperand);
4398   SDValue Src0 = getValue(Src0Operand);
4399   SDValue Mask = getValue(MaskOperand);
4400   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4401 
4402   EVT VT = Src0.getValueType();
4403   if (!Alignment)
4404     Alignment = DAG.getEVTAlign(VT);
4405 
4406   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4407       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
4408       MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
4409   SDValue StoreNode =
4410       DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4411                          ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4412   DAG.setRoot(StoreNode);
4413   setValue(&I, StoreNode);
4414 }
4415 
4416 // Get a uniform base for the Gather/Scatter intrinsic.
4417 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4418 // We try to represent it as a base pointer + vector of indices.
4419 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4420 // The first operand of the GEP may be a single pointer or a vector of pointers
4421 // Example:
4422 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4423 //  or
4424 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4425 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4426 //
4427 // When the first GEP operand is a single pointer - it is the uniform base we
4428 // are looking for. If first operand of the GEP is a splat vector - we
4429 // extract the splat value and use it as a uniform base.
4430 // In all other cases the function returns 'false'.
4431 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4432                            ISD::MemIndexType &IndexType, SDValue &Scale,
4433                            SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4434                            uint64_t ElemSize) {
4435   SelectionDAG& DAG = SDB->DAG;
4436   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4437   const DataLayout &DL = DAG.getDataLayout();
4438 
4439   assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4440 
4441   // Handle splat constant pointer.
4442   if (auto *C = dyn_cast<Constant>(Ptr)) {
4443     C = C->getSplatValue();
4444     if (!C)
4445       return false;
4446 
4447     Base = SDB->getValue(C);
4448 
4449     ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4450     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4451     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4452     IndexType = ISD::SIGNED_SCALED;
4453     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4454     return true;
4455   }
4456 
4457   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4458   if (!GEP || GEP->getParent() != CurBB)
4459     return false;
4460 
4461   if (GEP->getNumOperands() != 2)
4462     return false;
4463 
4464   const Value *BasePtr = GEP->getPointerOperand();
4465   const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4466 
4467   // Make sure the base is scalar and the index is a vector.
4468   if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4469     return false;
4470 
4471   uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4472 
4473   // Target may not support the required addressing mode.
4474   if (ScaleVal != 1 &&
4475       !TLI.isLegalScaleForGatherScatter(ScaleVal, ElemSize))
4476     return false;
4477 
4478   Base = SDB->getValue(BasePtr);
4479   Index = SDB->getValue(IndexVal);
4480   IndexType = ISD::SIGNED_SCALED;
4481 
4482   Scale =
4483       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4484   return true;
4485 }
4486 
4487 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4488   SDLoc sdl = getCurSDLoc();
4489 
4490   // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4491   const Value *Ptr = I.getArgOperand(1);
4492   SDValue Src0 = getValue(I.getArgOperand(0));
4493   SDValue Mask = getValue(I.getArgOperand(3));
4494   EVT VT = Src0.getValueType();
4495   Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4496                         ->getMaybeAlignValue()
4497                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4498   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4499 
4500   SDValue Base;
4501   SDValue Index;
4502   ISD::MemIndexType IndexType;
4503   SDValue Scale;
4504   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4505                                     I.getParent(), VT.getScalarStoreSize());
4506 
4507   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4508   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4509       MachinePointerInfo(AS), MachineMemOperand::MOStore,
4510       // TODO: Make MachineMemOperands aware of scalable
4511       // vectors.
4512       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4513   if (!UniformBase) {
4514     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4515     Index = getValue(Ptr);
4516     IndexType = ISD::SIGNED_SCALED;
4517     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4518   }
4519 
4520   EVT IdxVT = Index.getValueType();
4521   EVT EltTy = IdxVT.getVectorElementType();
4522   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4523     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4524     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4525   }
4526 
4527   SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4528   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4529                                          Ops, MMO, IndexType, false);
4530   DAG.setRoot(Scatter);
4531   setValue(&I, Scatter);
4532 }
4533 
4534 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4535   SDLoc sdl = getCurSDLoc();
4536 
4537   auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4538                               MaybeAlign &Alignment) {
4539     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4540     Ptr = I.getArgOperand(0);
4541     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4542     Mask = I.getArgOperand(2);
4543     Src0 = I.getArgOperand(3);
4544   };
4545   auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4546                                  MaybeAlign &Alignment) {
4547     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4548     Ptr = I.getArgOperand(0);
4549     Alignment = std::nullopt;
4550     Mask = I.getArgOperand(1);
4551     Src0 = I.getArgOperand(2);
4552   };
4553 
4554   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4555   MaybeAlign Alignment;
4556   if (IsExpanding)
4557     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4558   else
4559     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4560 
4561   SDValue Ptr = getValue(PtrOperand);
4562   SDValue Src0 = getValue(Src0Operand);
4563   SDValue Mask = getValue(MaskOperand);
4564   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4565 
4566   EVT VT = Src0.getValueType();
4567   if (!Alignment)
4568     Alignment = DAG.getEVTAlign(VT);
4569 
4570   AAMDNodes AAInfo = I.getAAMetadata();
4571   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4572 
4573   // Do not serialize masked loads of constant memory with anything.
4574   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4575   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4576 
4577   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4578 
4579   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4580       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
4581       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
4582 
4583   SDValue Load =
4584       DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4585                         ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4586   if (AddToChain)
4587     PendingLoads.push_back(Load.getValue(1));
4588   setValue(&I, Load);
4589 }
4590 
4591 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4592   SDLoc sdl = getCurSDLoc();
4593 
4594   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4595   const Value *Ptr = I.getArgOperand(0);
4596   SDValue Src0 = getValue(I.getArgOperand(3));
4597   SDValue Mask = getValue(I.getArgOperand(2));
4598 
4599   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4600   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4601   Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4602                         ->getMaybeAlignValue()
4603                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4604 
4605   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4606 
4607   SDValue Root = DAG.getRoot();
4608   SDValue Base;
4609   SDValue Index;
4610   ISD::MemIndexType IndexType;
4611   SDValue Scale;
4612   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4613                                     I.getParent(), VT.getScalarStoreSize());
4614   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4615   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4616       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
4617       // TODO: Make MachineMemOperands aware of scalable
4618       // vectors.
4619       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4620 
4621   if (!UniformBase) {
4622     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4623     Index = getValue(Ptr);
4624     IndexType = ISD::SIGNED_SCALED;
4625     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4626   }
4627 
4628   EVT IdxVT = Index.getValueType();
4629   EVT EltTy = IdxVT.getVectorElementType();
4630   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4631     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4632     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4633   }
4634 
4635   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4636   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4637                                        Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4638 
4639   PendingLoads.push_back(Gather.getValue(1));
4640   setValue(&I, Gather);
4641 }
4642 
4643 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4644   SDLoc dl = getCurSDLoc();
4645   AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4646   AtomicOrdering FailureOrdering = I.getFailureOrdering();
4647   SyncScope::ID SSID = I.getSyncScopeID();
4648 
4649   SDValue InChain = getRoot();
4650 
4651   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4652   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4653 
4654   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4655   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4656 
4657   MachineFunction &MF = DAG.getMachineFunction();
4658   MachineMemOperand *MMO = MF.getMachineMemOperand(
4659       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4660       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
4661       FailureOrdering);
4662 
4663   SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4664                                    dl, MemVT, VTs, InChain,
4665                                    getValue(I.getPointerOperand()),
4666                                    getValue(I.getCompareOperand()),
4667                                    getValue(I.getNewValOperand()), MMO);
4668 
4669   SDValue OutChain = L.getValue(2);
4670 
4671   setValue(&I, L);
4672   DAG.setRoot(OutChain);
4673 }
4674 
4675 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4676   SDLoc dl = getCurSDLoc();
4677   ISD::NodeType NT;
4678   switch (I.getOperation()) {
4679   default: llvm_unreachable("Unknown atomicrmw operation");
4680   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4681   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4682   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4683   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4684   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4685   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4686   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4687   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4688   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4689   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4690   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4691   case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4692   case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4693   case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
4694   case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
4695   case AtomicRMWInst::UIncWrap:
4696     NT = ISD::ATOMIC_LOAD_UINC_WRAP;
4697     break;
4698   case AtomicRMWInst::UDecWrap:
4699     NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
4700     break;
4701   }
4702   AtomicOrdering Ordering = I.getOrdering();
4703   SyncScope::ID SSID = I.getSyncScopeID();
4704 
4705   SDValue InChain = getRoot();
4706 
4707   auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4708   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4709   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4710 
4711   MachineFunction &MF = DAG.getMachineFunction();
4712   MachineMemOperand *MMO = MF.getMachineMemOperand(
4713       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4714       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
4715 
4716   SDValue L =
4717     DAG.getAtomic(NT, dl, MemVT, InChain,
4718                   getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4719                   MMO);
4720 
4721   SDValue OutChain = L.getValue(1);
4722 
4723   setValue(&I, L);
4724   DAG.setRoot(OutChain);
4725 }
4726 
4727 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4728   SDLoc dl = getCurSDLoc();
4729   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4730   SDValue Ops[3];
4731   Ops[0] = getRoot();
4732   Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
4733                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4734   Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
4735                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4736   SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
4737   setValue(&I, N);
4738   DAG.setRoot(N);
4739 }
4740 
4741 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4742   SDLoc dl = getCurSDLoc();
4743   AtomicOrdering Order = I.getOrdering();
4744   SyncScope::ID SSID = I.getSyncScopeID();
4745 
4746   SDValue InChain = getRoot();
4747 
4748   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4749   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4750   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4751 
4752   if (!TLI.supportsUnalignedAtomics() &&
4753       I.getAlign().value() < MemVT.getSizeInBits() / 8)
4754     report_fatal_error("Cannot generate unaligned atomic load");
4755 
4756   auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4757 
4758   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4759       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4760       I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
4761 
4762   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4763 
4764   SDValue Ptr = getValue(I.getPointerOperand());
4765 
4766   if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
4767     // TODO: Once this is better exercised by tests, it should be merged with
4768     // the normal path for loads to prevent future divergence.
4769     SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4770     if (MemVT != VT)
4771       L = DAG.getPtrExtOrTrunc(L, dl, VT);
4772 
4773     setValue(&I, L);
4774     SDValue OutChain = L.getValue(1);
4775     if (!I.isUnordered())
4776       DAG.setRoot(OutChain);
4777     else
4778       PendingLoads.push_back(OutChain);
4779     return;
4780   }
4781 
4782   SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4783                             Ptr, MMO);
4784 
4785   SDValue OutChain = L.getValue(1);
4786   if (MemVT != VT)
4787     L = DAG.getPtrExtOrTrunc(L, dl, VT);
4788 
4789   setValue(&I, L);
4790   DAG.setRoot(OutChain);
4791 }
4792 
4793 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4794   SDLoc dl = getCurSDLoc();
4795 
4796   AtomicOrdering Ordering = I.getOrdering();
4797   SyncScope::ID SSID = I.getSyncScopeID();
4798 
4799   SDValue InChain = getRoot();
4800 
4801   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4802   EVT MemVT =
4803       TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4804 
4805   if (!TLI.supportsUnalignedAtomics() &&
4806       I.getAlign().value() < MemVT.getSizeInBits() / 8)
4807     report_fatal_error("Cannot generate unaligned atomic store");
4808 
4809   auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4810 
4811   MachineFunction &MF = DAG.getMachineFunction();
4812   MachineMemOperand *MMO = MF.getMachineMemOperand(
4813       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4814       I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
4815 
4816   SDValue Val = getValue(I.getValueOperand());
4817   if (Val.getValueType() != MemVT)
4818     Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4819   SDValue Ptr = getValue(I.getPointerOperand());
4820 
4821   if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
4822     // TODO: Once this is better exercised by tests, it should be merged with
4823     // the normal path for stores to prevent future divergence.
4824     SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4825     setValue(&I, S);
4826     DAG.setRoot(S);
4827     return;
4828   }
4829   SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4830                                    Ptr, Val, MMO);
4831 
4832   setValue(&I, OutChain);
4833   DAG.setRoot(OutChain);
4834 }
4835 
4836 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4837 /// node.
4838 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4839                                                unsigned Intrinsic) {
4840   // Ignore the callsite's attributes. A specific call site may be marked with
4841   // readnone, but the lowering code will expect the chain based on the
4842   // definition.
4843   const Function *F = I.getCalledFunction();
4844   bool HasChain = !F->doesNotAccessMemory();
4845   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4846 
4847   // Build the operand list.
4848   SmallVector<SDValue, 8> Ops;
4849   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4850     if (OnlyLoad) {
4851       // We don't need to serialize loads against other loads.
4852       Ops.push_back(DAG.getRoot());
4853     } else {
4854       Ops.push_back(getRoot());
4855     }
4856   }
4857 
4858   // Info is set by getTgtMemIntrinsic
4859   TargetLowering::IntrinsicInfo Info;
4860   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4861   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4862                                                DAG.getMachineFunction(),
4863                                                Intrinsic);
4864 
4865   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4866   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4867       Info.opc == ISD::INTRINSIC_W_CHAIN)
4868     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4869                                         TLI.getPointerTy(DAG.getDataLayout())));
4870 
4871   // Add all operands of the call to the operand list.
4872   for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
4873     const Value *Arg = I.getArgOperand(i);
4874     if (!I.paramHasAttr(i, Attribute::ImmArg)) {
4875       Ops.push_back(getValue(Arg));
4876       continue;
4877     }
4878 
4879     // Use TargetConstant instead of a regular constant for immarg.
4880     EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
4881     if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4882       assert(CI->getBitWidth() <= 64 &&
4883              "large intrinsic immediates not handled");
4884       Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
4885     } else {
4886       Ops.push_back(
4887           DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
4888     }
4889   }
4890 
4891   SmallVector<EVT, 4> ValueVTs;
4892   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4893 
4894   if (HasChain)
4895     ValueVTs.push_back(MVT::Other);
4896 
4897   SDVTList VTs = DAG.getVTList(ValueVTs);
4898 
4899   // Propagate fast-math-flags from IR to node(s).
4900   SDNodeFlags Flags;
4901   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
4902     Flags.copyFMF(*FPMO);
4903   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
4904 
4905   // Create the node.
4906   SDValue Result;
4907   // In some cases, custom collection of operands from CallInst I may be needed.
4908   TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
4909   if (IsTgtIntrinsic) {
4910     // This is target intrinsic that touches memory
4911     //
4912     // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
4913     //       didn't yield anything useful.
4914     MachinePointerInfo MPI;
4915     if (Info.ptrVal)
4916       MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
4917     else if (Info.fallbackAddressSpace)
4918       MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
4919     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
4920                                      Info.memVT, MPI, Info.align, Info.flags,
4921                                      Info.size, I.getAAMetadata());
4922   } else if (!HasChain) {
4923     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4924   } else if (!I.getType()->isVoidTy()) {
4925     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4926   } else {
4927     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4928   }
4929 
4930   if (HasChain) {
4931     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4932     if (OnlyLoad)
4933       PendingLoads.push_back(Chain);
4934     else
4935       DAG.setRoot(Chain);
4936   }
4937 
4938   if (!I.getType()->isVoidTy()) {
4939     if (!isa<VectorType>(I.getType()))
4940       Result = lowerRangeToAssertZExt(DAG, I, Result);
4941 
4942     MaybeAlign Alignment = I.getRetAlign();
4943 
4944     // Insert `assertalign` node if there's an alignment.
4945     if (InsertAssertAlign && Alignment) {
4946       Result =
4947           DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
4948     }
4949 
4950     setValue(&I, Result);
4951   }
4952 }
4953 
4954 /// GetSignificand - Get the significand and build it into a floating-point
4955 /// number with exponent of 1:
4956 ///
4957 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4958 ///
4959 /// where Op is the hexadecimal representation of floating point value.
4960 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4961   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4962                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4963   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4964                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4965   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4966 }
4967 
4968 /// GetExponent - Get the exponent:
4969 ///
4970 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4971 ///
4972 /// where Op is the hexadecimal representation of floating point value.
4973 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4974                            const TargetLowering &TLI, const SDLoc &dl) {
4975   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4976                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4977   SDValue t1 = DAG.getNode(
4978       ISD::SRL, dl, MVT::i32, t0,
4979       DAG.getConstant(23, dl,
4980                       TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
4981   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4982                            DAG.getConstant(127, dl, MVT::i32));
4983   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4984 }
4985 
4986 /// getF32Constant - Get 32-bit floating point constant.
4987 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4988                               const SDLoc &dl) {
4989   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4990                            MVT::f32);
4991 }
4992 
4993 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4994                                        SelectionDAG &DAG) {
4995   // TODO: What fast-math-flags should be set on the floating-point nodes?
4996 
4997   //   IntegerPartOfX = ((int32_t)(t0);
4998   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4999 
5000   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
5001   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5002   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5003 
5004   //   IntegerPartOfX <<= 23;
5005   IntegerPartOfX =
5006       DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5007                   DAG.getConstant(23, dl,
5008                                   DAG.getTargetLoweringInfo().getShiftAmountTy(
5009                                       MVT::i32, DAG.getDataLayout())));
5010 
5011   SDValue TwoToFractionalPartOfX;
5012   if (LimitFloatPrecision <= 6) {
5013     // For floating-point precision of 6:
5014     //
5015     //   TwoToFractionalPartOfX =
5016     //     0.997535578f +
5017     //       (0.735607626f + 0.252464424f * x) * x;
5018     //
5019     // error 0.0144103317, which is 6 bits
5020     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5021                              getF32Constant(DAG, 0x3e814304, dl));
5022     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5023                              getF32Constant(DAG, 0x3f3c50c8, dl));
5024     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5025     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5026                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
5027   } else if (LimitFloatPrecision <= 12) {
5028     // For floating-point precision of 12:
5029     //
5030     //   TwoToFractionalPartOfX =
5031     //     0.999892986f +
5032     //       (0.696457318f +
5033     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
5034     //
5035     // error 0.000107046256, which is 13 to 14 bits
5036     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5037                              getF32Constant(DAG, 0x3da235e3, dl));
5038     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5039                              getF32Constant(DAG, 0x3e65b8f3, dl));
5040     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5041     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5042                              getF32Constant(DAG, 0x3f324b07, dl));
5043     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5044     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5045                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
5046   } else { // LimitFloatPrecision <= 18
5047     // For floating-point precision of 18:
5048     //
5049     //   TwoToFractionalPartOfX =
5050     //     0.999999982f +
5051     //       (0.693148872f +
5052     //         (0.240227044f +
5053     //           (0.554906021e-1f +
5054     //             (0.961591928e-2f +
5055     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5056     // error 2.47208000*10^(-7), which is better than 18 bits
5057     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5058                              getF32Constant(DAG, 0x3924b03e, dl));
5059     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5060                              getF32Constant(DAG, 0x3ab24b87, dl));
5061     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5062     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5063                              getF32Constant(DAG, 0x3c1d8c17, dl));
5064     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5065     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5066                              getF32Constant(DAG, 0x3d634a1d, dl));
5067     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5068     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5069                              getF32Constant(DAG, 0x3e75fe14, dl));
5070     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5071     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5072                               getF32Constant(DAG, 0x3f317234, dl));
5073     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5074     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5075                                          getF32Constant(DAG, 0x3f800000, dl));
5076   }
5077 
5078   // Add the exponent into the result in integer domain.
5079   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5080   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5081                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5082 }
5083 
5084 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
5085 /// limited-precision mode.
5086 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5087                          const TargetLowering &TLI, SDNodeFlags Flags) {
5088   if (Op.getValueType() == MVT::f32 &&
5089       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5090 
5091     // Put the exponent in the right bit position for later addition to the
5092     // final result:
5093     //
5094     // t0 = Op * log2(e)
5095 
5096     // TODO: What fast-math-flags should be set here?
5097     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5098                              DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5099     return getLimitedPrecisionExp2(t0, dl, DAG);
5100   }
5101 
5102   // No special expansion.
5103   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5104 }
5105 
5106 /// expandLog - Lower a log intrinsic. Handles the special sequences for
5107 /// limited-precision mode.
5108 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5109                          const TargetLowering &TLI, SDNodeFlags Flags) {
5110   // TODO: What fast-math-flags should be set on the floating-point nodes?
5111 
5112   if (Op.getValueType() == MVT::f32 &&
5113       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5114     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5115 
5116     // Scale the exponent by log(2).
5117     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5118     SDValue LogOfExponent =
5119         DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5120                     DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5121 
5122     // Get the significand and build it into a floating-point number with
5123     // exponent of 1.
5124     SDValue X = GetSignificand(DAG, Op1, dl);
5125 
5126     SDValue LogOfMantissa;
5127     if (LimitFloatPrecision <= 6) {
5128       // For floating-point precision of 6:
5129       //
5130       //   LogofMantissa =
5131       //     -1.1609546f +
5132       //       (1.4034025f - 0.23903021f * x) * x;
5133       //
5134       // error 0.0034276066, which is better than 8 bits
5135       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5136                                getF32Constant(DAG, 0xbe74c456, dl));
5137       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5138                                getF32Constant(DAG, 0x3fb3a2b1, dl));
5139       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5140       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5141                                   getF32Constant(DAG, 0x3f949a29, dl));
5142     } else if (LimitFloatPrecision <= 12) {
5143       // For floating-point precision of 12:
5144       //
5145       //   LogOfMantissa =
5146       //     -1.7417939f +
5147       //       (2.8212026f +
5148       //         (-1.4699568f +
5149       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5150       //
5151       // error 0.000061011436, which is 14 bits
5152       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5153                                getF32Constant(DAG, 0xbd67b6d6, dl));
5154       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5155                                getF32Constant(DAG, 0x3ee4f4b8, dl));
5156       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5157       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5158                                getF32Constant(DAG, 0x3fbc278b, dl));
5159       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5160       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5161                                getF32Constant(DAG, 0x40348e95, dl));
5162       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5163       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5164                                   getF32Constant(DAG, 0x3fdef31a, dl));
5165     } else { // LimitFloatPrecision <= 18
5166       // For floating-point precision of 18:
5167       //
5168       //   LogOfMantissa =
5169       //     -2.1072184f +
5170       //       (4.2372794f +
5171       //         (-3.7029485f +
5172       //           (2.2781945f +
5173       //             (-0.87823314f +
5174       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5175       //
5176       // error 0.0000023660568, which is better than 18 bits
5177       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5178                                getF32Constant(DAG, 0xbc91e5ac, dl));
5179       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5180                                getF32Constant(DAG, 0x3e4350aa, dl));
5181       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5182       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5183                                getF32Constant(DAG, 0x3f60d3e3, dl));
5184       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5185       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5186                                getF32Constant(DAG, 0x4011cdf0, dl));
5187       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5188       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5189                                getF32Constant(DAG, 0x406cfd1c, dl));
5190       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5191       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5192                                getF32Constant(DAG, 0x408797cb, dl));
5193       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5194       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5195                                   getF32Constant(DAG, 0x4006dcab, dl));
5196     }
5197 
5198     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5199   }
5200 
5201   // No special expansion.
5202   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5203 }
5204 
5205 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5206 /// limited-precision mode.
5207 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5208                           const TargetLowering &TLI, SDNodeFlags Flags) {
5209   // TODO: What fast-math-flags should be set on the floating-point nodes?
5210 
5211   if (Op.getValueType() == MVT::f32 &&
5212       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5213     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5214 
5215     // Get the exponent.
5216     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5217 
5218     // Get the significand and build it into a floating-point number with
5219     // exponent of 1.
5220     SDValue X = GetSignificand(DAG, Op1, dl);
5221 
5222     // Different possible minimax approximations of significand in
5223     // floating-point for various degrees of accuracy over [1,2].
5224     SDValue Log2ofMantissa;
5225     if (LimitFloatPrecision <= 6) {
5226       // For floating-point precision of 6:
5227       //
5228       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5229       //
5230       // error 0.0049451742, which is more than 7 bits
5231       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5232                                getF32Constant(DAG, 0xbeb08fe0, dl));
5233       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5234                                getF32Constant(DAG, 0x40019463, dl));
5235       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5236       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5237                                    getF32Constant(DAG, 0x3fd6633d, dl));
5238     } else if (LimitFloatPrecision <= 12) {
5239       // For floating-point precision of 12:
5240       //
5241       //   Log2ofMantissa =
5242       //     -2.51285454f +
5243       //       (4.07009056f +
5244       //         (-2.12067489f +
5245       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5246       //
5247       // error 0.0000876136000, which is better than 13 bits
5248       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5249                                getF32Constant(DAG, 0xbda7262e, dl));
5250       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5251                                getF32Constant(DAG, 0x3f25280b, dl));
5252       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5253       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5254                                getF32Constant(DAG, 0x4007b923, dl));
5255       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5256       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5257                                getF32Constant(DAG, 0x40823e2f, dl));
5258       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5259       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5260                                    getF32Constant(DAG, 0x4020d29c, dl));
5261     } else { // LimitFloatPrecision <= 18
5262       // For floating-point precision of 18:
5263       //
5264       //   Log2ofMantissa =
5265       //     -3.0400495f +
5266       //       (6.1129976f +
5267       //         (-5.3420409f +
5268       //           (3.2865683f +
5269       //             (-1.2669343f +
5270       //               (0.27515199f -
5271       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5272       //
5273       // error 0.0000018516, which is better than 18 bits
5274       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5275                                getF32Constant(DAG, 0xbcd2769e, dl));
5276       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5277                                getF32Constant(DAG, 0x3e8ce0b9, dl));
5278       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5279       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5280                                getF32Constant(DAG, 0x3fa22ae7, dl));
5281       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5282       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5283                                getF32Constant(DAG, 0x40525723, dl));
5284       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5285       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5286                                getF32Constant(DAG, 0x40aaf200, dl));
5287       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5288       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5289                                getF32Constant(DAG, 0x40c39dad, dl));
5290       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5291       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5292                                    getF32Constant(DAG, 0x4042902c, dl));
5293     }
5294 
5295     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5296   }
5297 
5298   // No special expansion.
5299   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5300 }
5301 
5302 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5303 /// limited-precision mode.
5304 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5305                            const TargetLowering &TLI, SDNodeFlags Flags) {
5306   // TODO: What fast-math-flags should be set on the floating-point nodes?
5307 
5308   if (Op.getValueType() == MVT::f32 &&
5309       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5310     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5311 
5312     // Scale the exponent by log10(2) [0.30102999f].
5313     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5314     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5315                                         getF32Constant(DAG, 0x3e9a209a, dl));
5316 
5317     // Get the significand and build it into a floating-point number with
5318     // exponent of 1.
5319     SDValue X = GetSignificand(DAG, Op1, dl);
5320 
5321     SDValue Log10ofMantissa;
5322     if (LimitFloatPrecision <= 6) {
5323       // For floating-point precision of 6:
5324       //
5325       //   Log10ofMantissa =
5326       //     -0.50419619f +
5327       //       (0.60948995f - 0.10380950f * x) * x;
5328       //
5329       // error 0.0014886165, which is 6 bits
5330       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5331                                getF32Constant(DAG, 0xbdd49a13, dl));
5332       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5333                                getF32Constant(DAG, 0x3f1c0789, dl));
5334       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5335       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5336                                     getF32Constant(DAG, 0x3f011300, dl));
5337     } else if (LimitFloatPrecision <= 12) {
5338       // For floating-point precision of 12:
5339       //
5340       //   Log10ofMantissa =
5341       //     -0.64831180f +
5342       //       (0.91751397f +
5343       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5344       //
5345       // error 0.00019228036, which is better than 12 bits
5346       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5347                                getF32Constant(DAG, 0x3d431f31, dl));
5348       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5349                                getF32Constant(DAG, 0x3ea21fb2, dl));
5350       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5351       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5352                                getF32Constant(DAG, 0x3f6ae232, dl));
5353       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5354       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5355                                     getF32Constant(DAG, 0x3f25f7c3, dl));
5356     } else { // LimitFloatPrecision <= 18
5357       // For floating-point precision of 18:
5358       //
5359       //   Log10ofMantissa =
5360       //     -0.84299375f +
5361       //       (1.5327582f +
5362       //         (-1.0688956f +
5363       //           (0.49102474f +
5364       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5365       //
5366       // error 0.0000037995730, which is better than 18 bits
5367       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5368                                getF32Constant(DAG, 0x3c5d51ce, dl));
5369       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5370                                getF32Constant(DAG, 0x3e00685a, dl));
5371       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5372       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5373                                getF32Constant(DAG, 0x3efb6798, dl));
5374       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5375       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5376                                getF32Constant(DAG, 0x3f88d192, dl));
5377       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5378       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5379                                getF32Constant(DAG, 0x3fc4316c, dl));
5380       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5381       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5382                                     getF32Constant(DAG, 0x3f57ce70, dl));
5383     }
5384 
5385     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5386   }
5387 
5388   // No special expansion.
5389   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5390 }
5391 
5392 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5393 /// limited-precision mode.
5394 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5395                           const TargetLowering &TLI, SDNodeFlags Flags) {
5396   if (Op.getValueType() == MVT::f32 &&
5397       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5398     return getLimitedPrecisionExp2(Op, dl, DAG);
5399 
5400   // No special expansion.
5401   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5402 }
5403 
5404 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
5405 /// limited-precision mode with x == 10.0f.
5406 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5407                          SelectionDAG &DAG, const TargetLowering &TLI,
5408                          SDNodeFlags Flags) {
5409   bool IsExp10 = false;
5410   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5411       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5412     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5413       APFloat Ten(10.0f);
5414       IsExp10 = LHSC->isExactlyValue(Ten);
5415     }
5416   }
5417 
5418   // TODO: What fast-math-flags should be set on the FMUL node?
5419   if (IsExp10) {
5420     // Put the exponent in the right bit position for later addition to the
5421     // final result:
5422     //
5423     //   #define LOG2OF10 3.3219281f
5424     //   t0 = Op * LOG2OF10;
5425     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5426                              getF32Constant(DAG, 0x40549a78, dl));
5427     return getLimitedPrecisionExp2(t0, dl, DAG);
5428   }
5429 
5430   // No special expansion.
5431   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5432 }
5433 
5434 /// ExpandPowI - Expand a llvm.powi intrinsic.
5435 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5436                           SelectionDAG &DAG) {
5437   // If RHS is a constant, we can expand this out to a multiplication tree if
5438   // it's beneficial on the target, otherwise we end up lowering to a call to
5439   // __powidf2 (for example).
5440   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5441     unsigned Val = RHSC->getSExtValue();
5442 
5443     // powi(x, 0) -> 1.0
5444     if (Val == 0)
5445       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5446 
5447     if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
5448             Val, DAG.shouldOptForSize())) {
5449       // Get the exponent as a positive value.
5450       if ((int)Val < 0)
5451         Val = -Val;
5452       // We use the simple binary decomposition method to generate the multiply
5453       // sequence.  There are more optimal ways to do this (for example,
5454       // powi(x,15) generates one more multiply than it should), but this has
5455       // the benefit of being both really simple and much better than a libcall.
5456       SDValue Res; // Logically starts equal to 1.0
5457       SDValue CurSquare = LHS;
5458       // TODO: Intrinsics should have fast-math-flags that propagate to these
5459       // nodes.
5460       while (Val) {
5461         if (Val & 1) {
5462           if (Res.getNode())
5463             Res =
5464                 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5465           else
5466             Res = CurSquare; // 1.0*CurSquare.
5467         }
5468 
5469         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5470                                 CurSquare, CurSquare);
5471         Val >>= 1;
5472       }
5473 
5474       // If the original was negative, invert the result, producing 1/(x*x*x).
5475       if (RHSC->getSExtValue() < 0)
5476         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5477                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5478       return Res;
5479     }
5480   }
5481 
5482   // Otherwise, expand to a libcall.
5483   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5484 }
5485 
5486 static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5487                             SDValue LHS, SDValue RHS, SDValue Scale,
5488                             SelectionDAG &DAG, const TargetLowering &TLI) {
5489   EVT VT = LHS.getValueType();
5490   bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5491   bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5492   LLVMContext &Ctx = *DAG.getContext();
5493 
5494   // If the type is legal but the operation isn't, this node might survive all
5495   // the way to operation legalization. If we end up there and we do not have
5496   // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5497   // node.
5498 
5499   // Coax the legalizer into expanding the node during type legalization instead
5500   // by bumping the size by one bit. This will force it to Promote, enabling the
5501   // early expansion and avoiding the need to expand later.
5502 
5503   // We don't have to do this if Scale is 0; that can always be expanded, unless
5504   // it's a saturating signed operation. Those can experience true integer
5505   // division overflow, a case which we must avoid.
5506 
5507   // FIXME: We wouldn't have to do this (or any of the early
5508   // expansion/promotion) if it was possible to expand a libcall of an
5509   // illegal type during operation legalization. But it's not, so things
5510   // get a bit hacky.
5511   unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
5512   if ((ScaleInt > 0 || (Saturating && Signed)) &&
5513       (TLI.isTypeLegal(VT) ||
5514        (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5515     TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5516         Opcode, VT, ScaleInt);
5517     if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5518       EVT PromVT;
5519       if (VT.isScalarInteger())
5520         PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5521       else if (VT.isVector()) {
5522         PromVT = VT.getVectorElementType();
5523         PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5524         PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5525       } else
5526         llvm_unreachable("Wrong VT for DIVFIX?");
5527       if (Signed) {
5528         LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
5529         RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
5530       } else {
5531         LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
5532         RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
5533       }
5534       EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5535       // For saturating operations, we need to shift up the LHS to get the
5536       // proper saturation width, and then shift down again afterwards.
5537       if (Saturating)
5538         LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5539                           DAG.getConstant(1, DL, ShiftTy));
5540       SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5541       if (Saturating)
5542         Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5543                           DAG.getConstant(1, DL, ShiftTy));
5544       return DAG.getZExtOrTrunc(Res, DL, VT);
5545     }
5546   }
5547 
5548   return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5549 }
5550 
5551 // getUnderlyingArgRegs - Find underlying registers used for a truncated,
5552 // bitcasted, or split argument. Returns a list of <Register, size in bits>
5553 static void
5554 getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5555                      const SDValue &N) {
5556   switch (N.getOpcode()) {
5557   case ISD::CopyFromReg: {
5558     SDValue Op = N.getOperand(1);
5559     Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5560                       Op.getValueType().getSizeInBits());
5561     return;
5562   }
5563   case ISD::BITCAST:
5564   case ISD::AssertZext:
5565   case ISD::AssertSext:
5566   case ISD::TRUNCATE:
5567     getUnderlyingArgRegs(Regs, N.getOperand(0));
5568     return;
5569   case ISD::BUILD_PAIR:
5570   case ISD::BUILD_VECTOR:
5571   case ISD::CONCAT_VECTORS:
5572     for (SDValue Op : N->op_values())
5573       getUnderlyingArgRegs(Regs, Op);
5574     return;
5575   default:
5576     return;
5577   }
5578 }
5579 
5580 /// If the DbgValueInst is a dbg_value of a function argument, create the
5581 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
5582 /// instruction selection, they will be inserted to the entry BB.
5583 /// We don't currently support this for variadic dbg_values, as they shouldn't
5584 /// appear for function arguments or in the prologue.
5585 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5586     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5587     DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5588   const Argument *Arg = dyn_cast<Argument>(V);
5589   if (!Arg)
5590     return false;
5591 
5592   MachineFunction &MF = DAG.getMachineFunction();
5593   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5594 
5595   // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5596   // we've been asked to pursue.
5597   auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5598                               bool Indirect) {
5599     if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5600       // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5601       // pointing at the VReg, which will be patched up later.
5602       auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5603       SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
5604           /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5605           /* isKill */ false, /* isDead */ false,
5606           /* isUndef */ false, /* isEarlyClobber */ false,
5607           /* SubReg */ 0, /* isDebug */ true)});
5608 
5609       auto *NewDIExpr = FragExpr;
5610       // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5611       // the DIExpression.
5612       if (Indirect)
5613         NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5614       SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
5615       NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
5616       return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
5617     } else {
5618       // Create a completely standard DBG_VALUE.
5619       auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5620       return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5621     }
5622   };
5623 
5624   if (Kind == FuncArgumentDbgValueKind::Value) {
5625     // ArgDbgValues are hoisted to the beginning of the entry block. So we
5626     // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5627     // the entry block.
5628     bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5629     if (!IsInEntryBlock)
5630       return false;
5631 
5632     // ArgDbgValues are hoisted to the beginning of the entry block.  So we
5633     // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5634     // variable that also is a param.
5635     //
5636     // Although, if we are at the top of the entry block already, we can still
5637     // emit using ArgDbgValue. This might catch some situations when the
5638     // dbg.value refers to an argument that isn't used in the entry block, so
5639     // any CopyToReg node would be optimized out and the only way to express
5640     // this DBG_VALUE is by using the physical reg (or FI) as done in this
5641     // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
5642     // we should only emit as ArgDbgValue if the Variable is an argument to the
5643     // current function, and the dbg.value intrinsic is found in the entry
5644     // block.
5645     bool VariableIsFunctionInputArg = Variable->isParameter() &&
5646         !DL->getInlinedAt();
5647     bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5648     if (!IsInPrologue && !VariableIsFunctionInputArg)
5649       return false;
5650 
5651     // Here we assume that a function argument on IR level only can be used to
5652     // describe one input parameter on source level. If we for example have
5653     // source code like this
5654     //
5655     //    struct A { long x, y; };
5656     //    void foo(struct A a, long b) {
5657     //      ...
5658     //      b = a.x;
5659     //      ...
5660     //    }
5661     //
5662     // and IR like this
5663     //
5664     //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
5665     //  entry:
5666     //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5667     //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5668     //    call void @llvm.dbg.value(metadata i32 %b, "b",
5669     //    ...
5670     //    call void @llvm.dbg.value(metadata i32 %a1, "b"
5671     //    ...
5672     //
5673     // then the last dbg.value is describing a parameter "b" using a value that
5674     // is an argument. But since we already has used %a1 to describe a parameter
5675     // we should not handle that last dbg.value here (that would result in an
5676     // incorrect hoisting of the DBG_VALUE to the function entry).
5677     // Notice that we allow one dbg.value per IR level argument, to accommodate
5678     // for the situation with fragments above.
5679     if (VariableIsFunctionInputArg) {
5680       unsigned ArgNo = Arg->getArgNo();
5681       if (ArgNo >= FuncInfo.DescribedArgs.size())
5682         FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
5683       else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
5684         return false;
5685       FuncInfo.DescribedArgs.set(ArgNo);
5686     }
5687   }
5688 
5689   bool IsIndirect = false;
5690   std::optional<MachineOperand> Op;
5691   // Some arguments' frame index is recorded during argument lowering.
5692   int FI = FuncInfo.getArgumentFrameIndex(Arg);
5693   if (FI != std::numeric_limits<int>::max())
5694     Op = MachineOperand::CreateFI(FI);
5695 
5696   SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
5697   if (!Op && N.getNode()) {
5698     getUnderlyingArgRegs(ArgRegsAndSizes, N);
5699     Register Reg;
5700     if (ArgRegsAndSizes.size() == 1)
5701       Reg = ArgRegsAndSizes.front().first;
5702 
5703     if (Reg && Reg.isVirtual()) {
5704       MachineRegisterInfo &RegInfo = MF.getRegInfo();
5705       Register PR = RegInfo.getLiveInPhysReg(Reg);
5706       if (PR)
5707         Reg = PR;
5708     }
5709     if (Reg) {
5710       Op = MachineOperand::CreateReg(Reg, false);
5711       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5712     }
5713   }
5714 
5715   if (!Op && N.getNode()) {
5716     // Check if frame index is available.
5717     SDValue LCandidate = peekThroughBitcasts(N);
5718     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
5719       if (FrameIndexSDNode *FINode =
5720           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
5721         Op = MachineOperand::CreateFI(FINode->getIndex());
5722   }
5723 
5724   if (!Op) {
5725     // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
5726     auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
5727                                          SplitRegs) {
5728       unsigned Offset = 0;
5729       for (const auto &RegAndSize : SplitRegs) {
5730         // If the expression is already a fragment, the current register
5731         // offset+size might extend beyond the fragment. In this case, only
5732         // the register bits that are inside the fragment are relevant.
5733         int RegFragmentSizeInBits = RegAndSize.second;
5734         if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
5735           uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
5736           // The register is entirely outside the expression fragment,
5737           // so is irrelevant for debug info.
5738           if (Offset >= ExprFragmentSizeInBits)
5739             break;
5740           // The register is partially outside the expression fragment, only
5741           // the low bits within the fragment are relevant for debug info.
5742           if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
5743             RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
5744           }
5745         }
5746 
5747         auto FragmentExpr = DIExpression::createFragmentExpression(
5748             Expr, Offset, RegFragmentSizeInBits);
5749         Offset += RegAndSize.second;
5750         // If a valid fragment expression cannot be created, the variable's
5751         // correct value cannot be determined and so it is set as Undef.
5752         if (!FragmentExpr) {
5753           SDDbgValue *SDV = DAG.getConstantDbgValue(
5754               Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
5755           DAG.AddDbgValue(SDV, false);
5756           continue;
5757         }
5758         MachineInstr *NewMI =
5759             MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
5760                              Kind != FuncArgumentDbgValueKind::Value);
5761         FuncInfo.ArgDbgValues.push_back(NewMI);
5762       }
5763     };
5764 
5765     // Check if ValueMap has reg number.
5766     DenseMap<const Value *, Register>::const_iterator
5767       VMI = FuncInfo.ValueMap.find(V);
5768     if (VMI != FuncInfo.ValueMap.end()) {
5769       const auto &TLI = DAG.getTargetLoweringInfo();
5770       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
5771                        V->getType(), std::nullopt);
5772       if (RFV.occupiesMultipleRegs()) {
5773         splitMultiRegDbgValue(RFV.getRegsAndSizes());
5774         return true;
5775       }
5776 
5777       Op = MachineOperand::CreateReg(VMI->second, false);
5778       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5779     } else if (ArgRegsAndSizes.size() > 1) {
5780       // This was split due to the calling convention, and no virtual register
5781       // mapping exists for the value.
5782       splitMultiRegDbgValue(ArgRegsAndSizes);
5783       return true;
5784     }
5785   }
5786 
5787   if (!Op)
5788     return false;
5789 
5790   assert(Variable->isValidLocationForIntrinsic(DL) &&
5791          "Expected inlined-at fields to agree");
5792   MachineInstr *NewMI = nullptr;
5793 
5794   if (Op->isReg())
5795     NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
5796   else
5797     NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
5798                     Variable, Expr);
5799 
5800   // Otherwise, use ArgDbgValues.
5801   FuncInfo.ArgDbgValues.push_back(NewMI);
5802   return true;
5803 }
5804 
5805 /// Return the appropriate SDDbgValue based on N.
5806 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5807                                              DILocalVariable *Variable,
5808                                              DIExpression *Expr,
5809                                              const DebugLoc &dl,
5810                                              unsigned DbgSDNodeOrder) {
5811   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5812     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5813     // stack slot locations.
5814     //
5815     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5816     // debug values here after optimization:
5817     //
5818     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5819     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5820     //
5821     // Both describe the direct values of their associated variables.
5822     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5823                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5824   }
5825   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5826                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5827 }
5828 
5829 static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
5830   switch (Intrinsic) {
5831   case Intrinsic::smul_fix:
5832     return ISD::SMULFIX;
5833   case Intrinsic::umul_fix:
5834     return ISD::UMULFIX;
5835   case Intrinsic::smul_fix_sat:
5836     return ISD::SMULFIXSAT;
5837   case Intrinsic::umul_fix_sat:
5838     return ISD::UMULFIXSAT;
5839   case Intrinsic::sdiv_fix:
5840     return ISD::SDIVFIX;
5841   case Intrinsic::udiv_fix:
5842     return ISD::UDIVFIX;
5843   case Intrinsic::sdiv_fix_sat:
5844     return ISD::SDIVFIXSAT;
5845   case Intrinsic::udiv_fix_sat:
5846     return ISD::UDIVFIXSAT;
5847   default:
5848     llvm_unreachable("Unhandled fixed point intrinsic");
5849   }
5850 }
5851 
5852 void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
5853                                            const char *FunctionName) {
5854   assert(FunctionName && "FunctionName must not be nullptr");
5855   SDValue Callee = DAG.getExternalSymbol(
5856       FunctionName,
5857       DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
5858   LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5859 }
5860 
5861 /// Given a @llvm.call.preallocated.setup, return the corresponding
5862 /// preallocated call.
5863 static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
5864   assert(cast<CallBase>(PreallocatedSetup)
5865                  ->getCalledFunction()
5866                  ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
5867          "expected call_preallocated_setup Value");
5868   for (const auto *U : PreallocatedSetup->users()) {
5869     auto *UseCall = cast<CallBase>(U);
5870     const Function *Fn = UseCall->getCalledFunction();
5871     if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
5872       return UseCall;
5873     }
5874   }
5875   llvm_unreachable("expected corresponding call to preallocated setup/arg");
5876 }
5877 
5878 /// Lower the call to the specified intrinsic function.
5879 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
5880                                              unsigned Intrinsic) {
5881   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5882   SDLoc sdl = getCurSDLoc();
5883   DebugLoc dl = getCurDebugLoc();
5884   SDValue Res;
5885 
5886   SDNodeFlags Flags;
5887   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
5888     Flags.copyFMF(*FPOp);
5889 
5890   switch (Intrinsic) {
5891   default:
5892     // By default, turn this into a target intrinsic node.
5893     visitTargetIntrinsic(I, Intrinsic);
5894     return;
5895   case Intrinsic::vscale: {
5896     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5897     setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
5898     return;
5899   }
5900   case Intrinsic::vastart:  visitVAStart(I); return;
5901   case Intrinsic::vaend:    visitVAEnd(I); return;
5902   case Intrinsic::vacopy:   visitVACopy(I); return;
5903   case Intrinsic::returnaddress:
5904     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5905                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5906                              getValue(I.getArgOperand(0))));
5907     return;
5908   case Intrinsic::addressofreturnaddress:
5909     setValue(&I,
5910              DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5911                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5912     return;
5913   case Intrinsic::sponentry:
5914     setValue(&I,
5915              DAG.getNode(ISD::SPONENTRY, sdl,
5916                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5917     return;
5918   case Intrinsic::frameaddress:
5919     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5920                              TLI.getFrameIndexTy(DAG.getDataLayout()),
5921                              getValue(I.getArgOperand(0))));
5922     return;
5923   case Intrinsic::read_volatile_register:
5924   case Intrinsic::read_register: {
5925     Value *Reg = I.getArgOperand(0);
5926     SDValue Chain = getRoot();
5927     SDValue RegName =
5928         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5929     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5930     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5931       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5932     setValue(&I, Res);
5933     DAG.setRoot(Res.getValue(1));
5934     return;
5935   }
5936   case Intrinsic::write_register: {
5937     Value *Reg = I.getArgOperand(0);
5938     Value *RegValue = I.getArgOperand(1);
5939     SDValue Chain = getRoot();
5940     SDValue RegName =
5941         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5942     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5943                             RegName, getValue(RegValue)));
5944     return;
5945   }
5946   case Intrinsic::memcpy: {
5947     const auto &MCI = cast<MemCpyInst>(I);
5948     SDValue Op1 = getValue(I.getArgOperand(0));
5949     SDValue Op2 = getValue(I.getArgOperand(1));
5950     SDValue Op3 = getValue(I.getArgOperand(2));
5951     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5952     Align DstAlign = MCI.getDestAlign().valueOrOne();
5953     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5954     Align Alignment = std::min(DstAlign, SrcAlign);
5955     bool isVol = MCI.isVolatile();
5956     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5957     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5958     // node.
5959     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5960     SDValue MC = DAG.getMemcpy(
5961         Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5962         /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
5963         MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
5964     updateDAGForMaybeTailCall(MC);
5965     return;
5966   }
5967   case Intrinsic::memcpy_inline: {
5968     const auto &MCI = cast<MemCpyInlineInst>(I);
5969     SDValue Dst = getValue(I.getArgOperand(0));
5970     SDValue Src = getValue(I.getArgOperand(1));
5971     SDValue Size = getValue(I.getArgOperand(2));
5972     assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
5973     // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
5974     Align DstAlign = MCI.getDestAlign().valueOrOne();
5975     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5976     Align Alignment = std::min(DstAlign, SrcAlign);
5977     bool isVol = MCI.isVolatile();
5978     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5979     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5980     // node.
5981     SDValue MC = DAG.getMemcpy(
5982         getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
5983         /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
5984         MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
5985     updateDAGForMaybeTailCall(MC);
5986     return;
5987   }
5988   case Intrinsic::memset: {
5989     const auto &MSI = cast<MemSetInst>(I);
5990     SDValue Op1 = getValue(I.getArgOperand(0));
5991     SDValue Op2 = getValue(I.getArgOperand(1));
5992     SDValue Op3 = getValue(I.getArgOperand(2));
5993     // @llvm.memset defines 0 and 1 to both mean no alignment.
5994     Align Alignment = MSI.getDestAlign().valueOrOne();
5995     bool isVol = MSI.isVolatile();
5996     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5997     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5998     SDValue MS = DAG.getMemset(
5999         Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6000         isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6001     updateDAGForMaybeTailCall(MS);
6002     return;
6003   }
6004   case Intrinsic::memset_inline: {
6005     const auto &MSII = cast<MemSetInlineInst>(I);
6006     SDValue Dst = getValue(I.getArgOperand(0));
6007     SDValue Value = getValue(I.getArgOperand(1));
6008     SDValue Size = getValue(I.getArgOperand(2));
6009     assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6010     // @llvm.memset defines 0 and 1 to both mean no alignment.
6011     Align DstAlign = MSII.getDestAlign().valueOrOne();
6012     bool isVol = MSII.isVolatile();
6013     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6014     SDValue Root = isVol ? getRoot() : getMemoryRoot();
6015     SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6016                                /* AlwaysInline */ true, isTC,
6017                                MachinePointerInfo(I.getArgOperand(0)),
6018                                I.getAAMetadata());
6019     updateDAGForMaybeTailCall(MC);
6020     return;
6021   }
6022   case Intrinsic::memmove: {
6023     const auto &MMI = cast<MemMoveInst>(I);
6024     SDValue Op1 = getValue(I.getArgOperand(0));
6025     SDValue Op2 = getValue(I.getArgOperand(1));
6026     SDValue Op3 = getValue(I.getArgOperand(2));
6027     // @llvm.memmove defines 0 and 1 to both mean no alignment.
6028     Align DstAlign = MMI.getDestAlign().valueOrOne();
6029     Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6030     Align Alignment = std::min(DstAlign, SrcAlign);
6031     bool isVol = MMI.isVolatile();
6032     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6033     // FIXME: Support passing different dest/src alignments to the memmove DAG
6034     // node.
6035     SDValue Root = isVol ? getRoot() : getMemoryRoot();
6036     SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6037                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
6038                                 MachinePointerInfo(I.getArgOperand(1)),
6039                                 I.getAAMetadata(), AA);
6040     updateDAGForMaybeTailCall(MM);
6041     return;
6042   }
6043   case Intrinsic::memcpy_element_unordered_atomic: {
6044     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6045     SDValue Dst = getValue(MI.getRawDest());
6046     SDValue Src = getValue(MI.getRawSource());
6047     SDValue Length = getValue(MI.getLength());
6048 
6049     Type *LengthTy = MI.getLength()->getType();
6050     unsigned ElemSz = MI.getElementSizeInBytes();
6051     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6052     SDValue MC =
6053         DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6054                             isTC, MachinePointerInfo(MI.getRawDest()),
6055                             MachinePointerInfo(MI.getRawSource()));
6056     updateDAGForMaybeTailCall(MC);
6057     return;
6058   }
6059   case Intrinsic::memmove_element_unordered_atomic: {
6060     auto &MI = cast<AtomicMemMoveInst>(I);
6061     SDValue Dst = getValue(MI.getRawDest());
6062     SDValue Src = getValue(MI.getRawSource());
6063     SDValue Length = getValue(MI.getLength());
6064 
6065     Type *LengthTy = MI.getLength()->getType();
6066     unsigned ElemSz = MI.getElementSizeInBytes();
6067     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6068     SDValue MC =
6069         DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6070                              isTC, MachinePointerInfo(MI.getRawDest()),
6071                              MachinePointerInfo(MI.getRawSource()));
6072     updateDAGForMaybeTailCall(MC);
6073     return;
6074   }
6075   case Intrinsic::memset_element_unordered_atomic: {
6076     auto &MI = cast<AtomicMemSetInst>(I);
6077     SDValue Dst = getValue(MI.getRawDest());
6078     SDValue Val = getValue(MI.getValue());
6079     SDValue Length = getValue(MI.getLength());
6080 
6081     Type *LengthTy = MI.getLength()->getType();
6082     unsigned ElemSz = MI.getElementSizeInBytes();
6083     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6084     SDValue MC =
6085         DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6086                             isTC, MachinePointerInfo(MI.getRawDest()));
6087     updateDAGForMaybeTailCall(MC);
6088     return;
6089   }
6090   case Intrinsic::call_preallocated_setup: {
6091     const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6092     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6093     SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6094                               getRoot(), SrcValue);
6095     setValue(&I, Res);
6096     DAG.setRoot(Res);
6097     return;
6098   }
6099   case Intrinsic::call_preallocated_arg: {
6100     const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6101     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6102     SDValue Ops[3];
6103     Ops[0] = getRoot();
6104     Ops[1] = SrcValue;
6105     Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6106                                    MVT::i32); // arg index
6107     SDValue Res = DAG.getNode(
6108         ISD::PREALLOCATED_ARG, sdl,
6109         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6110     setValue(&I, Res);
6111     DAG.setRoot(Res.getValue(1));
6112     return;
6113   }
6114   case Intrinsic::dbg_declare: {
6115     // Debug intrinsics are handled separately in assignment tracking mode.
6116     if (isAssignmentTrackingEnabled(*I.getFunction()->getParent()))
6117       return;
6118     // Assume dbg.declare can not currently use DIArgList, i.e.
6119     // it is non-variadic.
6120     const auto &DI = cast<DbgVariableIntrinsic>(I);
6121     assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6122     DILocalVariable *Variable = DI.getVariable();
6123     DIExpression *Expression = DI.getExpression();
6124     dropDanglingDebugInfo(Variable, Expression);
6125     assert(Variable && "Missing variable");
6126     LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DI
6127                       << "\n");
6128     // Check if address has undef value.
6129     const Value *Address = DI.getVariableLocationOp(0);
6130     if (!Address || isa<UndefValue>(Address) ||
6131         (Address->use_empty() && !isa<Argument>(Address))) {
6132       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6133                         << " (bad/undef/unused-arg address)\n");
6134       return;
6135     }
6136 
6137     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
6138 
6139     // Check if this variable can be described by a frame index, typically
6140     // either as a static alloca or a byval parameter.
6141     int FI = std::numeric_limits<int>::max();
6142     if (const auto *AI =
6143             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
6144       if (AI->isStaticAlloca()) {
6145         auto I = FuncInfo.StaticAllocaMap.find(AI);
6146         if (I != FuncInfo.StaticAllocaMap.end())
6147           FI = I->second;
6148       }
6149     } else if (const auto *Arg = dyn_cast<Argument>(
6150                    Address->stripInBoundsConstantOffsets())) {
6151       FI = FuncInfo.getArgumentFrameIndex(Arg);
6152     }
6153 
6154     // llvm.dbg.declare is handled as a frame index in the MachineFunction
6155     // variable table.
6156     if (FI != std::numeric_limits<int>::max()) {
6157       LLVM_DEBUG(dbgs() << "Skipping " << DI
6158                         << " (variable info stashed in MF side table)\n");
6159       return;
6160     }
6161 
6162     SDValue &N = NodeMap[Address];
6163     if (!N.getNode() && isa<Argument>(Address))
6164       // Check unused arguments map.
6165       N = UnusedArgNodeMap[Address];
6166     SDDbgValue *SDV;
6167     if (N.getNode()) {
6168       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
6169         Address = BCI->getOperand(0);
6170       // Parameters are handled specially.
6171       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
6172       if (isParameter && FINode) {
6173         // Byval parameter. We have a frame index at this point.
6174         SDV =
6175             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
6176                                       /*IsIndirect*/ true, dl, SDNodeOrder);
6177       } else if (isa<Argument>(Address)) {
6178         // Address is an argument, so try to emit its dbg value using
6179         // virtual register info from the FuncInfo.ValueMap.
6180         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6181                                  FuncArgumentDbgValueKind::Declare, N);
6182         return;
6183       } else {
6184         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
6185                               true, dl, SDNodeOrder);
6186       }
6187       DAG.AddDbgValue(SDV, isParameter);
6188     } else {
6189       // If Address is an argument then try to emit its dbg value using
6190       // virtual register info from the FuncInfo.ValueMap.
6191       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6192                                     FuncArgumentDbgValueKind::Declare, N)) {
6193         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6194                           << " (could not emit func-arg dbg_value)\n");
6195       }
6196     }
6197     return;
6198   }
6199   case Intrinsic::dbg_label: {
6200     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6201     DILabel *Label = DI.getLabel();
6202     assert(Label && "Missing label");
6203 
6204     SDDbgLabel *SDV;
6205     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6206     DAG.AddDbgLabel(SDV);
6207     return;
6208   }
6209   case Intrinsic::dbg_assign: {
6210     // Debug intrinsics are handled seperately in assignment tracking mode.
6211     assert(isAssignmentTrackingEnabled(*I.getFunction()->getParent()) &&
6212            "expected assignment tracking to be enabled");
6213     return;
6214   }
6215   case Intrinsic::dbg_value: {
6216     // Debug intrinsics are handled seperately in assignment tracking mode.
6217     if (isAssignmentTrackingEnabled(*I.getFunction()->getParent()))
6218       return;
6219     const DbgValueInst &DI = cast<DbgValueInst>(I);
6220     assert(DI.getVariable() && "Missing variable");
6221 
6222     DILocalVariable *Variable = DI.getVariable();
6223     DIExpression *Expression = DI.getExpression();
6224     dropDanglingDebugInfo(Variable, Expression);
6225     SmallVector<Value *, 4> Values(DI.getValues());
6226     if (Values.empty())
6227       return;
6228 
6229     if (llvm::is_contained(Values, nullptr))
6230       return;
6231 
6232     bool IsVariadic = DI.hasArgList();
6233     if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6234                           SDNodeOrder, IsVariadic))
6235       addDanglingDebugInfo(&DI, SDNodeOrder);
6236     return;
6237   }
6238 
6239   case Intrinsic::eh_typeid_for: {
6240     // Find the type id for the given typeinfo.
6241     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6242     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6243     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6244     setValue(&I, Res);
6245     return;
6246   }
6247 
6248   case Intrinsic::eh_return_i32:
6249   case Intrinsic::eh_return_i64:
6250     DAG.getMachineFunction().setCallsEHReturn(true);
6251     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6252                             MVT::Other,
6253                             getControlRoot(),
6254                             getValue(I.getArgOperand(0)),
6255                             getValue(I.getArgOperand(1))));
6256     return;
6257   case Intrinsic::eh_unwind_init:
6258     DAG.getMachineFunction().setCallsUnwindInit(true);
6259     return;
6260   case Intrinsic::eh_dwarf_cfa:
6261     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
6262                              TLI.getPointerTy(DAG.getDataLayout()),
6263                              getValue(I.getArgOperand(0))));
6264     return;
6265   case Intrinsic::eh_sjlj_callsite: {
6266     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6267     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6268     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6269 
6270     MMI.setCurrentCallSite(CI->getZExtValue());
6271     return;
6272   }
6273   case Intrinsic::eh_sjlj_functioncontext: {
6274     // Get and store the index of the function context.
6275     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6276     AllocaInst *FnCtx =
6277       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6278     int FI = FuncInfo.StaticAllocaMap[FnCtx];
6279     MFI.setFunctionContextIndex(FI);
6280     return;
6281   }
6282   case Intrinsic::eh_sjlj_setjmp: {
6283     SDValue Ops[2];
6284     Ops[0] = getRoot();
6285     Ops[1] = getValue(I.getArgOperand(0));
6286     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6287                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
6288     setValue(&I, Op.getValue(0));
6289     DAG.setRoot(Op.getValue(1));
6290     return;
6291   }
6292   case Intrinsic::eh_sjlj_longjmp:
6293     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6294                             getRoot(), getValue(I.getArgOperand(0))));
6295     return;
6296   case Intrinsic::eh_sjlj_setup_dispatch:
6297     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6298                             getRoot()));
6299     return;
6300   case Intrinsic::masked_gather:
6301     visitMaskedGather(I);
6302     return;
6303   case Intrinsic::masked_load:
6304     visitMaskedLoad(I);
6305     return;
6306   case Intrinsic::masked_scatter:
6307     visitMaskedScatter(I);
6308     return;
6309   case Intrinsic::masked_store:
6310     visitMaskedStore(I);
6311     return;
6312   case Intrinsic::masked_expandload:
6313     visitMaskedLoad(I, true /* IsExpanding */);
6314     return;
6315   case Intrinsic::masked_compressstore:
6316     visitMaskedStore(I, true /* IsCompressing */);
6317     return;
6318   case Intrinsic::powi:
6319     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6320                             getValue(I.getArgOperand(1)), DAG));
6321     return;
6322   case Intrinsic::log:
6323     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6324     return;
6325   case Intrinsic::log2:
6326     setValue(&I,
6327              expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6328     return;
6329   case Intrinsic::log10:
6330     setValue(&I,
6331              expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6332     return;
6333   case Intrinsic::exp:
6334     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6335     return;
6336   case Intrinsic::exp2:
6337     setValue(&I,
6338              expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6339     return;
6340   case Intrinsic::pow:
6341     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6342                            getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6343     return;
6344   case Intrinsic::sqrt:
6345   case Intrinsic::fabs:
6346   case Intrinsic::sin:
6347   case Intrinsic::cos:
6348   case Intrinsic::floor:
6349   case Intrinsic::ceil:
6350   case Intrinsic::trunc:
6351   case Intrinsic::rint:
6352   case Intrinsic::nearbyint:
6353   case Intrinsic::round:
6354   case Intrinsic::roundeven:
6355   case Intrinsic::canonicalize: {
6356     unsigned Opcode;
6357     switch (Intrinsic) {
6358     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6359     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
6360     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
6361     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
6362     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
6363     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
6364     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
6365     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
6366     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
6367     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6368     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
6369     case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6370     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6371     }
6372 
6373     setValue(&I, DAG.getNode(Opcode, sdl,
6374                              getValue(I.getArgOperand(0)).getValueType(),
6375                              getValue(I.getArgOperand(0)), Flags));
6376     return;
6377   }
6378   case Intrinsic::lround:
6379   case Intrinsic::llround:
6380   case Intrinsic::lrint:
6381   case Intrinsic::llrint: {
6382     unsigned Opcode;
6383     switch (Intrinsic) {
6384     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6385     case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
6386     case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6387     case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
6388     case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
6389     }
6390 
6391     EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6392     setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6393                              getValue(I.getArgOperand(0))));
6394     return;
6395   }
6396   case Intrinsic::minnum:
6397     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
6398                              getValue(I.getArgOperand(0)).getValueType(),
6399                              getValue(I.getArgOperand(0)),
6400                              getValue(I.getArgOperand(1)), Flags));
6401     return;
6402   case Intrinsic::maxnum:
6403     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
6404                              getValue(I.getArgOperand(0)).getValueType(),
6405                              getValue(I.getArgOperand(0)),
6406                              getValue(I.getArgOperand(1)), Flags));
6407     return;
6408   case Intrinsic::minimum:
6409     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
6410                              getValue(I.getArgOperand(0)).getValueType(),
6411                              getValue(I.getArgOperand(0)),
6412                              getValue(I.getArgOperand(1)), Flags));
6413     return;
6414   case Intrinsic::maximum:
6415     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
6416                              getValue(I.getArgOperand(0)).getValueType(),
6417                              getValue(I.getArgOperand(0)),
6418                              getValue(I.getArgOperand(1)), Flags));
6419     return;
6420   case Intrinsic::copysign:
6421     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
6422                              getValue(I.getArgOperand(0)).getValueType(),
6423                              getValue(I.getArgOperand(0)),
6424                              getValue(I.getArgOperand(1)), Flags));
6425     return;
6426   case Intrinsic::arithmetic_fence: {
6427     setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
6428                              getValue(I.getArgOperand(0)).getValueType(),
6429                              getValue(I.getArgOperand(0)), Flags));
6430     return;
6431   }
6432   case Intrinsic::fma:
6433     setValue(&I, DAG.getNode(
6434                      ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6435                      getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6436                      getValue(I.getArgOperand(2)), Flags));
6437     return;
6438 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
6439   case Intrinsic::INTRINSIC:
6440 #include "llvm/IR/ConstrainedOps.def"
6441     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6442     return;
6443 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6444 #include "llvm/IR/VPIntrinsics.def"
6445     visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6446     return;
6447   case Intrinsic::fptrunc_round: {
6448     // Get the last argument, the metadata and convert it to an integer in the
6449     // call
6450     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6451     std::optional<RoundingMode> RoundMode =
6452         convertStrToRoundingMode(cast<MDString>(MD)->getString());
6453 
6454     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6455 
6456     // Propagate fast-math-flags from IR to node(s).
6457     SDNodeFlags Flags;
6458     Flags.copyFMF(*cast<FPMathOperator>(&I));
6459     SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6460 
6461     SDValue Result;
6462     Result = DAG.getNode(
6463         ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6464         DAG.getTargetConstant((int)*RoundMode, sdl,
6465                               TLI.getPointerTy(DAG.getDataLayout())));
6466     setValue(&I, Result);
6467 
6468     return;
6469   }
6470   case Intrinsic::fmuladd: {
6471     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6472     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6473         TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
6474       setValue(&I, DAG.getNode(ISD::FMA, sdl,
6475                                getValue(I.getArgOperand(0)).getValueType(),
6476                                getValue(I.getArgOperand(0)),
6477                                getValue(I.getArgOperand(1)),
6478                                getValue(I.getArgOperand(2)), Flags));
6479     } else {
6480       // TODO: Intrinsic calls should have fast-math-flags.
6481       SDValue Mul = DAG.getNode(
6482           ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6483           getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6484       SDValue Add = DAG.getNode(ISD::FADD, sdl,
6485                                 getValue(I.getArgOperand(0)).getValueType(),
6486                                 Mul, getValue(I.getArgOperand(2)), Flags);
6487       setValue(&I, Add);
6488     }
6489     return;
6490   }
6491   case Intrinsic::convert_to_fp16:
6492     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6493                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6494                                          getValue(I.getArgOperand(0)),
6495                                          DAG.getTargetConstant(0, sdl,
6496                                                                MVT::i32))));
6497     return;
6498   case Intrinsic::convert_from_fp16:
6499     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6500                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
6501                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6502                                          getValue(I.getArgOperand(0)))));
6503     return;
6504   case Intrinsic::fptosi_sat: {
6505     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6506     setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
6507                              getValue(I.getArgOperand(0)),
6508                              DAG.getValueType(VT.getScalarType())));
6509     return;
6510   }
6511   case Intrinsic::fptoui_sat: {
6512     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6513     setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
6514                              getValue(I.getArgOperand(0)),
6515                              DAG.getValueType(VT.getScalarType())));
6516     return;
6517   }
6518   case Intrinsic::set_rounding:
6519     Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6520                       {getRoot(), getValue(I.getArgOperand(0))});
6521     setValue(&I, Res);
6522     DAG.setRoot(Res.getValue(0));
6523     return;
6524   case Intrinsic::is_fpclass: {
6525     const DataLayout DLayout = DAG.getDataLayout();
6526     EVT DestVT = TLI.getValueType(DLayout, I.getType());
6527     EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6528     FPClassTest Test = static_cast<FPClassTest>(
6529         cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
6530     MachineFunction &MF = DAG.getMachineFunction();
6531     const Function &F = MF.getFunction();
6532     SDValue Op = getValue(I.getArgOperand(0));
6533     SDNodeFlags Flags;
6534     Flags.setNoFPExcept(
6535         !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6536     // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6537     // expansion can use illegal types. Making expansion early allows
6538     // legalizing these types prior to selection.
6539     if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6540       SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6541       setValue(&I, Result);
6542       return;
6543     }
6544 
6545     SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6546     SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6547     setValue(&I, V);
6548     return;
6549   }
6550   case Intrinsic::pcmarker: {
6551     SDValue Tmp = getValue(I.getArgOperand(0));
6552     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6553     return;
6554   }
6555   case Intrinsic::readcyclecounter: {
6556     SDValue Op = getRoot();
6557     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
6558                       DAG.getVTList(MVT::i64, MVT::Other), Op);
6559     setValue(&I, Res);
6560     DAG.setRoot(Res.getValue(1));
6561     return;
6562   }
6563   case Intrinsic::bitreverse:
6564     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
6565                              getValue(I.getArgOperand(0)).getValueType(),
6566                              getValue(I.getArgOperand(0))));
6567     return;
6568   case Intrinsic::bswap:
6569     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
6570                              getValue(I.getArgOperand(0)).getValueType(),
6571                              getValue(I.getArgOperand(0))));
6572     return;
6573   case Intrinsic::cttz: {
6574     SDValue Arg = getValue(I.getArgOperand(0));
6575     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6576     EVT Ty = Arg.getValueType();
6577     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
6578                              sdl, Ty, Arg));
6579     return;
6580   }
6581   case Intrinsic::ctlz: {
6582     SDValue Arg = getValue(I.getArgOperand(0));
6583     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6584     EVT Ty = Arg.getValueType();
6585     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
6586                              sdl, Ty, Arg));
6587     return;
6588   }
6589   case Intrinsic::ctpop: {
6590     SDValue Arg = getValue(I.getArgOperand(0));
6591     EVT Ty = Arg.getValueType();
6592     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
6593     return;
6594   }
6595   case Intrinsic::fshl:
6596   case Intrinsic::fshr: {
6597     bool IsFSHL = Intrinsic == Intrinsic::fshl;
6598     SDValue X = getValue(I.getArgOperand(0));
6599     SDValue Y = getValue(I.getArgOperand(1));
6600     SDValue Z = getValue(I.getArgOperand(2));
6601     EVT VT = X.getValueType();
6602 
6603     if (X == Y) {
6604       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
6605       setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
6606     } else {
6607       auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
6608       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
6609     }
6610     return;
6611   }
6612   case Intrinsic::sadd_sat: {
6613     SDValue Op1 = getValue(I.getArgOperand(0));
6614     SDValue Op2 = getValue(I.getArgOperand(1));
6615     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6616     return;
6617   }
6618   case Intrinsic::uadd_sat: {
6619     SDValue Op1 = getValue(I.getArgOperand(0));
6620     SDValue Op2 = getValue(I.getArgOperand(1));
6621     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6622     return;
6623   }
6624   case Intrinsic::ssub_sat: {
6625     SDValue Op1 = getValue(I.getArgOperand(0));
6626     SDValue Op2 = getValue(I.getArgOperand(1));
6627     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6628     return;
6629   }
6630   case Intrinsic::usub_sat: {
6631     SDValue Op1 = getValue(I.getArgOperand(0));
6632     SDValue Op2 = getValue(I.getArgOperand(1));
6633     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6634     return;
6635   }
6636   case Intrinsic::sshl_sat: {
6637     SDValue Op1 = getValue(I.getArgOperand(0));
6638     SDValue Op2 = getValue(I.getArgOperand(1));
6639     setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6640     return;
6641   }
6642   case Intrinsic::ushl_sat: {
6643     SDValue Op1 = getValue(I.getArgOperand(0));
6644     SDValue Op2 = getValue(I.getArgOperand(1));
6645     setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6646     return;
6647   }
6648   case Intrinsic::smul_fix:
6649   case Intrinsic::umul_fix:
6650   case Intrinsic::smul_fix_sat:
6651   case Intrinsic::umul_fix_sat: {
6652     SDValue Op1 = getValue(I.getArgOperand(0));
6653     SDValue Op2 = getValue(I.getArgOperand(1));
6654     SDValue Op3 = getValue(I.getArgOperand(2));
6655     setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6656                              Op1.getValueType(), Op1, Op2, Op3));
6657     return;
6658   }
6659   case Intrinsic::sdiv_fix:
6660   case Intrinsic::udiv_fix:
6661   case Intrinsic::sdiv_fix_sat:
6662   case Intrinsic::udiv_fix_sat: {
6663     SDValue Op1 = getValue(I.getArgOperand(0));
6664     SDValue Op2 = getValue(I.getArgOperand(1));
6665     SDValue Op3 = getValue(I.getArgOperand(2));
6666     setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6667                               Op1, Op2, Op3, DAG, TLI));
6668     return;
6669   }
6670   case Intrinsic::smax: {
6671     SDValue Op1 = getValue(I.getArgOperand(0));
6672     SDValue Op2 = getValue(I.getArgOperand(1));
6673     setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
6674     return;
6675   }
6676   case Intrinsic::smin: {
6677     SDValue Op1 = getValue(I.getArgOperand(0));
6678     SDValue Op2 = getValue(I.getArgOperand(1));
6679     setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
6680     return;
6681   }
6682   case Intrinsic::umax: {
6683     SDValue Op1 = getValue(I.getArgOperand(0));
6684     SDValue Op2 = getValue(I.getArgOperand(1));
6685     setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
6686     return;
6687   }
6688   case Intrinsic::umin: {
6689     SDValue Op1 = getValue(I.getArgOperand(0));
6690     SDValue Op2 = getValue(I.getArgOperand(1));
6691     setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
6692     return;
6693   }
6694   case Intrinsic::abs: {
6695     // TODO: Preserve "int min is poison" arg in SDAG?
6696     SDValue Op1 = getValue(I.getArgOperand(0));
6697     setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
6698     return;
6699   }
6700   case Intrinsic::stacksave: {
6701     SDValue Op = getRoot();
6702     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6703     Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
6704     setValue(&I, Res);
6705     DAG.setRoot(Res.getValue(1));
6706     return;
6707   }
6708   case Intrinsic::stackrestore:
6709     Res = getValue(I.getArgOperand(0));
6710     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
6711     return;
6712   case Intrinsic::get_dynamic_area_offset: {
6713     SDValue Op = getRoot();
6714     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6715     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6716     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
6717     // target.
6718     if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
6719       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
6720                          " intrinsic!");
6721     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
6722                       Op);
6723     DAG.setRoot(Op);
6724     setValue(&I, Res);
6725     return;
6726   }
6727   case Intrinsic::stackguard: {
6728     MachineFunction &MF = DAG.getMachineFunction();
6729     const Module &M = *MF.getFunction().getParent();
6730     SDValue Chain = getRoot();
6731     if (TLI.useLoadStackGuardNode()) {
6732       Res = getLoadStackGuard(DAG, sdl, Chain);
6733     } else {
6734       EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6735       const Value *Global = TLI.getSDagStackGuard(M);
6736       Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
6737       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
6738                         MachinePointerInfo(Global, 0), Align,
6739                         MachineMemOperand::MOVolatile);
6740     }
6741     if (TLI.useStackGuardXorFP())
6742       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
6743     DAG.setRoot(Chain);
6744     setValue(&I, Res);
6745     return;
6746   }
6747   case Intrinsic::stackprotector: {
6748     // Emit code into the DAG to store the stack guard onto the stack.
6749     MachineFunction &MF = DAG.getMachineFunction();
6750     MachineFrameInfo &MFI = MF.getFrameInfo();
6751     SDValue Src, Chain = getRoot();
6752 
6753     if (TLI.useLoadStackGuardNode())
6754       Src = getLoadStackGuard(DAG, sdl, Chain);
6755     else
6756       Src = getValue(I.getArgOperand(0));   // The guard's value.
6757 
6758     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
6759 
6760     int FI = FuncInfo.StaticAllocaMap[Slot];
6761     MFI.setStackProtectorIndex(FI);
6762     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6763 
6764     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
6765 
6766     // Store the stack protector onto the stack.
6767     Res = DAG.getStore(
6768         Chain, sdl, Src, FIN,
6769         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
6770         MaybeAlign(), MachineMemOperand::MOVolatile);
6771     setValue(&I, Res);
6772     DAG.setRoot(Res);
6773     return;
6774   }
6775   case Intrinsic::objectsize:
6776     llvm_unreachable("llvm.objectsize.* should have been lowered already");
6777 
6778   case Intrinsic::is_constant:
6779     llvm_unreachable("llvm.is.constant.* should have been lowered already");
6780 
6781   case Intrinsic::annotation:
6782   case Intrinsic::ptr_annotation:
6783   case Intrinsic::launder_invariant_group:
6784   case Intrinsic::strip_invariant_group:
6785     // Drop the intrinsic, but forward the value
6786     setValue(&I, getValue(I.getOperand(0)));
6787     return;
6788 
6789   case Intrinsic::assume:
6790   case Intrinsic::experimental_noalias_scope_decl:
6791   case Intrinsic::var_annotation:
6792   case Intrinsic::sideeffect:
6793     // Discard annotate attributes, noalias scope declarations, assumptions, and
6794     // artificial side-effects.
6795     return;
6796 
6797   case Intrinsic::codeview_annotation: {
6798     // Emit a label associated with this metadata.
6799     MachineFunction &MF = DAG.getMachineFunction();
6800     MCSymbol *Label =
6801         MF.getMMI().getContext().createTempSymbol("annotation", true);
6802     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
6803     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
6804     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
6805     DAG.setRoot(Res);
6806     return;
6807   }
6808 
6809   case Intrinsic::init_trampoline: {
6810     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
6811 
6812     SDValue Ops[6];
6813     Ops[0] = getRoot();
6814     Ops[1] = getValue(I.getArgOperand(0));
6815     Ops[2] = getValue(I.getArgOperand(1));
6816     Ops[3] = getValue(I.getArgOperand(2));
6817     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
6818     Ops[5] = DAG.getSrcValue(F);
6819 
6820     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
6821 
6822     DAG.setRoot(Res);
6823     return;
6824   }
6825   case Intrinsic::adjust_trampoline:
6826     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
6827                              TLI.getPointerTy(DAG.getDataLayout()),
6828                              getValue(I.getArgOperand(0))));
6829     return;
6830   case Intrinsic::gcroot: {
6831     assert(DAG.getMachineFunction().getFunction().hasGC() &&
6832            "only valid in functions with gc specified, enforced by Verifier");
6833     assert(GFI && "implied by previous");
6834     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
6835     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
6836 
6837     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
6838     GFI->addStackRoot(FI->getIndex(), TypeMap);
6839     return;
6840   }
6841   case Intrinsic::gcread:
6842   case Intrinsic::gcwrite:
6843     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
6844   case Intrinsic::get_rounding:
6845     Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
6846     setValue(&I, Res);
6847     DAG.setRoot(Res.getValue(1));
6848     return;
6849 
6850   case Intrinsic::expect:
6851     // Just replace __builtin_expect(exp, c) with EXP.
6852     setValue(&I, getValue(I.getArgOperand(0)));
6853     return;
6854 
6855   case Intrinsic::ubsantrap:
6856   case Intrinsic::debugtrap:
6857   case Intrinsic::trap: {
6858     StringRef TrapFuncName =
6859         I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
6860     if (TrapFuncName.empty()) {
6861       switch (Intrinsic) {
6862       case Intrinsic::trap:
6863         DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
6864         break;
6865       case Intrinsic::debugtrap:
6866         DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
6867         break;
6868       case Intrinsic::ubsantrap:
6869         DAG.setRoot(DAG.getNode(
6870             ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
6871             DAG.getTargetConstant(
6872                 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
6873                 MVT::i32)));
6874         break;
6875       default: llvm_unreachable("unknown trap intrinsic");
6876       }
6877       return;
6878     }
6879     TargetLowering::ArgListTy Args;
6880     if (Intrinsic == Intrinsic::ubsantrap) {
6881       Args.push_back(TargetLoweringBase::ArgListEntry());
6882       Args[0].Val = I.getArgOperand(0);
6883       Args[0].Node = getValue(Args[0].Val);
6884       Args[0].Ty = Args[0].Val->getType();
6885     }
6886 
6887     TargetLowering::CallLoweringInfo CLI(DAG);
6888     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6889         CallingConv::C, I.getType(),
6890         DAG.getExternalSymbol(TrapFuncName.data(),
6891                               TLI.getPointerTy(DAG.getDataLayout())),
6892         std::move(Args));
6893 
6894     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6895     DAG.setRoot(Result.second);
6896     return;
6897   }
6898 
6899   case Intrinsic::uadd_with_overflow:
6900   case Intrinsic::sadd_with_overflow:
6901   case Intrinsic::usub_with_overflow:
6902   case Intrinsic::ssub_with_overflow:
6903   case Intrinsic::umul_with_overflow:
6904   case Intrinsic::smul_with_overflow: {
6905     ISD::NodeType Op;
6906     switch (Intrinsic) {
6907     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6908     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6909     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6910     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6911     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6912     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6913     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6914     }
6915     SDValue Op1 = getValue(I.getArgOperand(0));
6916     SDValue Op2 = getValue(I.getArgOperand(1));
6917 
6918     EVT ResultVT = Op1.getValueType();
6919     EVT OverflowVT = MVT::i1;
6920     if (ResultVT.isVector())
6921       OverflowVT = EVT::getVectorVT(
6922           *Context, OverflowVT, ResultVT.getVectorElementCount());
6923 
6924     SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
6925     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6926     return;
6927   }
6928   case Intrinsic::prefetch: {
6929     SDValue Ops[5];
6930     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6931     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6932     Ops[0] = DAG.getRoot();
6933     Ops[1] = getValue(I.getArgOperand(0));
6934     Ops[2] = getValue(I.getArgOperand(1));
6935     Ops[3] = getValue(I.getArgOperand(2));
6936     Ops[4] = getValue(I.getArgOperand(3));
6937     SDValue Result = DAG.getMemIntrinsicNode(
6938         ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
6939         EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
6940         /* align */ std::nullopt, Flags);
6941 
6942     // Chain the prefetch in parallell with any pending loads, to stay out of
6943     // the way of later optimizations.
6944     PendingLoads.push_back(Result);
6945     Result = getRoot();
6946     DAG.setRoot(Result);
6947     return;
6948   }
6949   case Intrinsic::lifetime_start:
6950   case Intrinsic::lifetime_end: {
6951     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6952     // Stack coloring is not enabled in O0, discard region information.
6953     if (TM.getOptLevel() == CodeGenOpt::None)
6954       return;
6955 
6956     const int64_t ObjectSize =
6957         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
6958     Value *const ObjectPtr = I.getArgOperand(1);
6959     SmallVector<const Value *, 4> Allocas;
6960     getUnderlyingObjects(ObjectPtr, Allocas);
6961 
6962     for (const Value *Alloca : Allocas) {
6963       const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
6964 
6965       // Could not find an Alloca.
6966       if (!LifetimeObject)
6967         continue;
6968 
6969       // First check that the Alloca is static, otherwise it won't have a
6970       // valid frame index.
6971       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6972       if (SI == FuncInfo.StaticAllocaMap.end())
6973         return;
6974 
6975       const int FrameIndex = SI->second;
6976       int64_t Offset;
6977       if (GetPointerBaseWithConstantOffset(
6978               ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
6979         Offset = -1; // Cannot determine offset from alloca to lifetime object.
6980       Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
6981                                 Offset);
6982       DAG.setRoot(Res);
6983     }
6984     return;
6985   }
6986   case Intrinsic::pseudoprobe: {
6987     auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
6988     auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6989     auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
6990     Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
6991     DAG.setRoot(Res);
6992     return;
6993   }
6994   case Intrinsic::invariant_start:
6995     // Discard region information.
6996     setValue(&I,
6997              DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
6998     return;
6999   case Intrinsic::invariant_end:
7000     // Discard region information.
7001     return;
7002   case Intrinsic::clear_cache:
7003     /// FunctionName may be null.
7004     if (const char *FunctionName = TLI.getClearCacheBuiltinName())
7005       lowerCallToExternalSymbol(I, FunctionName);
7006     return;
7007   case Intrinsic::donothing:
7008   case Intrinsic::seh_try_begin:
7009   case Intrinsic::seh_scope_begin:
7010   case Intrinsic::seh_try_end:
7011   case Intrinsic::seh_scope_end:
7012     // ignore
7013     return;
7014   case Intrinsic::experimental_stackmap:
7015     visitStackmap(I);
7016     return;
7017   case Intrinsic::experimental_patchpoint_void:
7018   case Intrinsic::experimental_patchpoint_i64:
7019     visitPatchpoint(I);
7020     return;
7021   case Intrinsic::experimental_gc_statepoint:
7022     LowerStatepoint(cast<GCStatepointInst>(I));
7023     return;
7024   case Intrinsic::experimental_gc_result:
7025     visitGCResult(cast<GCResultInst>(I));
7026     return;
7027   case Intrinsic::experimental_gc_relocate:
7028     visitGCRelocate(cast<GCRelocateInst>(I));
7029     return;
7030   case Intrinsic::instrprof_cover:
7031     llvm_unreachable("instrprof failed to lower a cover");
7032   case Intrinsic::instrprof_increment:
7033     llvm_unreachable("instrprof failed to lower an increment");
7034   case Intrinsic::instrprof_value_profile:
7035     llvm_unreachable("instrprof failed to lower a value profiling call");
7036   case Intrinsic::localescape: {
7037     MachineFunction &MF = DAG.getMachineFunction();
7038     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
7039 
7040     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7041     // is the same on all targets.
7042     for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7043       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7044       if (isa<ConstantPointerNull>(Arg))
7045         continue; // Skip null pointers. They represent a hole in index space.
7046       AllocaInst *Slot = cast<AllocaInst>(Arg);
7047       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7048              "can only escape static allocas");
7049       int FI = FuncInfo.StaticAllocaMap[Slot];
7050       MCSymbol *FrameAllocSym =
7051           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
7052               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
7053       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
7054               TII->get(TargetOpcode::LOCAL_ESCAPE))
7055           .addSym(FrameAllocSym)
7056           .addFrameIndex(FI);
7057     }
7058 
7059     return;
7060   }
7061 
7062   case Intrinsic::localrecover: {
7063     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7064     MachineFunction &MF = DAG.getMachineFunction();
7065 
7066     // Get the symbol that defines the frame offset.
7067     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7068     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7069     unsigned IdxVal =
7070         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7071     MCSymbol *FrameAllocSym =
7072         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
7073             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
7074 
7075     Value *FP = I.getArgOperand(1);
7076     SDValue FPVal = getValue(FP);
7077     EVT PtrVT = FPVal.getValueType();
7078 
7079     // Create a MCSymbol for the label to avoid any target lowering
7080     // that would make this PC relative.
7081     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7082     SDValue OffsetVal =
7083         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7084 
7085     // Add the offset to the FP.
7086     SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7087     setValue(&I, Add);
7088 
7089     return;
7090   }
7091 
7092   case Intrinsic::eh_exceptionpointer:
7093   case Intrinsic::eh_exceptioncode: {
7094     // Get the exception pointer vreg, copy from it, and resize it to fit.
7095     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7096     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7097     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7098     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7099     SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7100     if (Intrinsic == Intrinsic::eh_exceptioncode)
7101       N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7102     setValue(&I, N);
7103     return;
7104   }
7105   case Intrinsic::xray_customevent: {
7106     // Here we want to make sure that the intrinsic behaves as if it has a
7107     // specific calling convention, and only for x86_64.
7108     // FIXME: Support other platforms later.
7109     const auto &Triple = DAG.getTarget().getTargetTriple();
7110     if (Triple.getArch() != Triple::x86_64)
7111       return;
7112 
7113     SmallVector<SDValue, 8> Ops;
7114 
7115     // We want to say that we always want the arguments in registers.
7116     SDValue LogEntryVal = getValue(I.getArgOperand(0));
7117     SDValue StrSizeVal = getValue(I.getArgOperand(1));
7118     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7119     SDValue Chain = getRoot();
7120     Ops.push_back(LogEntryVal);
7121     Ops.push_back(StrSizeVal);
7122     Ops.push_back(Chain);
7123 
7124     // We need to enforce the calling convention for the callsite, so that
7125     // argument ordering is enforced correctly, and that register allocation can
7126     // see that some registers may be assumed clobbered and have to preserve
7127     // them across calls to the intrinsic.
7128     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7129                                            sdl, NodeTys, Ops);
7130     SDValue patchableNode = SDValue(MN, 0);
7131     DAG.setRoot(patchableNode);
7132     setValue(&I, patchableNode);
7133     return;
7134   }
7135   case Intrinsic::xray_typedevent: {
7136     // Here we want to make sure that the intrinsic behaves as if it has a
7137     // specific calling convention, and only for x86_64.
7138     // FIXME: Support other platforms later.
7139     const auto &Triple = DAG.getTarget().getTargetTriple();
7140     if (Triple.getArch() != Triple::x86_64)
7141       return;
7142 
7143     SmallVector<SDValue, 8> Ops;
7144 
7145     // We want to say that we always want the arguments in registers.
7146     // It's unclear to me how manipulating the selection DAG here forces callers
7147     // to provide arguments in registers instead of on the stack.
7148     SDValue LogTypeId = getValue(I.getArgOperand(0));
7149     SDValue LogEntryVal = getValue(I.getArgOperand(1));
7150     SDValue StrSizeVal = getValue(I.getArgOperand(2));
7151     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7152     SDValue Chain = getRoot();
7153     Ops.push_back(LogTypeId);
7154     Ops.push_back(LogEntryVal);
7155     Ops.push_back(StrSizeVal);
7156     Ops.push_back(Chain);
7157 
7158     // We need to enforce the calling convention for the callsite, so that
7159     // argument ordering is enforced correctly, and that register allocation can
7160     // see that some registers may be assumed clobbered and have to preserve
7161     // them across calls to the intrinsic.
7162     MachineSDNode *MN = DAG.getMachineNode(
7163         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7164     SDValue patchableNode = SDValue(MN, 0);
7165     DAG.setRoot(patchableNode);
7166     setValue(&I, patchableNode);
7167     return;
7168   }
7169   case Intrinsic::experimental_deoptimize:
7170     LowerDeoptimizeCall(&I);
7171     return;
7172   case Intrinsic::experimental_stepvector:
7173     visitStepVector(I);
7174     return;
7175   case Intrinsic::vector_reduce_fadd:
7176   case Intrinsic::vector_reduce_fmul:
7177   case Intrinsic::vector_reduce_add:
7178   case Intrinsic::vector_reduce_mul:
7179   case Intrinsic::vector_reduce_and:
7180   case Intrinsic::vector_reduce_or:
7181   case Intrinsic::vector_reduce_xor:
7182   case Intrinsic::vector_reduce_smax:
7183   case Intrinsic::vector_reduce_smin:
7184   case Intrinsic::vector_reduce_umax:
7185   case Intrinsic::vector_reduce_umin:
7186   case Intrinsic::vector_reduce_fmax:
7187   case Intrinsic::vector_reduce_fmin:
7188     visitVectorReduce(I, Intrinsic);
7189     return;
7190 
7191   case Intrinsic::icall_branch_funnel: {
7192     SmallVector<SDValue, 16> Ops;
7193     Ops.push_back(getValue(I.getArgOperand(0)));
7194 
7195     int64_t Offset;
7196     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7197         I.getArgOperand(1), Offset, DAG.getDataLayout()));
7198     if (!Base)
7199       report_fatal_error(
7200           "llvm.icall.branch.funnel operand must be a GlobalValue");
7201     Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7202 
7203     struct BranchFunnelTarget {
7204       int64_t Offset;
7205       SDValue Target;
7206     };
7207     SmallVector<BranchFunnelTarget, 8> Targets;
7208 
7209     for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7210       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7211           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7212       if (ElemBase != Base)
7213         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7214                            "to the same GlobalValue");
7215 
7216       SDValue Val = getValue(I.getArgOperand(Op + 1));
7217       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7218       if (!GA)
7219         report_fatal_error(
7220             "llvm.icall.branch.funnel operand must be a GlobalValue");
7221       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
7222                                      GA->getGlobal(), sdl, Val.getValueType(),
7223                                      GA->getOffset())});
7224     }
7225     llvm::sort(Targets,
7226                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7227                  return T1.Offset < T2.Offset;
7228                });
7229 
7230     for (auto &T : Targets) {
7231       Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7232       Ops.push_back(T.Target);
7233     }
7234 
7235     Ops.push_back(DAG.getRoot()); // Chain
7236     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7237                                  MVT::Other, Ops),
7238               0);
7239     DAG.setRoot(N);
7240     setValue(&I, N);
7241     HasTailCall = true;
7242     return;
7243   }
7244 
7245   case Intrinsic::wasm_landingpad_index:
7246     // Information this intrinsic contained has been transferred to
7247     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7248     // delete it now.
7249     return;
7250 
7251   case Intrinsic::aarch64_settag:
7252   case Intrinsic::aarch64_settag_zero: {
7253     const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7254     bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7255     SDValue Val = TSI.EmitTargetCodeForSetTag(
7256         DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7257         getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7258         ZeroMemory);
7259     DAG.setRoot(Val);
7260     setValue(&I, Val);
7261     return;
7262   }
7263   case Intrinsic::ptrmask: {
7264     SDValue Ptr = getValue(I.getOperand(0));
7265     SDValue Const = getValue(I.getOperand(1));
7266 
7267     EVT PtrVT = Ptr.getValueType();
7268     setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr,
7269                              DAG.getZExtOrTrunc(Const, sdl, PtrVT)));
7270     return;
7271   }
7272   case Intrinsic::threadlocal_address: {
7273     setValue(&I, getValue(I.getOperand(0)));
7274     return;
7275   }
7276   case Intrinsic::get_active_lane_mask: {
7277     EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7278     SDValue Index = getValue(I.getOperand(0));
7279     EVT ElementVT = Index.getValueType();
7280 
7281     if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7282       visitTargetIntrinsic(I, Intrinsic);
7283       return;
7284     }
7285 
7286     SDValue TripCount = getValue(I.getOperand(1));
7287     auto VecTy = CCVT.changeVectorElementType(ElementVT);
7288 
7289     SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7290     SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7291     SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7292     SDValue VectorInduction = DAG.getNode(
7293         ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7294     SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7295                                  VectorTripCount, ISD::CondCode::SETULT);
7296     setValue(&I, SetCC);
7297     return;
7298   }
7299   case Intrinsic::vector_insert: {
7300     SDValue Vec = getValue(I.getOperand(0));
7301     SDValue SubVec = getValue(I.getOperand(1));
7302     SDValue Index = getValue(I.getOperand(2));
7303 
7304     // The intrinsic's index type is i64, but the SDNode requires an index type
7305     // suitable for the target. Convert the index as required.
7306     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7307     if (Index.getValueType() != VectorIdxTy)
7308       Index = DAG.getVectorIdxConstant(
7309           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7310 
7311     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7312     setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7313                              Index));
7314     return;
7315   }
7316   case Intrinsic::vector_extract: {
7317     SDValue Vec = getValue(I.getOperand(0));
7318     SDValue Index = getValue(I.getOperand(1));
7319     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7320 
7321     // The intrinsic's index type is i64, but the SDNode requires an index type
7322     // suitable for the target. Convert the index as required.
7323     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7324     if (Index.getValueType() != VectorIdxTy)
7325       Index = DAG.getVectorIdxConstant(
7326           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7327 
7328     setValue(&I,
7329              DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7330     return;
7331   }
7332   case Intrinsic::experimental_vector_reverse:
7333     visitVectorReverse(I);
7334     return;
7335   case Intrinsic::experimental_vector_splice:
7336     visitVectorSplice(I);
7337     return;
7338   case Intrinsic::callbr_landingpad:
7339     visitCallBrLandingPad(I);
7340     return;
7341   case Intrinsic::experimental_vector_interleave2:
7342     visitVectorInterleave(I);
7343     return;
7344   case Intrinsic::experimental_vector_deinterleave2:
7345     visitVectorDeinterleave(I);
7346     return;
7347   }
7348 }
7349 
7350 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7351     const ConstrainedFPIntrinsic &FPI) {
7352   SDLoc sdl = getCurSDLoc();
7353 
7354   // We do not need to serialize constrained FP intrinsics against
7355   // each other or against (nonvolatile) loads, so they can be
7356   // chained like loads.
7357   SDValue Chain = DAG.getRoot();
7358   SmallVector<SDValue, 4> Opers;
7359   Opers.push_back(Chain);
7360   if (FPI.isUnaryOp()) {
7361     Opers.push_back(getValue(FPI.getArgOperand(0)));
7362   } else if (FPI.isTernaryOp()) {
7363     Opers.push_back(getValue(FPI.getArgOperand(0)));
7364     Opers.push_back(getValue(FPI.getArgOperand(1)));
7365     Opers.push_back(getValue(FPI.getArgOperand(2)));
7366   } else {
7367     Opers.push_back(getValue(FPI.getArgOperand(0)));
7368     Opers.push_back(getValue(FPI.getArgOperand(1)));
7369   }
7370 
7371   auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7372     assert(Result.getNode()->getNumValues() == 2);
7373 
7374     // Push node to the appropriate list so that future instructions can be
7375     // chained up correctly.
7376     SDValue OutChain = Result.getValue(1);
7377     switch (EB) {
7378     case fp::ExceptionBehavior::ebIgnore:
7379       // The only reason why ebIgnore nodes still need to be chained is that
7380       // they might depend on the current rounding mode, and therefore must
7381       // not be moved across instruction that may change that mode.
7382       [[fallthrough]];
7383     case fp::ExceptionBehavior::ebMayTrap:
7384       // These must not be moved across calls or instructions that may change
7385       // floating-point exception masks.
7386       PendingConstrainedFP.push_back(OutChain);
7387       break;
7388     case fp::ExceptionBehavior::ebStrict:
7389       // These must not be moved across calls or instructions that may change
7390       // floating-point exception masks or read floating-point exception flags.
7391       // In addition, they cannot be optimized out even if unused.
7392       PendingConstrainedFPStrict.push_back(OutChain);
7393       break;
7394     }
7395   };
7396 
7397   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7398   EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
7399   SDVTList VTs = DAG.getVTList(VT, MVT::Other);
7400   fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
7401 
7402   SDNodeFlags Flags;
7403   if (EB == fp::ExceptionBehavior::ebIgnore)
7404     Flags.setNoFPExcept(true);
7405 
7406   if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7407     Flags.copyFMF(*FPOp);
7408 
7409   unsigned Opcode;
7410   switch (FPI.getIntrinsicID()) {
7411   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
7412 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
7413   case Intrinsic::INTRINSIC:                                                   \
7414     Opcode = ISD::STRICT_##DAGN;                                               \
7415     break;
7416 #include "llvm/IR/ConstrainedOps.def"
7417   case Intrinsic::experimental_constrained_fmuladd: {
7418     Opcode = ISD::STRICT_FMA;
7419     // Break fmuladd into fmul and fadd.
7420     if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
7421         !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
7422       Opers.pop_back();
7423       SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
7424       pushOutChain(Mul, EB);
7425       Opcode = ISD::STRICT_FADD;
7426       Opers.clear();
7427       Opers.push_back(Mul.getValue(1));
7428       Opers.push_back(Mul.getValue(0));
7429       Opers.push_back(getValue(FPI.getArgOperand(2)));
7430     }
7431     break;
7432   }
7433   }
7434 
7435   // A few strict DAG nodes carry additional operands that are not
7436   // set up by the default code above.
7437   switch (Opcode) {
7438   default: break;
7439   case ISD::STRICT_FP_ROUND:
7440     Opers.push_back(
7441         DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
7442     break;
7443   case ISD::STRICT_FSETCC:
7444   case ISD::STRICT_FSETCCS: {
7445     auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
7446     ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
7447     if (TM.Options.NoNaNsFPMath)
7448       Condition = getFCmpCodeWithoutNaN(Condition);
7449     Opers.push_back(DAG.getCondCode(Condition));
7450     break;
7451   }
7452   }
7453 
7454   SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
7455   pushOutChain(Result, EB);
7456 
7457   SDValue FPResult = Result.getValue(0);
7458   setValue(&FPI, FPResult);
7459 }
7460 
7461 static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
7462   std::optional<unsigned> ResOPC;
7463   switch (VPIntrin.getIntrinsicID()) {
7464   case Intrinsic::vp_ctlz: {
7465     bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
7466     ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
7467     break;
7468   }
7469   case Intrinsic::vp_cttz: {
7470     bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
7471     ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
7472     break;
7473   }
7474 #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)                                    \
7475   case Intrinsic::VPID:                                                        \
7476     ResOPC = ISD::VPSD;                                                        \
7477     break;
7478 #include "llvm/IR/VPIntrinsics.def"
7479   }
7480 
7481   if (!ResOPC)
7482     llvm_unreachable(
7483         "Inconsistency: no SDNode available for this VPIntrinsic!");
7484 
7485   if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
7486       *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
7487     if (VPIntrin.getFastMathFlags().allowReassoc())
7488       return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
7489                                                 : ISD::VP_REDUCE_FMUL;
7490   }
7491 
7492   return *ResOPC;
7493 }
7494 
7495 void SelectionDAGBuilder::visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
7496                                       SmallVector<SDValue, 7> &OpValues) {
7497   SDLoc DL = getCurSDLoc();
7498   Value *PtrOperand = VPIntrin.getArgOperand(0);
7499   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7500   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7501   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7502   SDValue LD;
7503   bool AddToChain = true;
7504   // Do not serialize variable-length loads of constant memory with
7505   // anything.
7506   if (!Alignment)
7507     Alignment = DAG.getEVTAlign(VT);
7508   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7509   AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7510   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7511   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7512       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7513       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7514   LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
7515                      MMO, false /*IsExpanding */);
7516   if (AddToChain)
7517     PendingLoads.push_back(LD.getValue(1));
7518   setValue(&VPIntrin, LD);
7519 }
7520 
7521 void SelectionDAGBuilder::visitVPGather(const VPIntrinsic &VPIntrin, EVT VT,
7522                                         SmallVector<SDValue, 7> &OpValues) {
7523   SDLoc DL = getCurSDLoc();
7524   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7525   Value *PtrOperand = VPIntrin.getArgOperand(0);
7526   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7527   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7528   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7529   SDValue LD;
7530   if (!Alignment)
7531     Alignment = DAG.getEVTAlign(VT.getScalarType());
7532   unsigned AS =
7533     PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7534   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7535      MachinePointerInfo(AS), MachineMemOperand::MOLoad,
7536      MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7537   SDValue Base, Index, Scale;
7538   ISD::MemIndexType IndexType;
7539   bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7540                                     this, VPIntrin.getParent(),
7541                                     VT.getScalarStoreSize());
7542   if (!UniformBase) {
7543     Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7544     Index = getValue(PtrOperand);
7545     IndexType = ISD::SIGNED_SCALED;
7546     Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7547   }
7548   EVT IdxVT = Index.getValueType();
7549   EVT EltTy = IdxVT.getVectorElementType();
7550   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7551     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7552     Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7553   }
7554   LD = DAG.getGatherVP(
7555       DAG.getVTList(VT, MVT::Other), VT, DL,
7556       {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
7557       IndexType);
7558   PendingLoads.push_back(LD.getValue(1));
7559   setValue(&VPIntrin, LD);
7560 }
7561 
7562 void SelectionDAGBuilder::visitVPStore(const VPIntrinsic &VPIntrin,
7563                                        SmallVector<SDValue, 7> &OpValues) {
7564   SDLoc DL = getCurSDLoc();
7565   Value *PtrOperand = VPIntrin.getArgOperand(1);
7566   EVT VT = OpValues[0].getValueType();
7567   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7568   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7569   SDValue ST;
7570   if (!Alignment)
7571     Alignment = DAG.getEVTAlign(VT);
7572   SDValue Ptr = OpValues[1];
7573   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
7574   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7575       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7576       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7577   ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
7578                       OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
7579                       /* IsTruncating */ false, /*IsCompressing*/ false);
7580   DAG.setRoot(ST);
7581   setValue(&VPIntrin, ST);
7582 }
7583 
7584 void SelectionDAGBuilder::visitVPScatter(const VPIntrinsic &VPIntrin,
7585                                               SmallVector<SDValue, 7> &OpValues) {
7586   SDLoc DL = getCurSDLoc();
7587   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7588   Value *PtrOperand = VPIntrin.getArgOperand(1);
7589   EVT VT = OpValues[0].getValueType();
7590   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7591   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7592   SDValue ST;
7593   if (!Alignment)
7594     Alignment = DAG.getEVTAlign(VT.getScalarType());
7595   unsigned AS =
7596       PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7597   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7598       MachinePointerInfo(AS), MachineMemOperand::MOStore,
7599       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7600   SDValue Base, Index, Scale;
7601   ISD::MemIndexType IndexType;
7602   bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7603                                     this, VPIntrin.getParent(),
7604                                     VT.getScalarStoreSize());
7605   if (!UniformBase) {
7606     Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7607     Index = getValue(PtrOperand);
7608     IndexType = ISD::SIGNED_SCALED;
7609     Scale =
7610       DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7611   }
7612   EVT IdxVT = Index.getValueType();
7613   EVT EltTy = IdxVT.getVectorElementType();
7614   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7615     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7616     Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7617   }
7618   ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
7619                         {getMemoryRoot(), OpValues[0], Base, Index, Scale,
7620                          OpValues[2], OpValues[3]},
7621                         MMO, IndexType);
7622   DAG.setRoot(ST);
7623   setValue(&VPIntrin, ST);
7624 }
7625 
7626 void SelectionDAGBuilder::visitVPStridedLoad(
7627     const VPIntrinsic &VPIntrin, EVT VT, SmallVectorImpl<SDValue> &OpValues) {
7628   SDLoc DL = getCurSDLoc();
7629   Value *PtrOperand = VPIntrin.getArgOperand(0);
7630   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7631   if (!Alignment)
7632     Alignment = DAG.getEVTAlign(VT.getScalarType());
7633   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7634   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7635   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7636   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7637   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7638   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7639       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7640       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7641 
7642   SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
7643                                     OpValues[2], OpValues[3], MMO,
7644                                     false /*IsExpanding*/);
7645 
7646   if (AddToChain)
7647     PendingLoads.push_back(LD.getValue(1));
7648   setValue(&VPIntrin, LD);
7649 }
7650 
7651 void SelectionDAGBuilder::visitVPStridedStore(
7652     const VPIntrinsic &VPIntrin, SmallVectorImpl<SDValue> &OpValues) {
7653   SDLoc DL = getCurSDLoc();
7654   Value *PtrOperand = VPIntrin.getArgOperand(1);
7655   EVT VT = OpValues[0].getValueType();
7656   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7657   if (!Alignment)
7658     Alignment = DAG.getEVTAlign(VT.getScalarType());
7659   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7660   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7661       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7662       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7663 
7664   SDValue ST = DAG.getStridedStoreVP(
7665       getMemoryRoot(), DL, OpValues[0], OpValues[1],
7666       DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
7667       OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
7668       /*IsCompressing*/ false);
7669 
7670   DAG.setRoot(ST);
7671   setValue(&VPIntrin, ST);
7672 }
7673 
7674 void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
7675   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7676   SDLoc DL = getCurSDLoc();
7677 
7678   ISD::CondCode Condition;
7679   CmpInst::Predicate CondCode = VPIntrin.getPredicate();
7680   bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
7681   if (IsFP) {
7682     // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
7683     // flags, but calls that don't return floating-point types can't be
7684     // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
7685     Condition = getFCmpCondCode(CondCode);
7686     if (TM.Options.NoNaNsFPMath)
7687       Condition = getFCmpCodeWithoutNaN(Condition);
7688   } else {
7689     Condition = getICmpCondCode(CondCode);
7690   }
7691 
7692   SDValue Op1 = getValue(VPIntrin.getOperand(0));
7693   SDValue Op2 = getValue(VPIntrin.getOperand(1));
7694   // #2 is the condition code
7695   SDValue MaskOp = getValue(VPIntrin.getOperand(3));
7696   SDValue EVL = getValue(VPIntrin.getOperand(4));
7697   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7698   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7699          "Unexpected target EVL type");
7700   EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
7701 
7702   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7703                                                         VPIntrin.getType());
7704   setValue(&VPIntrin,
7705            DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
7706 }
7707 
7708 void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
7709     const VPIntrinsic &VPIntrin) {
7710   SDLoc DL = getCurSDLoc();
7711   unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
7712 
7713   auto IID = VPIntrin.getIntrinsicID();
7714 
7715   if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
7716     return visitVPCmp(*CmpI);
7717 
7718   SmallVector<EVT, 4> ValueVTs;
7719   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7720   ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
7721   SDVTList VTs = DAG.getVTList(ValueVTs);
7722 
7723   auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
7724 
7725   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7726   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7727          "Unexpected target EVL type");
7728 
7729   // Request operands.
7730   SmallVector<SDValue, 7> OpValues;
7731   for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
7732     auto Op = getValue(VPIntrin.getArgOperand(I));
7733     if (I == EVLParamPos)
7734       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
7735     OpValues.push_back(Op);
7736   }
7737 
7738   switch (Opcode) {
7739   default: {
7740     SDNodeFlags SDFlags;
7741     if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
7742       SDFlags.copyFMF(*FPMO);
7743     SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
7744     setValue(&VPIntrin, Result);
7745     break;
7746   }
7747   case ISD::VP_LOAD:
7748     visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
7749     break;
7750   case ISD::VP_GATHER:
7751     visitVPGather(VPIntrin, ValueVTs[0], OpValues);
7752     break;
7753   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7754     visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
7755     break;
7756   case ISD::VP_STORE:
7757     visitVPStore(VPIntrin, OpValues);
7758     break;
7759   case ISD::VP_SCATTER:
7760     visitVPScatter(VPIntrin, OpValues);
7761     break;
7762   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7763     visitVPStridedStore(VPIntrin, OpValues);
7764     break;
7765   case ISD::VP_FMULADD: {
7766     assert(OpValues.size() == 5 && "Unexpected number of operands");
7767     SDNodeFlags SDFlags;
7768     if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
7769       SDFlags.copyFMF(*FPMO);
7770     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
7771         TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) {
7772       setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
7773     } else {
7774       SDValue Mul = DAG.getNode(
7775           ISD::VP_FMUL, DL, VTs,
7776           {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
7777       SDValue Add =
7778           DAG.getNode(ISD::VP_FADD, DL, VTs,
7779                       {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
7780       setValue(&VPIntrin, Add);
7781     }
7782     break;
7783   }
7784   case ISD::VP_INTTOPTR: {
7785     SDValue N = OpValues[0];
7786     EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
7787     EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
7788     N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
7789                                OpValues[2]);
7790     N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
7791                              OpValues[2]);
7792     setValue(&VPIntrin, N);
7793     break;
7794   }
7795   case ISD::VP_PTRTOINT: {
7796     SDValue N = OpValues[0];
7797     EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7798                                                           VPIntrin.getType());
7799     EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
7800                                        VPIntrin.getOperand(0)->getType());
7801     N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
7802                                OpValues[2]);
7803     N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
7804                              OpValues[2]);
7805     setValue(&VPIntrin, N);
7806     break;
7807   }
7808   case ISD::VP_ABS:
7809   case ISD::VP_CTLZ:
7810   case ISD::VP_CTLZ_ZERO_UNDEF:
7811   case ISD::VP_CTTZ:
7812   case ISD::VP_CTTZ_ZERO_UNDEF: {
7813     SDValue Result =
7814         DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
7815     setValue(&VPIntrin, Result);
7816     break;
7817   }
7818   }
7819 }
7820 
7821 SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
7822                                           const BasicBlock *EHPadBB,
7823                                           MCSymbol *&BeginLabel) {
7824   MachineFunction &MF = DAG.getMachineFunction();
7825   MachineModuleInfo &MMI = MF.getMMI();
7826 
7827   // Insert a label before the invoke call to mark the try range.  This can be
7828   // used to detect deletion of the invoke via the MachineModuleInfo.
7829   BeginLabel = MMI.getContext().createTempSymbol();
7830 
7831   // For SjLj, keep track of which landing pads go with which invokes
7832   // so as to maintain the ordering of pads in the LSDA.
7833   unsigned CallSiteIndex = MMI.getCurrentCallSite();
7834   if (CallSiteIndex) {
7835     MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
7836     LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
7837 
7838     // Now that the call site is handled, stop tracking it.
7839     MMI.setCurrentCallSite(0);
7840   }
7841 
7842   return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
7843 }
7844 
7845 SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
7846                                         const BasicBlock *EHPadBB,
7847                                         MCSymbol *BeginLabel) {
7848   assert(BeginLabel && "BeginLabel should've been set");
7849 
7850   MachineFunction &MF = DAG.getMachineFunction();
7851   MachineModuleInfo &MMI = MF.getMMI();
7852 
7853   // Insert a label at the end of the invoke call to mark the try range.  This
7854   // can be used to detect deletion of the invoke via the MachineModuleInfo.
7855   MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
7856   Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
7857 
7858   // Inform MachineModuleInfo of range.
7859   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
7860   // There is a platform (e.g. wasm) that uses funclet style IR but does not
7861   // actually use outlined funclets and their LSDA info style.
7862   if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
7863     assert(II && "II should've been set");
7864     WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
7865     EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
7866   } else if (!isScopedEHPersonality(Pers)) {
7867     assert(EHPadBB);
7868     MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
7869   }
7870 
7871   return Chain;
7872 }
7873 
7874 std::pair<SDValue, SDValue>
7875 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
7876                                     const BasicBlock *EHPadBB) {
7877   MCSymbol *BeginLabel = nullptr;
7878 
7879   if (EHPadBB) {
7880     // Both PendingLoads and PendingExports must be flushed here;
7881     // this call might not return.
7882     (void)getRoot();
7883     DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
7884     CLI.setChain(getRoot());
7885   }
7886 
7887   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7888   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7889 
7890   assert((CLI.IsTailCall || Result.second.getNode()) &&
7891          "Non-null chain expected with non-tail call!");
7892   assert((Result.second.getNode() || !Result.first.getNode()) &&
7893          "Null value expected with tail call!");
7894 
7895   if (!Result.second.getNode()) {
7896     // As a special case, a null chain means that a tail call has been emitted
7897     // and the DAG root is already updated.
7898     HasTailCall = true;
7899 
7900     // Since there's no actual continuation from this block, nothing can be
7901     // relying on us setting vregs for them.
7902     PendingExports.clear();
7903   } else {
7904     DAG.setRoot(Result.second);
7905   }
7906 
7907   if (EHPadBB) {
7908     DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
7909                            BeginLabel));
7910   }
7911 
7912   return Result;
7913 }
7914 
7915 void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
7916                                       bool isTailCall,
7917                                       bool isMustTailCall,
7918                                       const BasicBlock *EHPadBB) {
7919   auto &DL = DAG.getDataLayout();
7920   FunctionType *FTy = CB.getFunctionType();
7921   Type *RetTy = CB.getType();
7922 
7923   TargetLowering::ArgListTy Args;
7924   Args.reserve(CB.arg_size());
7925 
7926   const Value *SwiftErrorVal = nullptr;
7927   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7928 
7929   if (isTailCall) {
7930     // Avoid emitting tail calls in functions with the disable-tail-calls
7931     // attribute.
7932     auto *Caller = CB.getParent()->getParent();
7933     if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
7934         "true" && !isMustTailCall)
7935       isTailCall = false;
7936 
7937     // We can't tail call inside a function with a swifterror argument. Lowering
7938     // does not support this yet. It would have to move into the swifterror
7939     // register before the call.
7940     if (TLI.supportSwiftError() &&
7941         Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
7942       isTailCall = false;
7943   }
7944 
7945   for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
7946     TargetLowering::ArgListEntry Entry;
7947     const Value *V = *I;
7948 
7949     // Skip empty types
7950     if (V->getType()->isEmptyTy())
7951       continue;
7952 
7953     SDValue ArgNode = getValue(V);
7954     Entry.Node = ArgNode; Entry.Ty = V->getType();
7955 
7956     Entry.setAttributes(&CB, I - CB.arg_begin());
7957 
7958     // Use swifterror virtual register as input to the call.
7959     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
7960       SwiftErrorVal = V;
7961       // We find the virtual register for the actual swifterror argument.
7962       // Instead of using the Value, we use the virtual register instead.
7963       Entry.Node =
7964           DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
7965                           EVT(TLI.getPointerTy(DL)));
7966     }
7967 
7968     Args.push_back(Entry);
7969 
7970     // If we have an explicit sret argument that is an Instruction, (i.e., it
7971     // might point to function-local memory), we can't meaningfully tail-call.
7972     if (Entry.IsSRet && isa<Instruction>(V))
7973       isTailCall = false;
7974   }
7975 
7976   // If call site has a cfguardtarget operand bundle, create and add an
7977   // additional ArgListEntry.
7978   if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
7979     TargetLowering::ArgListEntry Entry;
7980     Value *V = Bundle->Inputs[0];
7981     SDValue ArgNode = getValue(V);
7982     Entry.Node = ArgNode;
7983     Entry.Ty = V->getType();
7984     Entry.IsCFGuardTarget = true;
7985     Args.push_back(Entry);
7986   }
7987 
7988   // Check if target-independent constraints permit a tail call here.
7989   // Target-dependent constraints are checked within TLI->LowerCallTo.
7990   if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
7991     isTailCall = false;
7992 
7993   // Disable tail calls if there is an swifterror argument. Targets have not
7994   // been updated to support tail calls.
7995   if (TLI.supportSwiftError() && SwiftErrorVal)
7996     isTailCall = false;
7997 
7998   ConstantInt *CFIType = nullptr;
7999   if (CB.isIndirectCall()) {
8000     if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8001       if (!TLI.supportKCFIBundles())
8002         report_fatal_error(
8003             "Target doesn't support calls with kcfi operand bundles.");
8004       CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8005       assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8006     }
8007   }
8008 
8009   TargetLowering::CallLoweringInfo CLI(DAG);
8010   CLI.setDebugLoc(getCurSDLoc())
8011       .setChain(getRoot())
8012       .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8013       .setTailCall(isTailCall)
8014       .setConvergent(CB.isConvergent())
8015       .setIsPreallocated(
8016           CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
8017       .setCFIType(CFIType);
8018   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8019 
8020   if (Result.first.getNode()) {
8021     Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8022     setValue(&CB, Result.first);
8023   }
8024 
8025   // The last element of CLI.InVals has the SDValue for swifterror return.
8026   // Here we copy it to a virtual register and update SwiftErrorMap for
8027   // book-keeping.
8028   if (SwiftErrorVal && TLI.supportSwiftError()) {
8029     // Get the last element of InVals.
8030     SDValue Src = CLI.InVals.back();
8031     Register VReg =
8032         SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8033     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8034     DAG.setRoot(CopyNode);
8035   }
8036 }
8037 
8038 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8039                              SelectionDAGBuilder &Builder) {
8040   // Check to see if this load can be trivially constant folded, e.g. if the
8041   // input is from a string literal.
8042   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8043     // Cast pointer to the type we really want to load.
8044     Type *LoadTy =
8045         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8046     if (LoadVT.isVector())
8047       LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8048 
8049     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8050                                          PointerType::getUnqual(LoadTy));
8051 
8052     if (const Constant *LoadCst =
8053             ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8054                                          LoadTy, Builder.DAG.getDataLayout()))
8055       return Builder.getValue(LoadCst);
8056   }
8057 
8058   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
8059   // still constant memory, the input chain can be the entry node.
8060   SDValue Root;
8061   bool ConstantMemory = false;
8062 
8063   // Do not serialize (non-volatile) loads of constant memory with anything.
8064   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8065     Root = Builder.DAG.getEntryNode();
8066     ConstantMemory = true;
8067   } else {
8068     // Do not serialize non-volatile loads against each other.
8069     Root = Builder.DAG.getRoot();
8070   }
8071 
8072   SDValue Ptr = Builder.getValue(PtrVal);
8073   SDValue LoadVal =
8074       Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8075                           MachinePointerInfo(PtrVal), Align(1));
8076 
8077   if (!ConstantMemory)
8078     Builder.PendingLoads.push_back(LoadVal.getValue(1));
8079   return LoadVal;
8080 }
8081 
8082 /// Record the value for an instruction that produces an integer result,
8083 /// converting the type where necessary.
8084 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8085                                                   SDValue Value,
8086                                                   bool IsSigned) {
8087   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
8088                                                     I.getType(), true);
8089   if (IsSigned)
8090     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
8091   else
8092     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
8093   setValue(&I, Value);
8094 }
8095 
8096 /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8097 /// true and lower it. Otherwise return false, and it will be lowered like a
8098 /// normal call.
8099 /// The caller already checked that \p I calls the appropriate LibFunc with a
8100 /// correct prototype.
8101 bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8102   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8103   const Value *Size = I.getArgOperand(2);
8104   const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8105   if (CSize && CSize->getZExtValue() == 0) {
8106     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
8107                                                           I.getType(), true);
8108     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8109     return true;
8110   }
8111 
8112   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8113   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8114       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8115       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
8116   if (Res.first.getNode()) {
8117     processIntegerCallValue(I, Res.first, true);
8118     PendingLoads.push_back(Res.second);
8119     return true;
8120   }
8121 
8122   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
8123   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
8124   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8125     return false;
8126 
8127   // If the target has a fast compare for the given size, it will return a
8128   // preferred load type for that size. Require that the load VT is legal and
8129   // that the target supports unaligned loads of that type. Otherwise, return
8130   // INVALID.
8131   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8132     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8133     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8134     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8135       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8136       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8137       // TODO: Check alignment of src and dest ptrs.
8138       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8139       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8140       if (!TLI.isTypeLegal(LVT) ||
8141           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8142           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8143         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
8144     }
8145 
8146     return LVT;
8147   };
8148 
8149   // This turns into unaligned loads. We only do this if the target natively
8150   // supports the MVT we'll be loading or if it is small enough (<= 4) that
8151   // we'll only produce a small number of byte loads.
8152   MVT LoadVT;
8153   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8154   switch (NumBitsToCompare) {
8155   default:
8156     return false;
8157   case 16:
8158     LoadVT = MVT::i16;
8159     break;
8160   case 32:
8161     LoadVT = MVT::i32;
8162     break;
8163   case 64:
8164   case 128:
8165   case 256:
8166     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8167     break;
8168   }
8169 
8170   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
8171     return false;
8172 
8173   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
8174   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
8175 
8176   // Bitcast to a wide integer type if the loads are vectors.
8177   if (LoadVT.isVector()) {
8178     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
8179     LoadL = DAG.getBitcast(CmpVT, LoadL);
8180     LoadR = DAG.getBitcast(CmpVT, LoadR);
8181   }
8182 
8183   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
8184   processIntegerCallValue(I, Cmp, false);
8185   return true;
8186 }
8187 
8188 /// See if we can lower a memchr call into an optimized form. If so, return
8189 /// true and lower it. Otherwise return false, and it will be lowered like a
8190 /// normal call.
8191 /// The caller already checked that \p I calls the appropriate LibFunc with a
8192 /// correct prototype.
8193 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8194   const Value *Src = I.getArgOperand(0);
8195   const Value *Char = I.getArgOperand(1);
8196   const Value *Length = I.getArgOperand(2);
8197 
8198   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8199   std::pair<SDValue, SDValue> Res =
8200     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
8201                                 getValue(Src), getValue(Char), getValue(Length),
8202                                 MachinePointerInfo(Src));
8203   if (Res.first.getNode()) {
8204     setValue(&I, Res.first);
8205     PendingLoads.push_back(Res.second);
8206     return true;
8207   }
8208 
8209   return false;
8210 }
8211 
8212 /// See if we can lower a mempcpy call into an optimized form. If so, return
8213 /// true and lower it. Otherwise return false, and it will be lowered like a
8214 /// normal call.
8215 /// The caller already checked that \p I calls the appropriate LibFunc with a
8216 /// correct prototype.
8217 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8218   SDValue Dst = getValue(I.getArgOperand(0));
8219   SDValue Src = getValue(I.getArgOperand(1));
8220   SDValue Size = getValue(I.getArgOperand(2));
8221 
8222   Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8223   Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8224   // DAG::getMemcpy needs Alignment to be defined.
8225   Align Alignment = std::min(DstAlign, SrcAlign);
8226 
8227   bool isVol = false;
8228   SDLoc sdl = getCurSDLoc();
8229 
8230   // In the mempcpy context we need to pass in a false value for isTailCall
8231   // because the return pointer needs to be adjusted by the size of
8232   // the copied memory.
8233   SDValue Root = isVol ? getRoot() : getMemoryRoot();
8234   SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false,
8235                              /*isTailCall=*/false,
8236                              MachinePointerInfo(I.getArgOperand(0)),
8237                              MachinePointerInfo(I.getArgOperand(1)),
8238                              I.getAAMetadata());
8239   assert(MC.getNode() != nullptr &&
8240          "** memcpy should not be lowered as TailCall in mempcpy context **");
8241   DAG.setRoot(MC);
8242 
8243   // Check if Size needs to be truncated or extended.
8244   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8245 
8246   // Adjust return pointer to point just past the last dst byte.
8247   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8248                                     Dst, Size);
8249   setValue(&I, DstPlusSize);
8250   return true;
8251 }
8252 
8253 /// See if we can lower a strcpy call into an optimized form.  If so, return
8254 /// true and lower it, otherwise return false and it will be lowered like a
8255 /// normal call.
8256 /// The caller already checked that \p I calls the appropriate LibFunc with a
8257 /// correct prototype.
8258 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8259   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8260 
8261   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8262   std::pair<SDValue, SDValue> Res =
8263     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
8264                                 getValue(Arg0), getValue(Arg1),
8265                                 MachinePointerInfo(Arg0),
8266                                 MachinePointerInfo(Arg1), isStpcpy);
8267   if (Res.first.getNode()) {
8268     setValue(&I, Res.first);
8269     DAG.setRoot(Res.second);
8270     return true;
8271   }
8272 
8273   return false;
8274 }
8275 
8276 /// See if we can lower a strcmp call into an optimized form.  If so, return
8277 /// true and lower it, otherwise return false and it will be lowered like a
8278 /// normal call.
8279 /// The caller already checked that \p I calls the appropriate LibFunc with a
8280 /// correct prototype.
8281 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8282   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8283 
8284   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8285   std::pair<SDValue, SDValue> Res =
8286     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
8287                                 getValue(Arg0), getValue(Arg1),
8288                                 MachinePointerInfo(Arg0),
8289                                 MachinePointerInfo(Arg1));
8290   if (Res.first.getNode()) {
8291     processIntegerCallValue(I, Res.first, true);
8292     PendingLoads.push_back(Res.second);
8293     return true;
8294   }
8295 
8296   return false;
8297 }
8298 
8299 /// See if we can lower a strlen call into an optimized form.  If so, return
8300 /// true and lower it, otherwise return false and it will be lowered like a
8301 /// normal call.
8302 /// The caller already checked that \p I calls the appropriate LibFunc with a
8303 /// correct prototype.
8304 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8305   const Value *Arg0 = I.getArgOperand(0);
8306 
8307   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8308   std::pair<SDValue, SDValue> Res =
8309     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
8310                                 getValue(Arg0), MachinePointerInfo(Arg0));
8311   if (Res.first.getNode()) {
8312     processIntegerCallValue(I, Res.first, false);
8313     PendingLoads.push_back(Res.second);
8314     return true;
8315   }
8316 
8317   return false;
8318 }
8319 
8320 /// See if we can lower a strnlen call into an optimized form.  If so, return
8321 /// true and lower it, otherwise return false and it will be lowered like a
8322 /// normal call.
8323 /// The caller already checked that \p I calls the appropriate LibFunc with a
8324 /// correct prototype.
8325 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8326   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8327 
8328   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8329   std::pair<SDValue, SDValue> Res =
8330     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
8331                                  getValue(Arg0), getValue(Arg1),
8332                                  MachinePointerInfo(Arg0));
8333   if (Res.first.getNode()) {
8334     processIntegerCallValue(I, Res.first, false);
8335     PendingLoads.push_back(Res.second);
8336     return true;
8337   }
8338 
8339   return false;
8340 }
8341 
8342 /// See if we can lower a unary floating-point operation into an SDNode with
8343 /// the specified Opcode.  If so, return true and lower it, otherwise return
8344 /// false and it will be lowered like a normal call.
8345 /// The caller already checked that \p I calls the appropriate LibFunc with a
8346 /// correct prototype.
8347 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8348                                               unsigned Opcode) {
8349   // We already checked this call's prototype; verify it doesn't modify errno.
8350   if (!I.onlyReadsMemory())
8351     return false;
8352 
8353   SDNodeFlags Flags;
8354   Flags.copyFMF(cast<FPMathOperator>(I));
8355 
8356   SDValue Tmp = getValue(I.getArgOperand(0));
8357   setValue(&I,
8358            DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8359   return true;
8360 }
8361 
8362 /// See if we can lower a binary floating-point operation into an SDNode with
8363 /// the specified Opcode. If so, return true and lower it. Otherwise return
8364 /// false, and it will be lowered like a normal call.
8365 /// The caller already checked that \p I calls the appropriate LibFunc with a
8366 /// correct prototype.
8367 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8368                                                unsigned Opcode) {
8369   // We already checked this call's prototype; verify it doesn't modify errno.
8370   if (!I.onlyReadsMemory())
8371     return false;
8372 
8373   SDNodeFlags Flags;
8374   Flags.copyFMF(cast<FPMathOperator>(I));
8375 
8376   SDValue Tmp0 = getValue(I.getArgOperand(0));
8377   SDValue Tmp1 = getValue(I.getArgOperand(1));
8378   EVT VT = Tmp0.getValueType();
8379   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8380   return true;
8381 }
8382 
8383 void SelectionDAGBuilder::visitCall(const CallInst &I) {
8384   // Handle inline assembly differently.
8385   if (I.isInlineAsm()) {
8386     visitInlineAsm(I);
8387     return;
8388   }
8389 
8390   diagnoseDontCall(I);
8391 
8392   if (Function *F = I.getCalledFunction()) {
8393     if (F->isDeclaration()) {
8394       // Is this an LLVM intrinsic or a target-specific intrinsic?
8395       unsigned IID = F->getIntrinsicID();
8396       if (!IID)
8397         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8398           IID = II->getIntrinsicID(F);
8399 
8400       if (IID) {
8401         visitIntrinsicCall(I, IID);
8402         return;
8403       }
8404     }
8405 
8406     // Check for well-known libc/libm calls.  If the function is internal, it
8407     // can't be a library call.  Don't do the check if marked as nobuiltin for
8408     // some reason or the call site requires strict floating point semantics.
8409     LibFunc Func;
8410     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
8411         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
8412         LibInfo->hasOptimizedCodeGen(Func)) {
8413       switch (Func) {
8414       default: break;
8415       case LibFunc_bcmp:
8416         if (visitMemCmpBCmpCall(I))
8417           return;
8418         break;
8419       case LibFunc_copysign:
8420       case LibFunc_copysignf:
8421       case LibFunc_copysignl:
8422         // We already checked this call's prototype; verify it doesn't modify
8423         // errno.
8424         if (I.onlyReadsMemory()) {
8425           SDValue LHS = getValue(I.getArgOperand(0));
8426           SDValue RHS = getValue(I.getArgOperand(1));
8427           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
8428                                    LHS.getValueType(), LHS, RHS));
8429           return;
8430         }
8431         break;
8432       case LibFunc_fabs:
8433       case LibFunc_fabsf:
8434       case LibFunc_fabsl:
8435         if (visitUnaryFloatCall(I, ISD::FABS))
8436           return;
8437         break;
8438       case LibFunc_fmin:
8439       case LibFunc_fminf:
8440       case LibFunc_fminl:
8441         if (visitBinaryFloatCall(I, ISD::FMINNUM))
8442           return;
8443         break;
8444       case LibFunc_fmax:
8445       case LibFunc_fmaxf:
8446       case LibFunc_fmaxl:
8447         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
8448           return;
8449         break;
8450       case LibFunc_sin:
8451       case LibFunc_sinf:
8452       case LibFunc_sinl:
8453         if (visitUnaryFloatCall(I, ISD::FSIN))
8454           return;
8455         break;
8456       case LibFunc_cos:
8457       case LibFunc_cosf:
8458       case LibFunc_cosl:
8459         if (visitUnaryFloatCall(I, ISD::FCOS))
8460           return;
8461         break;
8462       case LibFunc_sqrt:
8463       case LibFunc_sqrtf:
8464       case LibFunc_sqrtl:
8465       case LibFunc_sqrt_finite:
8466       case LibFunc_sqrtf_finite:
8467       case LibFunc_sqrtl_finite:
8468         if (visitUnaryFloatCall(I, ISD::FSQRT))
8469           return;
8470         break;
8471       case LibFunc_floor:
8472       case LibFunc_floorf:
8473       case LibFunc_floorl:
8474         if (visitUnaryFloatCall(I, ISD::FFLOOR))
8475           return;
8476         break;
8477       case LibFunc_nearbyint:
8478       case LibFunc_nearbyintf:
8479       case LibFunc_nearbyintl:
8480         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
8481           return;
8482         break;
8483       case LibFunc_ceil:
8484       case LibFunc_ceilf:
8485       case LibFunc_ceill:
8486         if (visitUnaryFloatCall(I, ISD::FCEIL))
8487           return;
8488         break;
8489       case LibFunc_rint:
8490       case LibFunc_rintf:
8491       case LibFunc_rintl:
8492         if (visitUnaryFloatCall(I, ISD::FRINT))
8493           return;
8494         break;
8495       case LibFunc_round:
8496       case LibFunc_roundf:
8497       case LibFunc_roundl:
8498         if (visitUnaryFloatCall(I, ISD::FROUND))
8499           return;
8500         break;
8501       case LibFunc_trunc:
8502       case LibFunc_truncf:
8503       case LibFunc_truncl:
8504         if (visitUnaryFloatCall(I, ISD::FTRUNC))
8505           return;
8506         break;
8507       case LibFunc_log2:
8508       case LibFunc_log2f:
8509       case LibFunc_log2l:
8510         if (visitUnaryFloatCall(I, ISD::FLOG2))
8511           return;
8512         break;
8513       case LibFunc_exp2:
8514       case LibFunc_exp2f:
8515       case LibFunc_exp2l:
8516         if (visitUnaryFloatCall(I, ISD::FEXP2))
8517           return;
8518         break;
8519       case LibFunc_memcmp:
8520         if (visitMemCmpBCmpCall(I))
8521           return;
8522         break;
8523       case LibFunc_mempcpy:
8524         if (visitMemPCpyCall(I))
8525           return;
8526         break;
8527       case LibFunc_memchr:
8528         if (visitMemChrCall(I))
8529           return;
8530         break;
8531       case LibFunc_strcpy:
8532         if (visitStrCpyCall(I, false))
8533           return;
8534         break;
8535       case LibFunc_stpcpy:
8536         if (visitStrCpyCall(I, true))
8537           return;
8538         break;
8539       case LibFunc_strcmp:
8540         if (visitStrCmpCall(I))
8541           return;
8542         break;
8543       case LibFunc_strlen:
8544         if (visitStrLenCall(I))
8545           return;
8546         break;
8547       case LibFunc_strnlen:
8548         if (visitStrNLenCall(I))
8549           return;
8550         break;
8551       }
8552     }
8553   }
8554 
8555   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
8556   // have to do anything here to lower funclet bundles.
8557   // CFGuardTarget bundles are lowered in LowerCallTo.
8558   assert(!I.hasOperandBundlesOtherThan(
8559              {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
8560               LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
8561               LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) &&
8562          "Cannot lower calls with arbitrary operand bundles!");
8563 
8564   SDValue Callee = getValue(I.getCalledOperand());
8565 
8566   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
8567     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
8568   else
8569     // Check if we can potentially perform a tail call. More detailed checking
8570     // is be done within LowerCallTo, after more information about the call is
8571     // known.
8572     LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
8573 }
8574 
8575 namespace {
8576 
8577 /// AsmOperandInfo - This contains information for each constraint that we are
8578 /// lowering.
8579 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
8580 public:
8581   /// CallOperand - If this is the result output operand or a clobber
8582   /// this is null, otherwise it is the incoming operand to the CallInst.
8583   /// This gets modified as the asm is processed.
8584   SDValue CallOperand;
8585 
8586   /// AssignedRegs - If this is a register or register class operand, this
8587   /// contains the set of register corresponding to the operand.
8588   RegsForValue AssignedRegs;
8589 
8590   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
8591     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
8592   }
8593 
8594   /// Whether or not this operand accesses memory
8595   bool hasMemory(const TargetLowering &TLI) const {
8596     // Indirect operand accesses access memory.
8597     if (isIndirect)
8598       return true;
8599 
8600     for (const auto &Code : Codes)
8601       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
8602         return true;
8603 
8604     return false;
8605   }
8606 };
8607 
8608 
8609 } // end anonymous namespace
8610 
8611 /// Make sure that the output operand \p OpInfo and its corresponding input
8612 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
8613 /// out).
8614 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
8615                                SDISelAsmOperandInfo &MatchingOpInfo,
8616                                SelectionDAG &DAG) {
8617   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
8618     return;
8619 
8620   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
8621   const auto &TLI = DAG.getTargetLoweringInfo();
8622 
8623   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
8624       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
8625                                        OpInfo.ConstraintVT);
8626   std::pair<unsigned, const TargetRegisterClass *> InputRC =
8627       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
8628                                        MatchingOpInfo.ConstraintVT);
8629   if ((OpInfo.ConstraintVT.isInteger() !=
8630        MatchingOpInfo.ConstraintVT.isInteger()) ||
8631       (MatchRC.second != InputRC.second)) {
8632     // FIXME: error out in a more elegant fashion
8633     report_fatal_error("Unsupported asm: input constraint"
8634                        " with a matching output constraint of"
8635                        " incompatible type!");
8636   }
8637   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
8638 }
8639 
8640 /// Get a direct memory input to behave well as an indirect operand.
8641 /// This may introduce stores, hence the need for a \p Chain.
8642 /// \return The (possibly updated) chain.
8643 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
8644                                         SDISelAsmOperandInfo &OpInfo,
8645                                         SelectionDAG &DAG) {
8646   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8647 
8648   // If we don't have an indirect input, put it in the constpool if we can,
8649   // otherwise spill it to a stack slot.
8650   // TODO: This isn't quite right. We need to handle these according to
8651   // the addressing mode that the constraint wants. Also, this may take
8652   // an additional register for the computation and we don't want that
8653   // either.
8654 
8655   // If the operand is a float, integer, or vector constant, spill to a
8656   // constant pool entry to get its address.
8657   const Value *OpVal = OpInfo.CallOperandVal;
8658   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
8659       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
8660     OpInfo.CallOperand = DAG.getConstantPool(
8661         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
8662     return Chain;
8663   }
8664 
8665   // Otherwise, create a stack slot and emit a store to it before the asm.
8666   Type *Ty = OpVal->getType();
8667   auto &DL = DAG.getDataLayout();
8668   uint64_t TySize = DL.getTypeAllocSize(Ty);
8669   MachineFunction &MF = DAG.getMachineFunction();
8670   int SSFI = MF.getFrameInfo().CreateStackObject(
8671       TySize, DL.getPrefTypeAlign(Ty), false);
8672   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
8673   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
8674                             MachinePointerInfo::getFixedStack(MF, SSFI),
8675                             TLI.getMemValueType(DL, Ty));
8676   OpInfo.CallOperand = StackSlot;
8677 
8678   return Chain;
8679 }
8680 
8681 /// GetRegistersForValue - Assign registers (virtual or physical) for the
8682 /// specified operand.  We prefer to assign virtual registers, to allow the
8683 /// register allocator to handle the assignment process.  However, if the asm
8684 /// uses features that we can't model on machineinstrs, we have SDISel do the
8685 /// allocation.  This produces generally horrible, but correct, code.
8686 ///
8687 ///   OpInfo describes the operand
8688 ///   RefOpInfo describes the matching operand if any, the operand otherwise
8689 static std::optional<unsigned>
8690 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
8691                      SDISelAsmOperandInfo &OpInfo,
8692                      SDISelAsmOperandInfo &RefOpInfo) {
8693   LLVMContext &Context = *DAG.getContext();
8694   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8695 
8696   MachineFunction &MF = DAG.getMachineFunction();
8697   SmallVector<unsigned, 4> Regs;
8698   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8699 
8700   // No work to do for memory/address operands.
8701   if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8702       OpInfo.ConstraintType == TargetLowering::C_Address)
8703     return std::nullopt;
8704 
8705   // If this is a constraint for a single physreg, or a constraint for a
8706   // register class, find it.
8707   unsigned AssignedReg;
8708   const TargetRegisterClass *RC;
8709   std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
8710       &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
8711   // RC is unset only on failure. Return immediately.
8712   if (!RC)
8713     return std::nullopt;
8714 
8715   // Get the actual register value type.  This is important, because the user
8716   // may have asked for (e.g.) the AX register in i32 type.  We need to
8717   // remember that AX is actually i16 to get the right extension.
8718   const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
8719 
8720   if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
8721     // If this is an FP operand in an integer register (or visa versa), or more
8722     // generally if the operand value disagrees with the register class we plan
8723     // to stick it in, fix the operand type.
8724     //
8725     // If this is an input value, the bitcast to the new type is done now.
8726     // Bitcast for output value is done at the end of visitInlineAsm().
8727     if ((OpInfo.Type == InlineAsm::isOutput ||
8728          OpInfo.Type == InlineAsm::isInput) &&
8729         !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
8730       // Try to convert to the first EVT that the reg class contains.  If the
8731       // types are identical size, use a bitcast to convert (e.g. two differing
8732       // vector types).  Note: output bitcast is done at the end of
8733       // visitInlineAsm().
8734       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
8735         // Exclude indirect inputs while they are unsupported because the code
8736         // to perform the load is missing and thus OpInfo.CallOperand still
8737         // refers to the input address rather than the pointed-to value.
8738         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
8739           OpInfo.CallOperand =
8740               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
8741         OpInfo.ConstraintVT = RegVT;
8742         // If the operand is an FP value and we want it in integer registers,
8743         // use the corresponding integer type. This turns an f64 value into
8744         // i64, which can be passed with two i32 values on a 32-bit machine.
8745       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
8746         MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
8747         if (OpInfo.Type == InlineAsm::isInput)
8748           OpInfo.CallOperand =
8749               DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
8750         OpInfo.ConstraintVT = VT;
8751       }
8752     }
8753   }
8754 
8755   // No need to allocate a matching input constraint since the constraint it's
8756   // matching to has already been allocated.
8757   if (OpInfo.isMatchingInputConstraint())
8758     return std::nullopt;
8759 
8760   EVT ValueVT = OpInfo.ConstraintVT;
8761   if (OpInfo.ConstraintVT == MVT::Other)
8762     ValueVT = RegVT;
8763 
8764   // Initialize NumRegs.
8765   unsigned NumRegs = 1;
8766   if (OpInfo.ConstraintVT != MVT::Other)
8767     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
8768 
8769   // If this is a constraint for a specific physical register, like {r17},
8770   // assign it now.
8771 
8772   // If this associated to a specific register, initialize iterator to correct
8773   // place. If virtual, make sure we have enough registers
8774 
8775   // Initialize iterator if necessary
8776   TargetRegisterClass::iterator I = RC->begin();
8777   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8778 
8779   // Do not check for single registers.
8780   if (AssignedReg) {
8781     I = std::find(I, RC->end(), AssignedReg);
8782     if (I == RC->end()) {
8783       // RC does not contain the selected register, which indicates a
8784       // mismatch between the register and the required type/bitwidth.
8785       return {AssignedReg};
8786     }
8787   }
8788 
8789   for (; NumRegs; --NumRegs, ++I) {
8790     assert(I != RC->end() && "Ran out of registers to allocate!");
8791     Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
8792     Regs.push_back(R);
8793   }
8794 
8795   OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
8796   return std::nullopt;
8797 }
8798 
8799 static unsigned
8800 findMatchingInlineAsmOperand(unsigned OperandNo,
8801                              const std::vector<SDValue> &AsmNodeOperands) {
8802   // Scan until we find the definition we already emitted of this operand.
8803   unsigned CurOp = InlineAsm::Op_FirstOperand;
8804   for (; OperandNo; --OperandNo) {
8805     // Advance to the next operand.
8806     unsigned OpFlag =
8807         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8808     assert((InlineAsm::isRegDefKind(OpFlag) ||
8809             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
8810             InlineAsm::isMemKind(OpFlag)) &&
8811            "Skipped past definitions?");
8812     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
8813   }
8814   return CurOp;
8815 }
8816 
8817 namespace {
8818 
8819 class ExtraFlags {
8820   unsigned Flags = 0;
8821 
8822 public:
8823   explicit ExtraFlags(const CallBase &Call) {
8824     const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8825     if (IA->hasSideEffects())
8826       Flags |= InlineAsm::Extra_HasSideEffects;
8827     if (IA->isAlignStack())
8828       Flags |= InlineAsm::Extra_IsAlignStack;
8829     if (Call.isConvergent())
8830       Flags |= InlineAsm::Extra_IsConvergent;
8831     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
8832   }
8833 
8834   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
8835     // Ideally, we would only check against memory constraints.  However, the
8836     // meaning of an Other constraint can be target-specific and we can't easily
8837     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
8838     // for Other constraints as well.
8839     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8840         OpInfo.ConstraintType == TargetLowering::C_Other) {
8841       if (OpInfo.Type == InlineAsm::isInput)
8842         Flags |= InlineAsm::Extra_MayLoad;
8843       else if (OpInfo.Type == InlineAsm::isOutput)
8844         Flags |= InlineAsm::Extra_MayStore;
8845       else if (OpInfo.Type == InlineAsm::isClobber)
8846         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
8847     }
8848   }
8849 
8850   unsigned get() const { return Flags; }
8851 };
8852 
8853 } // end anonymous namespace
8854 
8855 static bool isFunction(SDValue Op) {
8856   if (Op && Op.getOpcode() == ISD::GlobalAddress) {
8857     if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
8858       auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
8859 
8860       // In normal "call dllimport func" instruction (non-inlineasm) it force
8861       // indirect access by specifing call opcode. And usually specially print
8862       // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
8863       // not do in this way now. (In fact, this is similar with "Data Access"
8864       // action). So here we ignore dllimport function.
8865       if (Fn && !Fn->hasDLLImportStorageClass())
8866         return true;
8867     }
8868   }
8869   return false;
8870 }
8871 
8872 /// visitInlineAsm - Handle a call to an InlineAsm object.
8873 void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
8874                                          const BasicBlock *EHPadBB) {
8875   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8876 
8877   /// ConstraintOperands - Information about all of the constraints.
8878   SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
8879 
8880   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8881   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
8882       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
8883 
8884   // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
8885   // AsmDialect, MayLoad, MayStore).
8886   bool HasSideEffect = IA->hasSideEffects();
8887   ExtraFlags ExtraInfo(Call);
8888 
8889   for (auto &T : TargetConstraints) {
8890     ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
8891     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
8892 
8893     if (OpInfo.CallOperandVal)
8894       OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
8895 
8896     if (!HasSideEffect)
8897       HasSideEffect = OpInfo.hasMemory(TLI);
8898 
8899     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
8900     // FIXME: Could we compute this on OpInfo rather than T?
8901 
8902     // Compute the constraint code and ConstraintType to use.
8903     TLI.ComputeConstraintToUse(T, SDValue());
8904 
8905     if (T.ConstraintType == TargetLowering::C_Immediate &&
8906         OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
8907       // We've delayed emitting a diagnostic like the "n" constraint because
8908       // inlining could cause an integer showing up.
8909       return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
8910                                           "' expects an integer constant "
8911                                           "expression");
8912 
8913     ExtraInfo.update(T);
8914   }
8915 
8916   // We won't need to flush pending loads if this asm doesn't touch
8917   // memory and is nonvolatile.
8918   SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
8919 
8920   bool EmitEHLabels = isa<InvokeInst>(Call);
8921   if (EmitEHLabels) {
8922     assert(EHPadBB && "InvokeInst must have an EHPadBB");
8923   }
8924   bool IsCallBr = isa<CallBrInst>(Call);
8925 
8926   if (IsCallBr || EmitEHLabels) {
8927     // If this is a callbr or invoke we need to flush pending exports since
8928     // inlineasm_br and invoke are terminators.
8929     // We need to do this before nodes are glued to the inlineasm_br node.
8930     Chain = getControlRoot();
8931   }
8932 
8933   MCSymbol *BeginLabel = nullptr;
8934   if (EmitEHLabels) {
8935     Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
8936   }
8937 
8938   int OpNo = -1;
8939   SmallVector<StringRef> AsmStrs;
8940   IA->collectAsmStrs(AsmStrs);
8941 
8942   // Second pass over the constraints: compute which constraint option to use.
8943   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8944     if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
8945       OpNo++;
8946 
8947     // If this is an output operand with a matching input operand, look up the
8948     // matching input. If their types mismatch, e.g. one is an integer, the
8949     // other is floating point, or their sizes are different, flag it as an
8950     // error.
8951     if (OpInfo.hasMatchingInput()) {
8952       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
8953       patchMatchingInput(OpInfo, Input, DAG);
8954     }
8955 
8956     // Compute the constraint code and ConstraintType to use.
8957     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
8958 
8959     if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
8960          OpInfo.Type == InlineAsm::isClobber) ||
8961         OpInfo.ConstraintType == TargetLowering::C_Address)
8962       continue;
8963 
8964     // In Linux PIC model, there are 4 cases about value/label addressing:
8965     //
8966     // 1: Function call or Label jmp inside the module.
8967     // 2: Data access (such as global variable, static variable) inside module.
8968     // 3: Function call or Label jmp outside the module.
8969     // 4: Data access (such as global variable) outside the module.
8970     //
8971     // Due to current llvm inline asm architecture designed to not "recognize"
8972     // the asm code, there are quite troubles for us to treat mem addressing
8973     // differently for same value/adress used in different instuctions.
8974     // For example, in pic model, call a func may in plt way or direclty
8975     // pc-related, but lea/mov a function adress may use got.
8976     //
8977     // Here we try to "recognize" function call for the case 1 and case 3 in
8978     // inline asm. And try to adjust the constraint for them.
8979     //
8980     // TODO: Due to current inline asm didn't encourage to jmp to the outsider
8981     // label, so here we don't handle jmp function label now, but we need to
8982     // enhance it (especilly in PIC model) if we meet meaningful requirements.
8983     if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
8984         TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
8985         TM.getCodeModel() != CodeModel::Large) {
8986       OpInfo.isIndirect = false;
8987       OpInfo.ConstraintType = TargetLowering::C_Address;
8988     }
8989 
8990     // If this is a memory input, and if the operand is not indirect, do what we
8991     // need to provide an address for the memory input.
8992     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
8993         !OpInfo.isIndirect) {
8994       assert((OpInfo.isMultipleAlternative ||
8995               (OpInfo.Type == InlineAsm::isInput)) &&
8996              "Can only indirectify direct input operands!");
8997 
8998       // Memory operands really want the address of the value.
8999       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9000 
9001       // There is no longer a Value* corresponding to this operand.
9002       OpInfo.CallOperandVal = nullptr;
9003 
9004       // It is now an indirect operand.
9005       OpInfo.isIndirect = true;
9006     }
9007 
9008   }
9009 
9010   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9011   std::vector<SDValue> AsmNodeOperands;
9012   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
9013   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9014       IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9015 
9016   // If we have a !srcloc metadata node associated with it, we want to attach
9017   // this to the ultimately generated inline asm machineinstr.  To do this, we
9018   // pass in the third operand as this (potentially null) inline asm MDNode.
9019   const MDNode *SrcLoc = Call.getMetadata("srcloc");
9020   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9021 
9022   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9023   // bits as operand 3.
9024   AsmNodeOperands.push_back(DAG.getTargetConstant(
9025       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9026 
9027   // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9028   // this, assign virtual and physical registers for inputs and otput.
9029   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9030     // Assign Registers.
9031     SDISelAsmOperandInfo &RefOpInfo =
9032         OpInfo.isMatchingInputConstraint()
9033             ? ConstraintOperands[OpInfo.getMatchedOperand()]
9034             : OpInfo;
9035     const auto RegError =
9036         getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9037     if (RegError) {
9038       const MachineFunction &MF = DAG.getMachineFunction();
9039       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
9040       const char *RegName = TRI.getName(*RegError);
9041       emitInlineAsmError(Call, "register '" + Twine(RegName) +
9042                                    "' allocated for constraint '" +
9043                                    Twine(OpInfo.ConstraintCode) +
9044                                    "' does not match required type");
9045       return;
9046     }
9047 
9048     auto DetectWriteToReservedRegister = [&]() {
9049       const MachineFunction &MF = DAG.getMachineFunction();
9050       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
9051       for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9052         if (Register::isPhysicalRegister(Reg) &&
9053             TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9054           const char *RegName = TRI.getName(Reg);
9055           emitInlineAsmError(Call, "write to reserved register '" +
9056                                        Twine(RegName) + "'");
9057           return true;
9058         }
9059       }
9060       return false;
9061     };
9062     assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9063             (OpInfo.Type == InlineAsm::isInput &&
9064              !OpInfo.isMatchingInputConstraint())) &&
9065            "Only address as input operand is allowed.");
9066 
9067     switch (OpInfo.Type) {
9068     case InlineAsm::isOutput:
9069       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9070         unsigned ConstraintID =
9071             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9072         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
9073                "Failed to convert memory constraint code to constraint id.");
9074 
9075         // Add information to the INLINEASM node to know about this output.
9076         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
9077         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
9078         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
9079                                                         MVT::i32));
9080         AsmNodeOperands.push_back(OpInfo.CallOperand);
9081       } else {
9082         // Otherwise, this outputs to a register (directly for C_Register /
9083         // C_RegisterClass, and a target-defined fashion for
9084         // C_Immediate/C_Other). Find a register that we can use.
9085         if (OpInfo.AssignedRegs.Regs.empty()) {
9086           emitInlineAsmError(
9087               Call, "couldn't allocate output register for constraint '" +
9088                         Twine(OpInfo.ConstraintCode) + "'");
9089           return;
9090         }
9091 
9092         if (DetectWriteToReservedRegister())
9093           return;
9094 
9095         // Add information to the INLINEASM node to know that this register is
9096         // set.
9097         OpInfo.AssignedRegs.AddInlineAsmOperands(
9098             OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
9099                                   : InlineAsm::Kind_RegDef,
9100             false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
9101       }
9102       break;
9103 
9104     case InlineAsm::isInput:
9105     case InlineAsm::isLabel: {
9106       SDValue InOperandVal = OpInfo.CallOperand;
9107 
9108       if (OpInfo.isMatchingInputConstraint()) {
9109         // If this is required to match an output register we have already set,
9110         // just use its register.
9111         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
9112                                                   AsmNodeOperands);
9113         unsigned OpFlag =
9114           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
9115         if (InlineAsm::isRegDefKind(OpFlag) ||
9116             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
9117           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
9118           if (OpInfo.isIndirect) {
9119             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
9120             emitInlineAsmError(Call, "inline asm not supported yet: "
9121                                      "don't know how to handle tied "
9122                                      "indirect register inputs");
9123             return;
9124           }
9125 
9126           SmallVector<unsigned, 4> Regs;
9127           MachineFunction &MF = DAG.getMachineFunction();
9128           MachineRegisterInfo &MRI = MF.getRegInfo();
9129           const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
9130           auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
9131           Register TiedReg = R->getReg();
9132           MVT RegVT = R->getSimpleValueType(0);
9133           const TargetRegisterClass *RC =
9134               TiedReg.isVirtual()     ? MRI.getRegClass(TiedReg)
9135               : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
9136                                       : TRI.getMinimalPhysRegClass(TiedReg);
9137           unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
9138           for (unsigned i = 0; i != NumRegs; ++i)
9139             Regs.push_back(MRI.createVirtualRegister(RC));
9140 
9141           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
9142 
9143           SDLoc dl = getCurSDLoc();
9144           // Use the produced MatchedRegs object to
9145           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call);
9146           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
9147                                            true, OpInfo.getMatchedOperand(), dl,
9148                                            DAG, AsmNodeOperands);
9149           break;
9150         }
9151 
9152         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
9153         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
9154                "Unexpected number of operands");
9155         // Add information to the INLINEASM node to know about this input.
9156         // See InlineAsm.h isUseOperandTiedToDef.
9157         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
9158         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
9159                                                     OpInfo.getMatchedOperand());
9160         AsmNodeOperands.push_back(DAG.getTargetConstant(
9161             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9162         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
9163         break;
9164       }
9165 
9166       // Treat indirect 'X' constraint as memory.
9167       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
9168           OpInfo.isIndirect)
9169         OpInfo.ConstraintType = TargetLowering::C_Memory;
9170 
9171       if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
9172           OpInfo.ConstraintType == TargetLowering::C_Other) {
9173         std::vector<SDValue> Ops;
9174         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
9175                                           Ops, DAG);
9176         if (Ops.empty()) {
9177           if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
9178             if (isa<ConstantSDNode>(InOperandVal)) {
9179               emitInlineAsmError(Call, "value out of range for constraint '" +
9180                                            Twine(OpInfo.ConstraintCode) + "'");
9181               return;
9182             }
9183 
9184           emitInlineAsmError(Call,
9185                              "invalid operand for inline asm constraint '" +
9186                                  Twine(OpInfo.ConstraintCode) + "'");
9187           return;
9188         }
9189 
9190         // Add information to the INLINEASM node to know about this input.
9191         unsigned ResOpType =
9192           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
9193         AsmNodeOperands.push_back(DAG.getTargetConstant(
9194             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9195         llvm::append_range(AsmNodeOperands, Ops);
9196         break;
9197       }
9198 
9199       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9200         assert((OpInfo.isIndirect ||
9201                 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
9202                "Operand must be indirect to be a mem!");
9203         assert(InOperandVal.getValueType() ==
9204                    TLI.getPointerTy(DAG.getDataLayout()) &&
9205                "Memory operands expect pointer values");
9206 
9207         unsigned ConstraintID =
9208             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9209         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
9210                "Failed to convert memory constraint code to constraint id.");
9211 
9212         // Add information to the INLINEASM node to know about this input.
9213         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
9214         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
9215         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9216                                                         getCurSDLoc(),
9217                                                         MVT::i32));
9218         AsmNodeOperands.push_back(InOperandVal);
9219         break;
9220       }
9221 
9222       if (OpInfo.ConstraintType == TargetLowering::C_Address) {
9223         assert(InOperandVal.getValueType() ==
9224                    TLI.getPointerTy(DAG.getDataLayout()) &&
9225                "Address operands expect pointer values");
9226 
9227         unsigned ConstraintID =
9228             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9229         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
9230                "Failed to convert memory constraint code to constraint id.");
9231 
9232         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
9233 
9234         SDValue AsmOp = InOperandVal;
9235         if (isFunction(InOperandVal)) {
9236           auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
9237           ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Func, 1);
9238           AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
9239                                              InOperandVal.getValueType(),
9240                                              GA->getOffset());
9241         }
9242 
9243         // Add information to the INLINEASM node to know about this input.
9244         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
9245 
9246         AsmNodeOperands.push_back(
9247             DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
9248 
9249         AsmNodeOperands.push_back(AsmOp);
9250         break;
9251       }
9252 
9253       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9254               OpInfo.ConstraintType == TargetLowering::C_Register) &&
9255              "Unknown constraint type!");
9256 
9257       // TODO: Support this.
9258       if (OpInfo.isIndirect) {
9259         emitInlineAsmError(
9260             Call, "Don't know how to handle indirect register inputs yet "
9261                   "for constraint '" +
9262                       Twine(OpInfo.ConstraintCode) + "'");
9263         return;
9264       }
9265 
9266       // Copy the input into the appropriate registers.
9267       if (OpInfo.AssignedRegs.Regs.empty()) {
9268         emitInlineAsmError(Call,
9269                            "couldn't allocate input reg for constraint '" +
9270                                Twine(OpInfo.ConstraintCode) + "'");
9271         return;
9272       }
9273 
9274       if (DetectWriteToReservedRegister())
9275         return;
9276 
9277       SDLoc dl = getCurSDLoc();
9278 
9279       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
9280                                         &Call);
9281 
9282       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
9283                                                dl, DAG, AsmNodeOperands);
9284       break;
9285     }
9286     case InlineAsm::isClobber:
9287       // Add the clobbered value to the operand list, so that the register
9288       // allocator is aware that the physreg got clobbered.
9289       if (!OpInfo.AssignedRegs.Regs.empty())
9290         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
9291                                                  false, 0, getCurSDLoc(), DAG,
9292                                                  AsmNodeOperands);
9293       break;
9294     }
9295   }
9296 
9297   // Finish up input operands.  Set the input chain and add the flag last.
9298   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9299   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
9300 
9301   unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9302   Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9303                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9304   Flag = Chain.getValue(1);
9305 
9306   // Do additional work to generate outputs.
9307 
9308   SmallVector<EVT, 1> ResultVTs;
9309   SmallVector<SDValue, 1> ResultValues;
9310   SmallVector<SDValue, 8> OutChains;
9311 
9312   llvm::Type *CallResultType = Call.getType();
9313   ArrayRef<Type *> ResultTypes;
9314   if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9315     ResultTypes = StructResult->elements();
9316   else if (!CallResultType->isVoidTy())
9317     ResultTypes = ArrayRef(CallResultType);
9318 
9319   auto CurResultType = ResultTypes.begin();
9320   auto handleRegAssign = [&](SDValue V) {
9321     assert(CurResultType != ResultTypes.end() && "Unexpected value");
9322     assert((*CurResultType)->isSized() && "Unexpected unsized type");
9323     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9324     ++CurResultType;
9325     // If the type of the inline asm call site return value is different but has
9326     // same size as the type of the asm output bitcast it.  One example of this
9327     // is for vectors with different width / number of elements.  This can
9328     // happen for register classes that can contain multiple different value
9329     // types.  The preg or vreg allocated may not have the same VT as was
9330     // expected.
9331     //
9332     // This can also happen for a return value that disagrees with the register
9333     // class it is put in, eg. a double in a general-purpose register on a
9334     // 32-bit machine.
9335     if (ResultVT != V.getValueType() &&
9336         ResultVT.getSizeInBits() == V.getValueSizeInBits())
9337       V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9338     else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9339              V.getValueType().isInteger()) {
9340       // If a result value was tied to an input value, the computed result
9341       // may have a wider width than the expected result.  Extract the
9342       // relevant portion.
9343       V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9344     }
9345     assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9346     ResultVTs.push_back(ResultVT);
9347     ResultValues.push_back(V);
9348   };
9349 
9350   // Deal with output operands.
9351   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9352     if (OpInfo.Type == InlineAsm::isOutput) {
9353       SDValue Val;
9354       // Skip trivial output operands.
9355       if (OpInfo.AssignedRegs.Regs.empty())
9356         continue;
9357 
9358       switch (OpInfo.ConstraintType) {
9359       case TargetLowering::C_Register:
9360       case TargetLowering::C_RegisterClass:
9361         Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9362                                                   Chain, &Flag, &Call);
9363         break;
9364       case TargetLowering::C_Immediate:
9365       case TargetLowering::C_Other:
9366         Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
9367                                               OpInfo, DAG);
9368         break;
9369       case TargetLowering::C_Memory:
9370         break; // Already handled.
9371       case TargetLowering::C_Address:
9372         break; // Silence warning.
9373       case TargetLowering::C_Unknown:
9374         assert(false && "Unexpected unknown constraint");
9375       }
9376 
9377       // Indirect output manifest as stores. Record output chains.
9378       if (OpInfo.isIndirect) {
9379         const Value *Ptr = OpInfo.CallOperandVal;
9380         assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9381         SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9382                                      MachinePointerInfo(Ptr));
9383         OutChains.push_back(Store);
9384       } else {
9385         // generate CopyFromRegs to associated registers.
9386         assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9387         if (Val.getOpcode() == ISD::MERGE_VALUES) {
9388           for (const SDValue &V : Val->op_values())
9389             handleRegAssign(V);
9390         } else
9391           handleRegAssign(Val);
9392       }
9393     }
9394   }
9395 
9396   // Set results.
9397   if (!ResultValues.empty()) {
9398     assert(CurResultType == ResultTypes.end() &&
9399            "Mismatch in number of ResultTypes");
9400     assert(ResultValues.size() == ResultTypes.size() &&
9401            "Mismatch in number of output operands in asm result");
9402 
9403     SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
9404                             DAG.getVTList(ResultVTs), ResultValues);
9405     setValue(&Call, V);
9406   }
9407 
9408   // Collect store chains.
9409   if (!OutChains.empty())
9410     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
9411 
9412   if (EmitEHLabels) {
9413     Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
9414   }
9415 
9416   // Only Update Root if inline assembly has a memory effect.
9417   if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
9418       EmitEHLabels)
9419     DAG.setRoot(Chain);
9420 }
9421 
9422 void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
9423                                              const Twine &Message) {
9424   LLVMContext &Ctx = *DAG.getContext();
9425   Ctx.emitError(&Call, Message);
9426 
9427   // Make sure we leave the DAG in a valid state
9428   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9429   SmallVector<EVT, 1> ValueVTs;
9430   ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
9431 
9432   if (ValueVTs.empty())
9433     return;
9434 
9435   SmallVector<SDValue, 1> Ops;
9436   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
9437     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
9438 
9439   setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
9440 }
9441 
9442 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
9443   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
9444                           MVT::Other, getRoot(),
9445                           getValue(I.getArgOperand(0)),
9446                           DAG.getSrcValue(I.getArgOperand(0))));
9447 }
9448 
9449 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
9450   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9451   const DataLayout &DL = DAG.getDataLayout();
9452   SDValue V = DAG.getVAArg(
9453       TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
9454       getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
9455       DL.getABITypeAlign(I.getType()).value());
9456   DAG.setRoot(V.getValue(1));
9457 
9458   if (I.getType()->isPointerTy())
9459     V = DAG.getPtrExtOrTrunc(
9460         V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
9461   setValue(&I, V);
9462 }
9463 
9464 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
9465   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
9466                           MVT::Other, getRoot(),
9467                           getValue(I.getArgOperand(0)),
9468                           DAG.getSrcValue(I.getArgOperand(0))));
9469 }
9470 
9471 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
9472   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
9473                           MVT::Other, getRoot(),
9474                           getValue(I.getArgOperand(0)),
9475                           getValue(I.getArgOperand(1)),
9476                           DAG.getSrcValue(I.getArgOperand(0)),
9477                           DAG.getSrcValue(I.getArgOperand(1))));
9478 }
9479 
9480 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
9481                                                     const Instruction &I,
9482                                                     SDValue Op) {
9483   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
9484   if (!Range)
9485     return Op;
9486 
9487   ConstantRange CR = getConstantRangeFromMetadata(*Range);
9488   if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
9489     return Op;
9490 
9491   APInt Lo = CR.getUnsignedMin();
9492   if (!Lo.isMinValue())
9493     return Op;
9494 
9495   APInt Hi = CR.getUnsignedMax();
9496   unsigned Bits = std::max(Hi.getActiveBits(),
9497                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
9498 
9499   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
9500 
9501   SDLoc SL = getCurSDLoc();
9502 
9503   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
9504                              DAG.getValueType(SmallVT));
9505   unsigned NumVals = Op.getNode()->getNumValues();
9506   if (NumVals == 1)
9507     return ZExt;
9508 
9509   SmallVector<SDValue, 4> Ops;
9510 
9511   Ops.push_back(ZExt);
9512   for (unsigned I = 1; I != NumVals; ++I)
9513     Ops.push_back(Op.getValue(I));
9514 
9515   return DAG.getMergeValues(Ops, SL);
9516 }
9517 
9518 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
9519 /// the call being lowered.
9520 ///
9521 /// This is a helper for lowering intrinsics that follow a target calling
9522 /// convention or require stack pointer adjustment. Only a subset of the
9523 /// intrinsic's operands need to participate in the calling convention.
9524 void SelectionDAGBuilder::populateCallLoweringInfo(
9525     TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
9526     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
9527     bool IsPatchPoint) {
9528   TargetLowering::ArgListTy Args;
9529   Args.reserve(NumArgs);
9530 
9531   // Populate the argument list.
9532   // Attributes for args start at offset 1, after the return attribute.
9533   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
9534        ArgI != ArgE; ++ArgI) {
9535     const Value *V = Call->getOperand(ArgI);
9536 
9537     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
9538 
9539     TargetLowering::ArgListEntry Entry;
9540     Entry.Node = getValue(V);
9541     Entry.Ty = V->getType();
9542     Entry.setAttributes(Call, ArgI);
9543     Args.push_back(Entry);
9544   }
9545 
9546   CLI.setDebugLoc(getCurSDLoc())
9547       .setChain(getRoot())
9548       .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
9549       .setDiscardResult(Call->use_empty())
9550       .setIsPatchPoint(IsPatchPoint)
9551       .setIsPreallocated(
9552           Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
9553 }
9554 
9555 /// Add a stack map intrinsic call's live variable operands to a stackmap
9556 /// or patchpoint target node's operand list.
9557 ///
9558 /// Constants are converted to TargetConstants purely as an optimization to
9559 /// avoid constant materialization and register allocation.
9560 ///
9561 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
9562 /// generate addess computation nodes, and so FinalizeISel can convert the
9563 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
9564 /// address materialization and register allocation, but may also be required
9565 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
9566 /// alloca in the entry block, then the runtime may assume that the alloca's
9567 /// StackMap location can be read immediately after compilation and that the
9568 /// location is valid at any point during execution (this is similar to the
9569 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
9570 /// only available in a register, then the runtime would need to trap when
9571 /// execution reaches the StackMap in order to read the alloca's location.
9572 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
9573                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
9574                                 SelectionDAGBuilder &Builder) {
9575   SelectionDAG &DAG = Builder.DAG;
9576   for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
9577     SDValue Op = Builder.getValue(Call.getArgOperand(I));
9578 
9579     // Things on the stack are pointer-typed, meaning that they are already
9580     // legal and can be emitted directly to target nodes.
9581     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
9582       Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
9583     } else {
9584       // Otherwise emit a target independent node to be legalised.
9585       Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
9586     }
9587   }
9588 }
9589 
9590 /// Lower llvm.experimental.stackmap.
9591 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
9592   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
9593   //                                  [live variables...])
9594 
9595   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
9596 
9597   SDValue Chain, InFlag, Callee;
9598   SmallVector<SDValue, 32> Ops;
9599 
9600   SDLoc DL = getCurSDLoc();
9601   Callee = getValue(CI.getCalledOperand());
9602 
9603   // The stackmap intrinsic only records the live variables (the arguments
9604   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
9605   // intrinsic, this won't be lowered to a function call. This means we don't
9606   // have to worry about calling conventions and target specific lowering code.
9607   // Instead we perform the call lowering right here.
9608   //
9609   // chain, flag = CALLSEQ_START(chain, 0, 0)
9610   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
9611   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
9612   //
9613   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
9614   InFlag = Chain.getValue(1);
9615 
9616   // Add the STACKMAP operands, starting with DAG house-keeping.
9617   Ops.push_back(Chain);
9618   Ops.push_back(InFlag);
9619 
9620   // Add the <id>, <numShadowBytes> operands.
9621   //
9622   // These do not require legalisation, and can be emitted directly to target
9623   // constant nodes.
9624   SDValue ID = getValue(CI.getArgOperand(0));
9625   assert(ID.getValueType() == MVT::i64);
9626   SDValue IDConst = DAG.getTargetConstant(
9627       cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType());
9628   Ops.push_back(IDConst);
9629 
9630   SDValue Shad = getValue(CI.getArgOperand(1));
9631   assert(Shad.getValueType() == MVT::i32);
9632   SDValue ShadConst = DAG.getTargetConstant(
9633       cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
9634   Ops.push_back(ShadConst);
9635 
9636   // Add the live variables.
9637   addStackMapLiveVars(CI, 2, DL, Ops, *this);
9638 
9639   // Create the STACKMAP node.
9640   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9641   Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
9642   InFlag = Chain.getValue(1);
9643 
9644   Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InFlag, DL);
9645 
9646   // Stackmaps don't generate values, so nothing goes into the NodeMap.
9647 
9648   // Set the root to the target-lowered call chain.
9649   DAG.setRoot(Chain);
9650 
9651   // Inform the Frame Information that we have a stackmap in this function.
9652   FuncInfo.MF->getFrameInfo().setHasStackMap();
9653 }
9654 
9655 /// Lower llvm.experimental.patchpoint directly to its target opcode.
9656 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
9657                                           const BasicBlock *EHPadBB) {
9658   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
9659   //                                                 i32 <numBytes>,
9660   //                                                 i8* <target>,
9661   //                                                 i32 <numArgs>,
9662   //                                                 [Args...],
9663   //                                                 [live variables...])
9664 
9665   CallingConv::ID CC = CB.getCallingConv();
9666   bool IsAnyRegCC = CC == CallingConv::AnyReg;
9667   bool HasDef = !CB.getType()->isVoidTy();
9668   SDLoc dl = getCurSDLoc();
9669   SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));
9670 
9671   // Handle immediate and symbolic callees.
9672   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
9673     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
9674                                    /*isTarget=*/true);
9675   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
9676     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
9677                                          SDLoc(SymbolicCallee),
9678                                          SymbolicCallee->getValueType(0));
9679 
9680   // Get the real number of arguments participating in the call <numArgs>
9681   SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
9682   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
9683 
9684   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
9685   // Intrinsics include all meta-operands up to but not including CC.
9686   unsigned NumMetaOpers = PatchPointOpers::CCPos;
9687   assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
9688          "Not enough arguments provided to the patchpoint intrinsic");
9689 
9690   // For AnyRegCC the arguments are lowered later on manually.
9691   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
9692   Type *ReturnTy =
9693       IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
9694 
9695   TargetLowering::CallLoweringInfo CLI(DAG);
9696   populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
9697                            ReturnTy, true);
9698   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9699 
9700   SDNode *CallEnd = Result.second.getNode();
9701   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
9702     CallEnd = CallEnd->getOperand(0).getNode();
9703 
9704   /// Get a call instruction from the call sequence chain.
9705   /// Tail calls are not allowed.
9706   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
9707          "Expected a callseq node.");
9708   SDNode *Call = CallEnd->getOperand(0).getNode();
9709   bool HasGlue = Call->getGluedNode();
9710 
9711   // Replace the target specific call node with the patchable intrinsic.
9712   SmallVector<SDValue, 8> Ops;
9713 
9714   // Push the chain.
9715   Ops.push_back(*(Call->op_begin()));
9716 
9717   // Optionally, push the glue (if any).
9718   if (HasGlue)
9719     Ops.push_back(*(Call->op_end() - 1));
9720 
9721   // Push the register mask info.
9722   if (HasGlue)
9723     Ops.push_back(*(Call->op_end() - 2));
9724   else
9725     Ops.push_back(*(Call->op_end() - 1));
9726 
9727   // Add the <id> and <numBytes> constants.
9728   SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
9729   Ops.push_back(DAG.getTargetConstant(
9730                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
9731   SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
9732   Ops.push_back(DAG.getTargetConstant(
9733                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
9734                   MVT::i32));
9735 
9736   // Add the callee.
9737   Ops.push_back(Callee);
9738 
9739   // Adjust <numArgs> to account for any arguments that have been passed on the
9740   // stack instead.
9741   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
9742   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
9743   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
9744   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
9745 
9746   // Add the calling convention
9747   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
9748 
9749   // Add the arguments we omitted previously. The register allocator should
9750   // place these in any free register.
9751   if (IsAnyRegCC)
9752     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
9753       Ops.push_back(getValue(CB.getArgOperand(i)));
9754 
9755   // Push the arguments from the call instruction.
9756   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
9757   Ops.append(Call->op_begin() + 2, e);
9758 
9759   // Push live variables for the stack map.
9760   addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
9761 
9762   SDVTList NodeTys;
9763   if (IsAnyRegCC && HasDef) {
9764     // Create the return types based on the intrinsic definition
9765     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9766     SmallVector<EVT, 3> ValueVTs;
9767     ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
9768     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
9769 
9770     // There is always a chain and a glue type at the end
9771     ValueVTs.push_back(MVT::Other);
9772     ValueVTs.push_back(MVT::Glue);
9773     NodeTys = DAG.getVTList(ValueVTs);
9774   } else
9775     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9776 
9777   // Replace the target specific call node with a PATCHPOINT node.
9778   SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
9779 
9780   // Update the NodeMap.
9781   if (HasDef) {
9782     if (IsAnyRegCC)
9783       setValue(&CB, SDValue(PPV.getNode(), 0));
9784     else
9785       setValue(&CB, Result.first);
9786   }
9787 
9788   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
9789   // call sequence. Furthermore the location of the chain and glue can change
9790   // when the AnyReg calling convention is used and the intrinsic returns a
9791   // value.
9792   if (IsAnyRegCC && HasDef) {
9793     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
9794     SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
9795     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
9796   } else
9797     DAG.ReplaceAllUsesWith(Call, PPV.getNode());
9798   DAG.DeleteNode(Call);
9799 
9800   // Inform the Frame Information that we have a patchpoint in this function.
9801   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
9802 }
9803 
9804 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
9805                                             unsigned Intrinsic) {
9806   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9807   SDValue Op1 = getValue(I.getArgOperand(0));
9808   SDValue Op2;
9809   if (I.arg_size() > 1)
9810     Op2 = getValue(I.getArgOperand(1));
9811   SDLoc dl = getCurSDLoc();
9812   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
9813   SDValue Res;
9814   SDNodeFlags SDFlags;
9815   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
9816     SDFlags.copyFMF(*FPMO);
9817 
9818   switch (Intrinsic) {
9819   case Intrinsic::vector_reduce_fadd:
9820     if (SDFlags.hasAllowReassociation())
9821       Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
9822                         DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
9823                         SDFlags);
9824     else
9825       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
9826     break;
9827   case Intrinsic::vector_reduce_fmul:
9828     if (SDFlags.hasAllowReassociation())
9829       Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
9830                         DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
9831                         SDFlags);
9832     else
9833       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
9834     break;
9835   case Intrinsic::vector_reduce_add:
9836     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
9837     break;
9838   case Intrinsic::vector_reduce_mul:
9839     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
9840     break;
9841   case Intrinsic::vector_reduce_and:
9842     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
9843     break;
9844   case Intrinsic::vector_reduce_or:
9845     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
9846     break;
9847   case Intrinsic::vector_reduce_xor:
9848     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
9849     break;
9850   case Intrinsic::vector_reduce_smax:
9851     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
9852     break;
9853   case Intrinsic::vector_reduce_smin:
9854     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
9855     break;
9856   case Intrinsic::vector_reduce_umax:
9857     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
9858     break;
9859   case Intrinsic::vector_reduce_umin:
9860     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
9861     break;
9862   case Intrinsic::vector_reduce_fmax:
9863     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
9864     break;
9865   case Intrinsic::vector_reduce_fmin:
9866     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
9867     break;
9868   default:
9869     llvm_unreachable("Unhandled vector reduce intrinsic");
9870   }
9871   setValue(&I, Res);
9872 }
9873 
9874 /// Returns an AttributeList representing the attributes applied to the return
9875 /// value of the given call.
9876 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
9877   SmallVector<Attribute::AttrKind, 2> Attrs;
9878   if (CLI.RetSExt)
9879     Attrs.push_back(Attribute::SExt);
9880   if (CLI.RetZExt)
9881     Attrs.push_back(Attribute::ZExt);
9882   if (CLI.IsInReg)
9883     Attrs.push_back(Attribute::InReg);
9884 
9885   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
9886                             Attrs);
9887 }
9888 
9889 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
9890 /// implementation, which just calls LowerCall.
9891 /// FIXME: When all targets are
9892 /// migrated to using LowerCall, this hook should be integrated into SDISel.
9893 std::pair<SDValue, SDValue>
9894 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
9895   // Handle the incoming return values from the call.
9896   CLI.Ins.clear();
9897   Type *OrigRetTy = CLI.RetTy;
9898   SmallVector<EVT, 4> RetTys;
9899   SmallVector<uint64_t, 4> Offsets;
9900   auto &DL = CLI.DAG.getDataLayout();
9901   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
9902 
9903   if (CLI.IsPostTypeLegalization) {
9904     // If we are lowering a libcall after legalization, split the return type.
9905     SmallVector<EVT, 4> OldRetTys;
9906     SmallVector<uint64_t, 4> OldOffsets;
9907     RetTys.swap(OldRetTys);
9908     Offsets.swap(OldOffsets);
9909 
9910     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
9911       EVT RetVT = OldRetTys[i];
9912       uint64_t Offset = OldOffsets[i];
9913       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
9914       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
9915       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
9916       RetTys.append(NumRegs, RegisterVT);
9917       for (unsigned j = 0; j != NumRegs; ++j)
9918         Offsets.push_back(Offset + j * RegisterVTByteSZ);
9919     }
9920   }
9921 
9922   SmallVector<ISD::OutputArg, 4> Outs;
9923   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
9924 
9925   bool CanLowerReturn =
9926       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
9927                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
9928 
9929   SDValue DemoteStackSlot;
9930   int DemoteStackIdx = -100;
9931   if (!CanLowerReturn) {
9932     // FIXME: equivalent assert?
9933     // assert(!CS.hasInAllocaArgument() &&
9934     //        "sret demotion is incompatible with inalloca");
9935     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
9936     Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
9937     MachineFunction &MF = CLI.DAG.getMachineFunction();
9938     DemoteStackIdx =
9939         MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
9940     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
9941                                               DL.getAllocaAddrSpace());
9942 
9943     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
9944     ArgListEntry Entry;
9945     Entry.Node = DemoteStackSlot;
9946     Entry.Ty = StackSlotPtrType;
9947     Entry.IsSExt = false;
9948     Entry.IsZExt = false;
9949     Entry.IsInReg = false;
9950     Entry.IsSRet = true;
9951     Entry.IsNest = false;
9952     Entry.IsByVal = false;
9953     Entry.IsByRef = false;
9954     Entry.IsReturned = false;
9955     Entry.IsSwiftSelf = false;
9956     Entry.IsSwiftAsync = false;
9957     Entry.IsSwiftError = false;
9958     Entry.IsCFGuardTarget = false;
9959     Entry.Alignment = Alignment;
9960     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
9961     CLI.NumFixedArgs += 1;
9962     CLI.getArgs()[0].IndirectType = CLI.RetTy;
9963     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
9964 
9965     // sret demotion isn't compatible with tail-calls, since the sret argument
9966     // points into the callers stack frame.
9967     CLI.IsTailCall = false;
9968   } else {
9969     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9970         CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
9971     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
9972       ISD::ArgFlagsTy Flags;
9973       if (NeedsRegBlock) {
9974         Flags.setInConsecutiveRegs();
9975         if (I == RetTys.size() - 1)
9976           Flags.setInConsecutiveRegsLast();
9977       }
9978       EVT VT = RetTys[I];
9979       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9980                                                      CLI.CallConv, VT);
9981       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9982                                                        CLI.CallConv, VT);
9983       for (unsigned i = 0; i != NumRegs; ++i) {
9984         ISD::InputArg MyFlags;
9985         MyFlags.Flags = Flags;
9986         MyFlags.VT = RegisterVT;
9987         MyFlags.ArgVT = VT;
9988         MyFlags.Used = CLI.IsReturnValueUsed;
9989         if (CLI.RetTy->isPointerTy()) {
9990           MyFlags.Flags.setPointer();
9991           MyFlags.Flags.setPointerAddrSpace(
9992               cast<PointerType>(CLI.RetTy)->getAddressSpace());
9993         }
9994         if (CLI.RetSExt)
9995           MyFlags.Flags.setSExt();
9996         if (CLI.RetZExt)
9997           MyFlags.Flags.setZExt();
9998         if (CLI.IsInReg)
9999           MyFlags.Flags.setInReg();
10000         CLI.Ins.push_back(MyFlags);
10001       }
10002     }
10003   }
10004 
10005   // We push in swifterror return as the last element of CLI.Ins.
10006   ArgListTy &Args = CLI.getArgs();
10007   if (supportSwiftError()) {
10008     for (const ArgListEntry &Arg : Args) {
10009       if (Arg.IsSwiftError) {
10010         ISD::InputArg MyFlags;
10011         MyFlags.VT = getPointerTy(DL);
10012         MyFlags.ArgVT = EVT(getPointerTy(DL));
10013         MyFlags.Flags.setSwiftError();
10014         CLI.Ins.push_back(MyFlags);
10015       }
10016     }
10017   }
10018 
10019   // Handle all of the outgoing arguments.
10020   CLI.Outs.clear();
10021   CLI.OutVals.clear();
10022   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10023     SmallVector<EVT, 4> ValueVTs;
10024     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10025     // FIXME: Split arguments if CLI.IsPostTypeLegalization
10026     Type *FinalType = Args[i].Ty;
10027     if (Args[i].IsByVal)
10028       FinalType = Args[i].IndirectType;
10029     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10030         FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10031     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10032          ++Value) {
10033       EVT VT = ValueVTs[Value];
10034       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10035       SDValue Op = SDValue(Args[i].Node.getNode(),
10036                            Args[i].Node.getResNo() + Value);
10037       ISD::ArgFlagsTy Flags;
10038 
10039       // Certain targets (such as MIPS), may have a different ABI alignment
10040       // for a type depending on the context. Give the target a chance to
10041       // specify the alignment it wants.
10042       const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10043       Flags.setOrigAlign(OriginalAlignment);
10044 
10045       if (Args[i].Ty->isPointerTy()) {
10046         Flags.setPointer();
10047         Flags.setPointerAddrSpace(
10048             cast<PointerType>(Args[i].Ty)->getAddressSpace());
10049       }
10050       if (Args[i].IsZExt)
10051         Flags.setZExt();
10052       if (Args[i].IsSExt)
10053         Flags.setSExt();
10054       if (Args[i].IsInReg) {
10055         // If we are using vectorcall calling convention, a structure that is
10056         // passed InReg - is surely an HVA
10057         if (CLI.CallConv == CallingConv::X86_VectorCall &&
10058             isa<StructType>(FinalType)) {
10059           // The first value of a structure is marked
10060           if (0 == Value)
10061             Flags.setHvaStart();
10062           Flags.setHva();
10063         }
10064         // Set InReg Flag
10065         Flags.setInReg();
10066       }
10067       if (Args[i].IsSRet)
10068         Flags.setSRet();
10069       if (Args[i].IsSwiftSelf)
10070         Flags.setSwiftSelf();
10071       if (Args[i].IsSwiftAsync)
10072         Flags.setSwiftAsync();
10073       if (Args[i].IsSwiftError)
10074         Flags.setSwiftError();
10075       if (Args[i].IsCFGuardTarget)
10076         Flags.setCFGuardTarget();
10077       if (Args[i].IsByVal)
10078         Flags.setByVal();
10079       if (Args[i].IsByRef)
10080         Flags.setByRef();
10081       if (Args[i].IsPreallocated) {
10082         Flags.setPreallocated();
10083         // Set the byval flag for CCAssignFn callbacks that don't know about
10084         // preallocated.  This way we can know how many bytes we should've
10085         // allocated and how many bytes a callee cleanup function will pop.  If
10086         // we port preallocated to more targets, we'll have to add custom
10087         // preallocated handling in the various CC lowering callbacks.
10088         Flags.setByVal();
10089       }
10090       if (Args[i].IsInAlloca) {
10091         Flags.setInAlloca();
10092         // Set the byval flag for CCAssignFn callbacks that don't know about
10093         // inalloca.  This way we can know how many bytes we should've allocated
10094         // and how many bytes a callee cleanup function will pop.  If we port
10095         // inalloca to more targets, we'll have to add custom inalloca handling
10096         // in the various CC lowering callbacks.
10097         Flags.setByVal();
10098       }
10099       Align MemAlign;
10100       if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
10101         unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
10102         Flags.setByValSize(FrameSize);
10103 
10104         // info is not there but there are cases it cannot get right.
10105         if (auto MA = Args[i].Alignment)
10106           MemAlign = *MA;
10107         else
10108           MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
10109       } else if (auto MA = Args[i].Alignment) {
10110         MemAlign = *MA;
10111       } else {
10112         MemAlign = OriginalAlignment;
10113       }
10114       Flags.setMemAlign(MemAlign);
10115       if (Args[i].IsNest)
10116         Flags.setNest();
10117       if (NeedsRegBlock)
10118         Flags.setInConsecutiveRegs();
10119 
10120       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
10121                                                  CLI.CallConv, VT);
10122       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10123                                                         CLI.CallConv, VT);
10124       SmallVector<SDValue, 4> Parts(NumParts);
10125       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
10126 
10127       if (Args[i].IsSExt)
10128         ExtendKind = ISD::SIGN_EXTEND;
10129       else if (Args[i].IsZExt)
10130         ExtendKind = ISD::ZERO_EXTEND;
10131 
10132       // Conservatively only handle 'returned' on non-vectors that can be lowered,
10133       // for now.
10134       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
10135           CanLowerReturn) {
10136         assert((CLI.RetTy == Args[i].Ty ||
10137                 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
10138                  CLI.RetTy->getPointerAddressSpace() ==
10139                      Args[i].Ty->getPointerAddressSpace())) &&
10140                RetTys.size() == NumValues && "unexpected use of 'returned'");
10141         // Before passing 'returned' to the target lowering code, ensure that
10142         // either the register MVT and the actual EVT are the same size or that
10143         // the return value and argument are extended in the same way; in these
10144         // cases it's safe to pass the argument register value unchanged as the
10145         // return register value (although it's at the target's option whether
10146         // to do so)
10147         // TODO: allow code generation to take advantage of partially preserved
10148         // registers rather than clobbering the entire register when the
10149         // parameter extension method is not compatible with the return
10150         // extension method
10151         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
10152             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
10153              CLI.RetZExt == Args[i].IsZExt))
10154           Flags.setReturned();
10155       }
10156 
10157       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
10158                      CLI.CallConv, ExtendKind);
10159 
10160       for (unsigned j = 0; j != NumParts; ++j) {
10161         // if it isn't first piece, alignment must be 1
10162         // For scalable vectors the scalable part is currently handled
10163         // by individual targets, so we just use the known minimum size here.
10164         ISD::OutputArg MyFlags(
10165             Flags, Parts[j].getValueType().getSimpleVT(), VT,
10166             i < CLI.NumFixedArgs, i,
10167             j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
10168         if (NumParts > 1 && j == 0)
10169           MyFlags.Flags.setSplit();
10170         else if (j != 0) {
10171           MyFlags.Flags.setOrigAlign(Align(1));
10172           if (j == NumParts - 1)
10173             MyFlags.Flags.setSplitEnd();
10174         }
10175 
10176         CLI.Outs.push_back(MyFlags);
10177         CLI.OutVals.push_back(Parts[j]);
10178       }
10179 
10180       if (NeedsRegBlock && Value == NumValues - 1)
10181         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
10182     }
10183   }
10184 
10185   SmallVector<SDValue, 4> InVals;
10186   CLI.Chain = LowerCall(CLI, InVals);
10187 
10188   // Update CLI.InVals to use outside of this function.
10189   CLI.InVals = InVals;
10190 
10191   // Verify that the target's LowerCall behaved as expected.
10192   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
10193          "LowerCall didn't return a valid chain!");
10194   assert((!CLI.IsTailCall || InVals.empty()) &&
10195          "LowerCall emitted a return value for a tail call!");
10196   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
10197          "LowerCall didn't emit the correct number of values!");
10198 
10199   // For a tail call, the return value is merely live-out and there aren't
10200   // any nodes in the DAG representing it. Return a special value to
10201   // indicate that a tail call has been emitted and no more Instructions
10202   // should be processed in the current block.
10203   if (CLI.IsTailCall) {
10204     CLI.DAG.setRoot(CLI.Chain);
10205     return std::make_pair(SDValue(), SDValue());
10206   }
10207 
10208 #ifndef NDEBUG
10209   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
10210     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
10211     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
10212            "LowerCall emitted a value with the wrong type!");
10213   }
10214 #endif
10215 
10216   SmallVector<SDValue, 4> ReturnValues;
10217   if (!CanLowerReturn) {
10218     // The instruction result is the result of loading from the
10219     // hidden sret parameter.
10220     SmallVector<EVT, 1> PVTs;
10221     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
10222 
10223     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
10224     assert(PVTs.size() == 1 && "Pointers should fit in one register");
10225     EVT PtrVT = PVTs[0];
10226 
10227     unsigned NumValues = RetTys.size();
10228     ReturnValues.resize(NumValues);
10229     SmallVector<SDValue, 4> Chains(NumValues);
10230 
10231     // An aggregate return value cannot wrap around the address space, so
10232     // offsets to its parts don't wrap either.
10233     SDNodeFlags Flags;
10234     Flags.setNoUnsignedWrap(true);
10235 
10236     MachineFunction &MF = CLI.DAG.getMachineFunction();
10237     Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
10238     for (unsigned i = 0; i < NumValues; ++i) {
10239       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
10240                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
10241                                                         PtrVT), Flags);
10242       SDValue L = CLI.DAG.getLoad(
10243           RetTys[i], CLI.DL, CLI.Chain, Add,
10244           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
10245                                             DemoteStackIdx, Offsets[i]),
10246           HiddenSRetAlign);
10247       ReturnValues[i] = L;
10248       Chains[i] = L.getValue(1);
10249     }
10250 
10251     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10252   } else {
10253     // Collect the legal value parts into potentially illegal values
10254     // that correspond to the original function's return values.
10255     std::optional<ISD::NodeType> AssertOp;
10256     if (CLI.RetSExt)
10257       AssertOp = ISD::AssertSext;
10258     else if (CLI.RetZExt)
10259       AssertOp = ISD::AssertZext;
10260     unsigned CurReg = 0;
10261     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10262       EVT VT = RetTys[I];
10263       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
10264                                                      CLI.CallConv, VT);
10265       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10266                                                        CLI.CallConv, VT);
10267 
10268       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
10269                                               NumRegs, RegisterVT, VT, nullptr,
10270                                               CLI.CallConv, AssertOp));
10271       CurReg += NumRegs;
10272     }
10273 
10274     // For a function returning void, there is no return value. We can't create
10275     // such a node, so we just return a null return value in that case. In
10276     // that case, nothing will actually look at the value.
10277     if (ReturnValues.empty())
10278       return std::make_pair(SDValue(), CLI.Chain);
10279   }
10280 
10281   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10282                                 CLI.DAG.getVTList(RetTys), ReturnValues);
10283   return std::make_pair(Res, CLI.Chain);
10284 }
10285 
10286 /// Places new result values for the node in Results (their number
10287 /// and types must exactly match those of the original return values of
10288 /// the node), or leaves Results empty, which indicates that the node is not
10289 /// to be custom lowered after all.
10290 void TargetLowering::LowerOperationWrapper(SDNode *N,
10291                                            SmallVectorImpl<SDValue> &Results,
10292                                            SelectionDAG &DAG) const {
10293   SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10294 
10295   if (!Res.getNode())
10296     return;
10297 
10298   // If the original node has one result, take the return value from
10299   // LowerOperation as is. It might not be result number 0.
10300   if (N->getNumValues() == 1) {
10301     Results.push_back(Res);
10302     return;
10303   }
10304 
10305   // If the original node has multiple results, then the return node should
10306   // have the same number of results.
10307   assert((N->getNumValues() == Res->getNumValues()) &&
10308       "Lowering returned the wrong number of results!");
10309 
10310   // Places new result values base on N result number.
10311   for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10312     Results.push_back(Res.getValue(I));
10313 }
10314 
10315 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10316   llvm_unreachable("LowerOperation not implemented for this target!");
10317 }
10318 
10319 void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
10320                                                      unsigned Reg,
10321                                                      ISD::NodeType ExtendType) {
10322   SDValue Op = getNonRegisterValue(V);
10323   assert((Op.getOpcode() != ISD::CopyFromReg ||
10324           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10325          "Copy from a reg to the same reg!");
10326   assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10327 
10328   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10329   // If this is an InlineAsm we have to match the registers required, not the
10330   // notional registers required by the type.
10331 
10332   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10333                    std::nullopt); // This is not an ABI copy.
10334   SDValue Chain = DAG.getEntryNode();
10335 
10336   if (ExtendType == ISD::ANY_EXTEND) {
10337     auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10338     if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10339       ExtendType = PreferredExtendIt->second;
10340   }
10341   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10342   PendingExports.push_back(Chain);
10343 }
10344 
10345 #include "llvm/CodeGen/SelectionDAGISel.h"
10346 
10347 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10348 /// entry block, return true.  This includes arguments used by switches, since
10349 /// the switch may expand into multiple basic blocks.
10350 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10351   // With FastISel active, we may be splitting blocks, so force creation
10352   // of virtual registers for all non-dead arguments.
10353   if (FastISel)
10354     return A->use_empty();
10355 
10356   const BasicBlock &Entry = A->getParent()->front();
10357   for (const User *U : A->users())
10358     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10359       return false;  // Use not in entry block.
10360 
10361   return true;
10362 }
10363 
10364 using ArgCopyElisionMapTy =
10365     DenseMap<const Argument *,
10366              std::pair<const AllocaInst *, const StoreInst *>>;
10367 
10368 /// Scan the entry block of the function in FuncInfo for arguments that look
10369 /// like copies into a local alloca. Record any copied arguments in
10370 /// ArgCopyElisionCandidates.
10371 static void
10372 findArgumentCopyElisionCandidates(const DataLayout &DL,
10373                                   FunctionLoweringInfo *FuncInfo,
10374                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10375   // Record the state of every static alloca used in the entry block. Argument
10376   // allocas are all used in the entry block, so we need approximately as many
10377   // entries as we have arguments.
10378   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10379   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
10380   unsigned NumArgs = FuncInfo->Fn->arg_size();
10381   StaticAllocas.reserve(NumArgs * 2);
10382 
10383   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10384     if (!V)
10385       return nullptr;
10386     V = V->stripPointerCasts();
10387     const auto *AI = dyn_cast<AllocaInst>(V);
10388     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10389       return nullptr;
10390     auto Iter = StaticAllocas.insert({AI, Unknown});
10391     return &Iter.first->second;
10392   };
10393 
10394   // Look for stores of arguments to static allocas. Look through bitcasts and
10395   // GEPs to handle type coercions, as long as the alloca is fully initialized
10396   // by the store. Any non-store use of an alloca escapes it and any subsequent
10397   // unanalyzed store might write it.
10398   // FIXME: Handle structs initialized with multiple stores.
10399   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
10400     // Look for stores, and handle non-store uses conservatively.
10401     const auto *SI = dyn_cast<StoreInst>(&I);
10402     if (!SI) {
10403       // We will look through cast uses, so ignore them completely.
10404       if (I.isCast())
10405         continue;
10406       // Ignore debug info and pseudo op intrinsics, they don't escape or store
10407       // to allocas.
10408       if (I.isDebugOrPseudoInst())
10409         continue;
10410       // This is an unknown instruction. Assume it escapes or writes to all
10411       // static alloca operands.
10412       for (const Use &U : I.operands()) {
10413         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
10414           *Info = StaticAllocaInfo::Clobbered;
10415       }
10416       continue;
10417     }
10418 
10419     // If the stored value is a static alloca, mark it as escaped.
10420     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
10421       *Info = StaticAllocaInfo::Clobbered;
10422 
10423     // Check if the destination is a static alloca.
10424     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
10425     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
10426     if (!Info)
10427       continue;
10428     const AllocaInst *AI = cast<AllocaInst>(Dst);
10429 
10430     // Skip allocas that have been initialized or clobbered.
10431     if (*Info != StaticAllocaInfo::Unknown)
10432       continue;
10433 
10434     // Check if the stored value is an argument, and that this store fully
10435     // initializes the alloca.
10436     // If the argument type has padding bits we can't directly forward a pointer
10437     // as the upper bits may contain garbage.
10438     // Don't elide copies from the same argument twice.
10439     const Value *Val = SI->getValueOperand()->stripPointerCasts();
10440     const auto *Arg = dyn_cast<Argument>(Val);
10441     if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
10442         Arg->getType()->isEmptyTy() ||
10443         DL.getTypeStoreSize(Arg->getType()) !=
10444             DL.getTypeAllocSize(AI->getAllocatedType()) ||
10445         !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
10446         ArgCopyElisionCandidates.count(Arg)) {
10447       *Info = StaticAllocaInfo::Clobbered;
10448       continue;
10449     }
10450 
10451     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
10452                       << '\n');
10453 
10454     // Mark this alloca and store for argument copy elision.
10455     *Info = StaticAllocaInfo::Elidable;
10456     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
10457 
10458     // Stop scanning if we've seen all arguments. This will happen early in -O0
10459     // builds, which is useful, because -O0 builds have large entry blocks and
10460     // many allocas.
10461     if (ArgCopyElisionCandidates.size() == NumArgs)
10462       break;
10463   }
10464 }
10465 
10466 /// Try to elide argument copies from memory into a local alloca. Succeeds if
10467 /// ArgVal is a load from a suitable fixed stack object.
10468 static void tryToElideArgumentCopy(
10469     FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
10470     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
10471     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
10472     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
10473     SDValue ArgVal, bool &ArgHasUses) {
10474   // Check if this is a load from a fixed stack object.
10475   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
10476   if (!LNode)
10477     return;
10478   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
10479   if (!FINode)
10480     return;
10481 
10482   // Check that the fixed stack object is the right size and alignment.
10483   // Look at the alignment that the user wrote on the alloca instead of looking
10484   // at the stack object.
10485   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
10486   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
10487   const AllocaInst *AI = ArgCopyIter->second.first;
10488   int FixedIndex = FINode->getIndex();
10489   int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
10490   int OldIndex = AllocaIndex;
10491   MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
10492   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
10493     LLVM_DEBUG(
10494         dbgs() << "  argument copy elision failed due to bad fixed stack "
10495                   "object size\n");
10496     return;
10497   }
10498   Align RequiredAlignment = AI->getAlign();
10499   if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
10500     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
10501                          "greater than stack argument alignment ("
10502                       << DebugStr(RequiredAlignment) << " vs "
10503                       << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
10504     return;
10505   }
10506 
10507   // Perform the elision. Delete the old stack object and replace its only use
10508   // in the variable info map. Mark the stack object as mutable.
10509   LLVM_DEBUG({
10510     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
10511            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
10512            << '\n';
10513   });
10514   MFI.RemoveStackObject(OldIndex);
10515   MFI.setIsImmutableObjectIndex(FixedIndex, false);
10516   AllocaIndex = FixedIndex;
10517   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
10518   Chains.push_back(ArgVal.getValue(1));
10519 
10520   // Avoid emitting code for the store implementing the copy.
10521   const StoreInst *SI = ArgCopyIter->second.second;
10522   ElidedArgCopyInstrs.insert(SI);
10523 
10524   // Check for uses of the argument again so that we can avoid exporting ArgVal
10525   // if it is't used by anything other than the store.
10526   for (const Value *U : Arg.users()) {
10527     if (U != SI) {
10528       ArgHasUses = true;
10529       break;
10530     }
10531   }
10532 }
10533 
10534 void SelectionDAGISel::LowerArguments(const Function &F) {
10535   SelectionDAG &DAG = SDB->DAG;
10536   SDLoc dl = SDB->getCurSDLoc();
10537   const DataLayout &DL = DAG.getDataLayout();
10538   SmallVector<ISD::InputArg, 16> Ins;
10539 
10540   // In Naked functions we aren't going to save any registers.
10541   if (F.hasFnAttribute(Attribute::Naked))
10542     return;
10543 
10544   if (!FuncInfo->CanLowerReturn) {
10545     // Put in an sret pointer parameter before all the other parameters.
10546     SmallVector<EVT, 1> ValueVTs;
10547     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10548                     F.getReturnType()->getPointerTo(
10549                         DAG.getDataLayout().getAllocaAddrSpace()),
10550                     ValueVTs);
10551 
10552     // NOTE: Assuming that a pointer will never break down to more than one VT
10553     // or one register.
10554     ISD::ArgFlagsTy Flags;
10555     Flags.setSRet();
10556     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
10557     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
10558                          ISD::InputArg::NoArgIndex, 0);
10559     Ins.push_back(RetArg);
10560   }
10561 
10562   // Look for stores of arguments to static allocas. Mark such arguments with a
10563   // flag to ask the target to give us the memory location of that argument if
10564   // available.
10565   ArgCopyElisionMapTy ArgCopyElisionCandidates;
10566   findArgumentCopyElisionCandidates(DL, FuncInfo.get(),
10567                                     ArgCopyElisionCandidates);
10568 
10569   // Set up the incoming argument description vector.
10570   for (const Argument &Arg : F.args()) {
10571     unsigned ArgNo = Arg.getArgNo();
10572     SmallVector<EVT, 4> ValueVTs;
10573     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10574     bool isArgValueUsed = !Arg.use_empty();
10575     unsigned PartBase = 0;
10576     Type *FinalType = Arg.getType();
10577     if (Arg.hasAttribute(Attribute::ByVal))
10578       FinalType = Arg.getParamByValType();
10579     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
10580         FinalType, F.getCallingConv(), F.isVarArg(), DL);
10581     for (unsigned Value = 0, NumValues = ValueVTs.size();
10582          Value != NumValues; ++Value) {
10583       EVT VT = ValueVTs[Value];
10584       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
10585       ISD::ArgFlagsTy Flags;
10586 
10587 
10588       if (Arg.getType()->isPointerTy()) {
10589         Flags.setPointer();
10590         Flags.setPointerAddrSpace(
10591             cast<PointerType>(Arg.getType())->getAddressSpace());
10592       }
10593       if (Arg.hasAttribute(Attribute::ZExt))
10594         Flags.setZExt();
10595       if (Arg.hasAttribute(Attribute::SExt))
10596         Flags.setSExt();
10597       if (Arg.hasAttribute(Attribute::InReg)) {
10598         // If we are using vectorcall calling convention, a structure that is
10599         // passed InReg - is surely an HVA
10600         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
10601             isa<StructType>(Arg.getType())) {
10602           // The first value of a structure is marked
10603           if (0 == Value)
10604             Flags.setHvaStart();
10605           Flags.setHva();
10606         }
10607         // Set InReg Flag
10608         Flags.setInReg();
10609       }
10610       if (Arg.hasAttribute(Attribute::StructRet))
10611         Flags.setSRet();
10612       if (Arg.hasAttribute(Attribute::SwiftSelf))
10613         Flags.setSwiftSelf();
10614       if (Arg.hasAttribute(Attribute::SwiftAsync))
10615         Flags.setSwiftAsync();
10616       if (Arg.hasAttribute(Attribute::SwiftError))
10617         Flags.setSwiftError();
10618       if (Arg.hasAttribute(Attribute::ByVal))
10619         Flags.setByVal();
10620       if (Arg.hasAttribute(Attribute::ByRef))
10621         Flags.setByRef();
10622       if (Arg.hasAttribute(Attribute::InAlloca)) {
10623         Flags.setInAlloca();
10624         // Set the byval flag for CCAssignFn callbacks that don't know about
10625         // inalloca.  This way we can know how many bytes we should've allocated
10626         // and how many bytes a callee cleanup function will pop.  If we port
10627         // inalloca to more targets, we'll have to add custom inalloca handling
10628         // in the various CC lowering callbacks.
10629         Flags.setByVal();
10630       }
10631       if (Arg.hasAttribute(Attribute::Preallocated)) {
10632         Flags.setPreallocated();
10633         // Set the byval flag for CCAssignFn callbacks that don't know about
10634         // preallocated.  This way we can know how many bytes we should've
10635         // allocated and how many bytes a callee cleanup function will pop.  If
10636         // we port preallocated to more targets, we'll have to add custom
10637         // preallocated handling in the various CC lowering callbacks.
10638         Flags.setByVal();
10639       }
10640 
10641       // Certain targets (such as MIPS), may have a different ABI alignment
10642       // for a type depending on the context. Give the target a chance to
10643       // specify the alignment it wants.
10644       const Align OriginalAlignment(
10645           TLI->getABIAlignmentForCallingConv(ArgTy, DL));
10646       Flags.setOrigAlign(OriginalAlignment);
10647 
10648       Align MemAlign;
10649       Type *ArgMemTy = nullptr;
10650       if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
10651           Flags.isByRef()) {
10652         if (!ArgMemTy)
10653           ArgMemTy = Arg.getPointeeInMemoryValueType();
10654 
10655         uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
10656 
10657         // For in-memory arguments, size and alignment should be passed from FE.
10658         // BE will guess if this info is not there but there are cases it cannot
10659         // get right.
10660         if (auto ParamAlign = Arg.getParamStackAlign())
10661           MemAlign = *ParamAlign;
10662         else if ((ParamAlign = Arg.getParamAlign()))
10663           MemAlign = *ParamAlign;
10664         else
10665           MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
10666         if (Flags.isByRef())
10667           Flags.setByRefSize(MemSize);
10668         else
10669           Flags.setByValSize(MemSize);
10670       } else if (auto ParamAlign = Arg.getParamStackAlign()) {
10671         MemAlign = *ParamAlign;
10672       } else {
10673         MemAlign = OriginalAlignment;
10674       }
10675       Flags.setMemAlign(MemAlign);
10676 
10677       if (Arg.hasAttribute(Attribute::Nest))
10678         Flags.setNest();
10679       if (NeedsRegBlock)
10680         Flags.setInConsecutiveRegs();
10681       if (ArgCopyElisionCandidates.count(&Arg))
10682         Flags.setCopyElisionCandidate();
10683       if (Arg.hasAttribute(Attribute::Returned))
10684         Flags.setReturned();
10685 
10686       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
10687           *CurDAG->getContext(), F.getCallingConv(), VT);
10688       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
10689           *CurDAG->getContext(), F.getCallingConv(), VT);
10690       for (unsigned i = 0; i != NumRegs; ++i) {
10691         // For scalable vectors, use the minimum size; individual targets
10692         // are responsible for handling scalable vector arguments and
10693         // return values.
10694         ISD::InputArg MyFlags(
10695             Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
10696             PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
10697         if (NumRegs > 1 && i == 0)
10698           MyFlags.Flags.setSplit();
10699         // if it isn't first piece, alignment must be 1
10700         else if (i > 0) {
10701           MyFlags.Flags.setOrigAlign(Align(1));
10702           if (i == NumRegs - 1)
10703             MyFlags.Flags.setSplitEnd();
10704         }
10705         Ins.push_back(MyFlags);
10706       }
10707       if (NeedsRegBlock && Value == NumValues - 1)
10708         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
10709       PartBase += VT.getStoreSize().getKnownMinValue();
10710     }
10711   }
10712 
10713   // Call the target to set up the argument values.
10714   SmallVector<SDValue, 8> InVals;
10715   SDValue NewRoot = TLI->LowerFormalArguments(
10716       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
10717 
10718   // Verify that the target's LowerFormalArguments behaved as expected.
10719   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
10720          "LowerFormalArguments didn't return a valid chain!");
10721   assert(InVals.size() == Ins.size() &&
10722          "LowerFormalArguments didn't emit the correct number of values!");
10723   LLVM_DEBUG({
10724     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10725       assert(InVals[i].getNode() &&
10726              "LowerFormalArguments emitted a null value!");
10727       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
10728              "LowerFormalArguments emitted a value with the wrong type!");
10729     }
10730   });
10731 
10732   // Update the DAG with the new chain value resulting from argument lowering.
10733   DAG.setRoot(NewRoot);
10734 
10735   // Set up the argument values.
10736   unsigned i = 0;
10737   if (!FuncInfo->CanLowerReturn) {
10738     // Create a virtual register for the sret pointer, and put in a copy
10739     // from the sret argument into it.
10740     SmallVector<EVT, 1> ValueVTs;
10741     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10742                     F.getReturnType()->getPointerTo(
10743                         DAG.getDataLayout().getAllocaAddrSpace()),
10744                     ValueVTs);
10745     MVT VT = ValueVTs[0].getSimpleVT();
10746     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
10747     std::optional<ISD::NodeType> AssertOp;
10748     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
10749                                         nullptr, F.getCallingConv(), AssertOp);
10750 
10751     MachineFunction& MF = SDB->DAG.getMachineFunction();
10752     MachineRegisterInfo& RegInfo = MF.getRegInfo();
10753     Register SRetReg =
10754         RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
10755     FuncInfo->DemoteRegister = SRetReg;
10756     NewRoot =
10757         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
10758     DAG.setRoot(NewRoot);
10759 
10760     // i indexes lowered arguments.  Bump it past the hidden sret argument.
10761     ++i;
10762   }
10763 
10764   SmallVector<SDValue, 4> Chains;
10765   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
10766   for (const Argument &Arg : F.args()) {
10767     SmallVector<SDValue, 4> ArgValues;
10768     SmallVector<EVT, 4> ValueVTs;
10769     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10770     unsigned NumValues = ValueVTs.size();
10771     if (NumValues == 0)
10772       continue;
10773 
10774     bool ArgHasUses = !Arg.use_empty();
10775 
10776     // Elide the copying store if the target loaded this argument from a
10777     // suitable fixed stack object.
10778     if (Ins[i].Flags.isCopyElisionCandidate()) {
10779       tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
10780                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
10781                              InVals[i], ArgHasUses);
10782     }
10783 
10784     // If this argument is unused then remember its value. It is used to generate
10785     // debugging information.
10786     bool isSwiftErrorArg =
10787         TLI->supportSwiftError() &&
10788         Arg.hasAttribute(Attribute::SwiftError);
10789     if (!ArgHasUses && !isSwiftErrorArg) {
10790       SDB->setUnusedArgValue(&Arg, InVals[i]);
10791 
10792       // Also remember any frame index for use in FastISel.
10793       if (FrameIndexSDNode *FI =
10794           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
10795         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10796     }
10797 
10798     for (unsigned Val = 0; Val != NumValues; ++Val) {
10799       EVT VT = ValueVTs[Val];
10800       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
10801                                                       F.getCallingConv(), VT);
10802       unsigned NumParts = TLI->getNumRegistersForCallingConv(
10803           *CurDAG->getContext(), F.getCallingConv(), VT);
10804 
10805       // Even an apparent 'unused' swifterror argument needs to be returned. So
10806       // we do generate a copy for it that can be used on return from the
10807       // function.
10808       if (ArgHasUses || isSwiftErrorArg) {
10809         std::optional<ISD::NodeType> AssertOp;
10810         if (Arg.hasAttribute(Attribute::SExt))
10811           AssertOp = ISD::AssertSext;
10812         else if (Arg.hasAttribute(Attribute::ZExt))
10813           AssertOp = ISD::AssertZext;
10814 
10815         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
10816                                              PartVT, VT, nullptr,
10817                                              F.getCallingConv(), AssertOp));
10818       }
10819 
10820       i += NumParts;
10821     }
10822 
10823     // We don't need to do anything else for unused arguments.
10824     if (ArgValues.empty())
10825       continue;
10826 
10827     // Note down frame index.
10828     if (FrameIndexSDNode *FI =
10829         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
10830       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10831 
10832     SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
10833                                      SDB->getCurSDLoc());
10834 
10835     SDB->setValue(&Arg, Res);
10836     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
10837       // We want to associate the argument with the frame index, among
10838       // involved operands, that correspond to the lowest address. The
10839       // getCopyFromParts function, called earlier, is swapping the order of
10840       // the operands to BUILD_PAIR depending on endianness. The result of
10841       // that swapping is that the least significant bits of the argument will
10842       // be in the first operand of the BUILD_PAIR node, and the most
10843       // significant bits will be in the second operand.
10844       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
10845       if (LoadSDNode *LNode =
10846           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
10847         if (FrameIndexSDNode *FI =
10848             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
10849           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10850     }
10851 
10852     // Analyses past this point are naive and don't expect an assertion.
10853     if (Res.getOpcode() == ISD::AssertZext)
10854       Res = Res.getOperand(0);
10855 
10856     // Update the SwiftErrorVRegDefMap.
10857     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
10858       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10859       if (Register::isVirtualRegister(Reg))
10860         SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
10861                                    Reg);
10862     }
10863 
10864     // If this argument is live outside of the entry block, insert a copy from
10865     // wherever we got it to the vreg that other BB's will reference it as.
10866     if (Res.getOpcode() == ISD::CopyFromReg) {
10867       // If we can, though, try to skip creating an unnecessary vreg.
10868       // FIXME: This isn't very clean... it would be nice to make this more
10869       // general.
10870       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10871       if (Register::isVirtualRegister(Reg)) {
10872         FuncInfo->ValueMap[&Arg] = Reg;
10873         continue;
10874       }
10875     }
10876     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
10877       FuncInfo->InitializeRegForValue(&Arg);
10878       SDB->CopyToExportRegsIfNeeded(&Arg);
10879     }
10880   }
10881 
10882   if (!Chains.empty()) {
10883     Chains.push_back(NewRoot);
10884     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
10885   }
10886 
10887   DAG.setRoot(NewRoot);
10888 
10889   assert(i == InVals.size() && "Argument register count mismatch!");
10890 
10891   // If any argument copy elisions occurred and we have debug info, update the
10892   // stale frame indices used in the dbg.declare variable info table.
10893   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
10894   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
10895     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
10896       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
10897       if (I != ArgCopyElisionFrameIndexMap.end())
10898         VI.Slot = I->second;
10899     }
10900   }
10901 
10902   // Finally, if the target has anything special to do, allow it to do so.
10903   emitFunctionEntryCode();
10904 }
10905 
10906 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
10907 /// ensure constants are generated when needed.  Remember the virtual registers
10908 /// that need to be added to the Machine PHI nodes as input.  We cannot just
10909 /// directly add them, because expansion might result in multiple MBB's for one
10910 /// BB.  As such, the start of the BB might correspond to a different MBB than
10911 /// the end.
10912 void
10913 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
10914   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10915 
10916   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
10917 
10918   // Check PHI nodes in successors that expect a value to be available from this
10919   // block.
10920   for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
10921     if (!isa<PHINode>(SuccBB->begin())) continue;
10922     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
10923 
10924     // If this terminator has multiple identical successors (common for
10925     // switches), only handle each succ once.
10926     if (!SuccsHandled.insert(SuccMBB).second)
10927       continue;
10928 
10929     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
10930 
10931     // At this point we know that there is a 1-1 correspondence between LLVM PHI
10932     // nodes and Machine PHI nodes, but the incoming operands have not been
10933     // emitted yet.
10934     for (const PHINode &PN : SuccBB->phis()) {
10935       // Ignore dead phi's.
10936       if (PN.use_empty())
10937         continue;
10938 
10939       // Skip empty types
10940       if (PN.getType()->isEmptyTy())
10941         continue;
10942 
10943       unsigned Reg;
10944       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
10945 
10946       if (const auto *C = dyn_cast<Constant>(PHIOp)) {
10947         unsigned &RegOut = ConstantsOut[C];
10948         if (RegOut == 0) {
10949           RegOut = FuncInfo.CreateRegs(C);
10950           // We need to zero/sign extend ConstantInt phi operands to match
10951           // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
10952           ISD::NodeType ExtendType = ISD::ANY_EXTEND;
10953           if (auto *CI = dyn_cast<ConstantInt>(C))
10954             ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
10955                                                     : ISD::ZERO_EXTEND;
10956           CopyValueToVirtualRegister(C, RegOut, ExtendType);
10957         }
10958         Reg = RegOut;
10959       } else {
10960         DenseMap<const Value *, Register>::iterator I =
10961           FuncInfo.ValueMap.find(PHIOp);
10962         if (I != FuncInfo.ValueMap.end())
10963           Reg = I->second;
10964         else {
10965           assert(isa<AllocaInst>(PHIOp) &&
10966                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
10967                  "Didn't codegen value into a register!??");
10968           Reg = FuncInfo.CreateRegs(PHIOp);
10969           CopyValueToVirtualRegister(PHIOp, Reg);
10970         }
10971       }
10972 
10973       // Remember that this register needs to added to the machine PHI node as
10974       // the input for this MBB.
10975       SmallVector<EVT, 4> ValueVTs;
10976       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
10977       for (EVT VT : ValueVTs) {
10978         const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
10979         for (unsigned i = 0; i != NumRegisters; ++i)
10980           FuncInfo.PHINodesToUpdate.push_back(
10981               std::make_pair(&*MBBI++, Reg + i));
10982         Reg += NumRegisters;
10983       }
10984     }
10985   }
10986 
10987   ConstantsOut.clear();
10988 }
10989 
10990 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
10991   MachineFunction::iterator I(MBB);
10992   if (++I == FuncInfo.MF->end())
10993     return nullptr;
10994   return &*I;
10995 }
10996 
10997 /// During lowering new call nodes can be created (such as memset, etc.).
10998 /// Those will become new roots of the current DAG, but complications arise
10999 /// when they are tail calls. In such cases, the call lowering will update
11000 /// the root, but the builder still needs to know that a tail call has been
11001 /// lowered in order to avoid generating an additional return.
11002 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11003   // If the node is null, we do have a tail call.
11004   if (MaybeTC.getNode() != nullptr)
11005     DAG.setRoot(MaybeTC);
11006   else
11007     HasTailCall = true;
11008 }
11009 
11010 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11011                                         MachineBasicBlock *SwitchMBB,
11012                                         MachineBasicBlock *DefaultMBB) {
11013   MachineFunction *CurMF = FuncInfo.MF;
11014   MachineBasicBlock *NextMBB = nullptr;
11015   MachineFunction::iterator BBI(W.MBB);
11016   if (++BBI != FuncInfo.MF->end())
11017     NextMBB = &*BBI;
11018 
11019   unsigned Size = W.LastCluster - W.FirstCluster + 1;
11020 
11021   BranchProbabilityInfo *BPI = FuncInfo.BPI;
11022 
11023   if (Size == 2 && W.MBB == SwitchMBB) {
11024     // If any two of the cases has the same destination, and if one value
11025     // is the same as the other, but has one bit unset that the other has set,
11026     // use bit manipulation to do two compares at once.  For example:
11027     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11028     // TODO: This could be extended to merge any 2 cases in switches with 3
11029     // cases.
11030     // TODO: Handle cases where W.CaseBB != SwitchBB.
11031     CaseCluster &Small = *W.FirstCluster;
11032     CaseCluster &Big = *W.LastCluster;
11033 
11034     if (Small.Low == Small.High && Big.Low == Big.High &&
11035         Small.MBB == Big.MBB) {
11036       const APInt &SmallValue = Small.Low->getValue();
11037       const APInt &BigValue = Big.Low->getValue();
11038 
11039       // Check that there is only one bit different.
11040       APInt CommonBit = BigValue ^ SmallValue;
11041       if (CommonBit.isPowerOf2()) {
11042         SDValue CondLHS = getValue(Cond);
11043         EVT VT = CondLHS.getValueType();
11044         SDLoc DL = getCurSDLoc();
11045 
11046         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11047                                  DAG.getConstant(CommonBit, DL, VT));
11048         SDValue Cond = DAG.getSetCC(
11049             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11050             ISD::SETEQ);
11051 
11052         // Update successor info.
11053         // Both Small and Big will jump to Small.BB, so we sum up the
11054         // probabilities.
11055         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11056         if (BPI)
11057           addSuccessorWithProb(
11058               SwitchMBB, DefaultMBB,
11059               // The default destination is the first successor in IR.
11060               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
11061         else
11062           addSuccessorWithProb(SwitchMBB, DefaultMBB);
11063 
11064         // Insert the true branch.
11065         SDValue BrCond =
11066             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
11067                         DAG.getBasicBlock(Small.MBB));
11068         // Insert the false branch.
11069         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
11070                              DAG.getBasicBlock(DefaultMBB));
11071 
11072         DAG.setRoot(BrCond);
11073         return;
11074       }
11075     }
11076   }
11077 
11078   if (TM.getOptLevel() != CodeGenOpt::None) {
11079     // Here, we order cases by probability so the most likely case will be
11080     // checked first. However, two clusters can have the same probability in
11081     // which case their relative ordering is non-deterministic. So we use Low
11082     // as a tie-breaker as clusters are guaranteed to never overlap.
11083     llvm::sort(W.FirstCluster, W.LastCluster + 1,
11084                [](const CaseCluster &a, const CaseCluster &b) {
11085       return a.Prob != b.Prob ?
11086              a.Prob > b.Prob :
11087              a.Low->getValue().slt(b.Low->getValue());
11088     });
11089 
11090     // Rearrange the case blocks so that the last one falls through if possible
11091     // without changing the order of probabilities.
11092     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
11093       --I;
11094       if (I->Prob > W.LastCluster->Prob)
11095         break;
11096       if (I->Kind == CC_Range && I->MBB == NextMBB) {
11097         std::swap(*I, *W.LastCluster);
11098         break;
11099       }
11100     }
11101   }
11102 
11103   // Compute total probability.
11104   BranchProbability DefaultProb = W.DefaultProb;
11105   BranchProbability UnhandledProbs = DefaultProb;
11106   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
11107     UnhandledProbs += I->Prob;
11108 
11109   MachineBasicBlock *CurMBB = W.MBB;
11110   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
11111     bool FallthroughUnreachable = false;
11112     MachineBasicBlock *Fallthrough;
11113     if (I == W.LastCluster) {
11114       // For the last cluster, fall through to the default destination.
11115       Fallthrough = DefaultMBB;
11116       FallthroughUnreachable = isa<UnreachableInst>(
11117           DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
11118     } else {
11119       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
11120       CurMF->insert(BBI, Fallthrough);
11121       // Put Cond in a virtual register to make it available from the new blocks.
11122       ExportFromCurrentBlock(Cond);
11123     }
11124     UnhandledProbs -= I->Prob;
11125 
11126     switch (I->Kind) {
11127       case CC_JumpTable: {
11128         // FIXME: Optimize away range check based on pivot comparisons.
11129         JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
11130         SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
11131 
11132         // The jump block hasn't been inserted yet; insert it here.
11133         MachineBasicBlock *JumpMBB = JT->MBB;
11134         CurMF->insert(BBI, JumpMBB);
11135 
11136         auto JumpProb = I->Prob;
11137         auto FallthroughProb = UnhandledProbs;
11138 
11139         // If the default statement is a target of the jump table, we evenly
11140         // distribute the default probability to successors of CurMBB. Also
11141         // update the probability on the edge from JumpMBB to Fallthrough.
11142         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
11143                                               SE = JumpMBB->succ_end();
11144              SI != SE; ++SI) {
11145           if (*SI == DefaultMBB) {
11146             JumpProb += DefaultProb / 2;
11147             FallthroughProb -= DefaultProb / 2;
11148             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
11149             JumpMBB->normalizeSuccProbs();
11150             break;
11151           }
11152         }
11153 
11154         if (FallthroughUnreachable)
11155           JTH->FallthroughUnreachable = true;
11156 
11157         if (!JTH->FallthroughUnreachable)
11158           addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
11159         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
11160         CurMBB->normalizeSuccProbs();
11161 
11162         // The jump table header will be inserted in our current block, do the
11163         // range check, and fall through to our fallthrough block.
11164         JTH->HeaderBB = CurMBB;
11165         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
11166 
11167         // If we're in the right place, emit the jump table header right now.
11168         if (CurMBB == SwitchMBB) {
11169           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
11170           JTH->Emitted = true;
11171         }
11172         break;
11173       }
11174       case CC_BitTests: {
11175         // FIXME: Optimize away range check based on pivot comparisons.
11176         BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
11177 
11178         // The bit test blocks haven't been inserted yet; insert them here.
11179         for (BitTestCase &BTC : BTB->Cases)
11180           CurMF->insert(BBI, BTC.ThisBB);
11181 
11182         // Fill in fields of the BitTestBlock.
11183         BTB->Parent = CurMBB;
11184         BTB->Default = Fallthrough;
11185 
11186         BTB->DefaultProb = UnhandledProbs;
11187         // If the cases in bit test don't form a contiguous range, we evenly
11188         // distribute the probability on the edge to Fallthrough to two
11189         // successors of CurMBB.
11190         if (!BTB->ContiguousRange) {
11191           BTB->Prob += DefaultProb / 2;
11192           BTB->DefaultProb -= DefaultProb / 2;
11193         }
11194 
11195         if (FallthroughUnreachable)
11196           BTB->FallthroughUnreachable = true;
11197 
11198         // If we're in the right place, emit the bit test header right now.
11199         if (CurMBB == SwitchMBB) {
11200           visitBitTestHeader(*BTB, SwitchMBB);
11201           BTB->Emitted = true;
11202         }
11203         break;
11204       }
11205       case CC_Range: {
11206         const Value *RHS, *LHS, *MHS;
11207         ISD::CondCode CC;
11208         if (I->Low == I->High) {
11209           // Check Cond == I->Low.
11210           CC = ISD::SETEQ;
11211           LHS = Cond;
11212           RHS=I->Low;
11213           MHS = nullptr;
11214         } else {
11215           // Check I->Low <= Cond <= I->High.
11216           CC = ISD::SETLE;
11217           LHS = I->Low;
11218           MHS = Cond;
11219           RHS = I->High;
11220         }
11221 
11222         // If Fallthrough is unreachable, fold away the comparison.
11223         if (FallthroughUnreachable)
11224           CC = ISD::SETTRUE;
11225 
11226         // The false probability is the sum of all unhandled cases.
11227         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
11228                      getCurSDLoc(), I->Prob, UnhandledProbs);
11229 
11230         if (CurMBB == SwitchMBB)
11231           visitSwitchCase(CB, SwitchMBB);
11232         else
11233           SL->SwitchCases.push_back(CB);
11234 
11235         break;
11236       }
11237     }
11238     CurMBB = Fallthrough;
11239   }
11240 }
11241 
11242 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
11243                                               CaseClusterIt First,
11244                                               CaseClusterIt Last) {
11245   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
11246     if (X.Prob != CC.Prob)
11247       return X.Prob > CC.Prob;
11248 
11249     // Ties are broken by comparing the case value.
11250     return X.Low->getValue().slt(CC.Low->getValue());
11251   });
11252 }
11253 
11254 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11255                                         const SwitchWorkListItem &W,
11256                                         Value *Cond,
11257                                         MachineBasicBlock *SwitchMBB) {
11258   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11259          "Clusters not sorted?");
11260 
11261   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11262 
11263   // Balance the tree based on branch probabilities to create a near-optimal (in
11264   // terms of search time given key frequency) binary search tree. See e.g. Kurt
11265   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
11266   CaseClusterIt LastLeft = W.FirstCluster;
11267   CaseClusterIt FirstRight = W.LastCluster;
11268   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
11269   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
11270 
11271   // Move LastLeft and FirstRight towards each other from opposite directions to
11272   // find a partitioning of the clusters which balances the probability on both
11273   // sides. If LeftProb and RightProb are equal, alternate which side is
11274   // taken to ensure 0-probability nodes are distributed evenly.
11275   unsigned I = 0;
11276   while (LastLeft + 1 < FirstRight) {
11277     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
11278       LeftProb += (++LastLeft)->Prob;
11279     else
11280       RightProb += (--FirstRight)->Prob;
11281     I++;
11282   }
11283 
11284   while (true) {
11285     // Our binary search tree differs from a typical BST in that ours can have up
11286     // to three values in each leaf. The pivot selection above doesn't take that
11287     // into account, which means the tree might require more nodes and be less
11288     // efficient. We compensate for this here.
11289 
11290     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
11291     unsigned NumRight = W.LastCluster - FirstRight + 1;
11292 
11293     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
11294       // If one side has less than 3 clusters, and the other has more than 3,
11295       // consider taking a cluster from the other side.
11296 
11297       if (NumLeft < NumRight) {
11298         // Consider moving the first cluster on the right to the left side.
11299         CaseCluster &CC = *FirstRight;
11300         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11301         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11302         if (LeftSideRank <= RightSideRank) {
11303           // Moving the cluster to the left does not demote it.
11304           ++LastLeft;
11305           ++FirstRight;
11306           continue;
11307         }
11308       } else {
11309         assert(NumRight < NumLeft);
11310         // Consider moving the last element on the left to the right side.
11311         CaseCluster &CC = *LastLeft;
11312         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11313         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11314         if (RightSideRank <= LeftSideRank) {
11315           // Moving the cluster to the right does not demot it.
11316           --LastLeft;
11317           --FirstRight;
11318           continue;
11319         }
11320       }
11321     }
11322     break;
11323   }
11324 
11325   assert(LastLeft + 1 == FirstRight);
11326   assert(LastLeft >= W.FirstCluster);
11327   assert(FirstRight <= W.LastCluster);
11328 
11329   // Use the first element on the right as pivot since we will make less-than
11330   // comparisons against it.
11331   CaseClusterIt PivotCluster = FirstRight;
11332   assert(PivotCluster > W.FirstCluster);
11333   assert(PivotCluster <= W.LastCluster);
11334 
11335   CaseClusterIt FirstLeft = W.FirstCluster;
11336   CaseClusterIt LastRight = W.LastCluster;
11337 
11338   const ConstantInt *Pivot = PivotCluster->Low;
11339 
11340   // New blocks will be inserted immediately after the current one.
11341   MachineFunction::iterator BBI(W.MBB);
11342   ++BBI;
11343 
11344   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11345   // we can branch to its destination directly if it's squeezed exactly in
11346   // between the known lower bound and Pivot - 1.
11347   MachineBasicBlock *LeftMBB;
11348   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11349       FirstLeft->Low == W.GE &&
11350       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11351     LeftMBB = FirstLeft->MBB;
11352   } else {
11353     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11354     FuncInfo.MF->insert(BBI, LeftMBB);
11355     WorkList.push_back(
11356         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11357     // Put Cond in a virtual register to make it available from the new blocks.
11358     ExportFromCurrentBlock(Cond);
11359   }
11360 
11361   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11362   // single cluster, RHS.Low == Pivot, and we can branch to its destination
11363   // directly if RHS.High equals the current upper bound.
11364   MachineBasicBlock *RightMBB;
11365   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11366       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11367     RightMBB = FirstRight->MBB;
11368   } else {
11369     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11370     FuncInfo.MF->insert(BBI, RightMBB);
11371     WorkList.push_back(
11372         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11373     // Put Cond in a virtual register to make it available from the new blocks.
11374     ExportFromCurrentBlock(Cond);
11375   }
11376 
11377   // Create the CaseBlock record that will be used to lower the branch.
11378   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11379                getCurSDLoc(), LeftProb, RightProb);
11380 
11381   if (W.MBB == SwitchMBB)
11382     visitSwitchCase(CB, SwitchMBB);
11383   else
11384     SL->SwitchCases.push_back(CB);
11385 }
11386 
11387 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11388 // from the swith statement.
11389 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
11390                                             BranchProbability PeeledCaseProb) {
11391   if (PeeledCaseProb == BranchProbability::getOne())
11392     return BranchProbability::getZero();
11393   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11394 
11395   uint32_t Numerator = CaseProb.getNumerator();
11396   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11397   return BranchProbability(Numerator, std::max(Numerator, Denominator));
11398 }
11399 
11400 // Try to peel the top probability case if it exceeds the threshold.
11401 // Return current MachineBasicBlock for the switch statement if the peeling
11402 // does not occur.
11403 // If the peeling is performed, return the newly created MachineBasicBlock
11404 // for the peeled switch statement. Also update Clusters to remove the peeled
11405 // case. PeeledCaseProb is the BranchProbability for the peeled case.
11406 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11407     const SwitchInst &SI, CaseClusterVector &Clusters,
11408     BranchProbability &PeeledCaseProb) {
11409   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11410   // Don't perform if there is only one cluster or optimizing for size.
11411   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11412       TM.getOptLevel() == CodeGenOpt::None ||
11413       SwitchMBB->getParent()->getFunction().hasMinSize())
11414     return SwitchMBB;
11415 
11416   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
11417   unsigned PeeledCaseIndex = 0;
11418   bool SwitchPeeled = false;
11419   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11420     CaseCluster &CC = Clusters[Index];
11421     if (CC.Prob < TopCaseProb)
11422       continue;
11423     TopCaseProb = CC.Prob;
11424     PeeledCaseIndex = Index;
11425     SwitchPeeled = true;
11426   }
11427   if (!SwitchPeeled)
11428     return SwitchMBB;
11429 
11430   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11431                     << TopCaseProb << "\n");
11432 
11433   // Record the MBB for the peeled switch statement.
11434   MachineFunction::iterator BBI(SwitchMBB);
11435   ++BBI;
11436   MachineBasicBlock *PeeledSwitchMBB =
11437       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
11438   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
11439 
11440   ExportFromCurrentBlock(SI.getCondition());
11441   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
11442   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
11443                           nullptr,   nullptr,      TopCaseProb.getCompl()};
11444   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
11445 
11446   Clusters.erase(PeeledCaseIt);
11447   for (CaseCluster &CC : Clusters) {
11448     LLVM_DEBUG(
11449         dbgs() << "Scale the probablity for one cluster, before scaling: "
11450                << CC.Prob << "\n");
11451     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
11452     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
11453   }
11454   PeeledCaseProb = TopCaseProb;
11455   return PeeledSwitchMBB;
11456 }
11457 
11458 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
11459   // Extract cases from the switch.
11460   BranchProbabilityInfo *BPI = FuncInfo.BPI;
11461   CaseClusterVector Clusters;
11462   Clusters.reserve(SI.getNumCases());
11463   for (auto I : SI.cases()) {
11464     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
11465     const ConstantInt *CaseVal = I.getCaseValue();
11466     BranchProbability Prob =
11467         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
11468             : BranchProbability(1, SI.getNumCases() + 1);
11469     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
11470   }
11471 
11472   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
11473 
11474   // Cluster adjacent cases with the same destination. We do this at all
11475   // optimization levels because it's cheap to do and will make codegen faster
11476   // if there are many clusters.
11477   sortAndRangeify(Clusters);
11478 
11479   // The branch probablity of the peeled case.
11480   BranchProbability PeeledCaseProb = BranchProbability::getZero();
11481   MachineBasicBlock *PeeledSwitchMBB =
11482       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
11483 
11484   // If there is only the default destination, jump there directly.
11485   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11486   if (Clusters.empty()) {
11487     assert(PeeledSwitchMBB == SwitchMBB);
11488     SwitchMBB->addSuccessor(DefaultMBB);
11489     if (DefaultMBB != NextBlock(SwitchMBB)) {
11490       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
11491                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
11492     }
11493     return;
11494   }
11495 
11496   SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI());
11497   SL->findBitTestClusters(Clusters, &SI);
11498 
11499   LLVM_DEBUG({
11500     dbgs() << "Case clusters: ";
11501     for (const CaseCluster &C : Clusters) {
11502       if (C.Kind == CC_JumpTable)
11503         dbgs() << "JT:";
11504       if (C.Kind == CC_BitTests)
11505         dbgs() << "BT:";
11506 
11507       C.Low->getValue().print(dbgs(), true);
11508       if (C.Low != C.High) {
11509         dbgs() << '-';
11510         C.High->getValue().print(dbgs(), true);
11511       }
11512       dbgs() << ' ';
11513     }
11514     dbgs() << '\n';
11515   });
11516 
11517   assert(!Clusters.empty());
11518   SwitchWorkList WorkList;
11519   CaseClusterIt First = Clusters.begin();
11520   CaseClusterIt Last = Clusters.end() - 1;
11521   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
11522   // Scale the branchprobability for DefaultMBB if the peel occurs and
11523   // DefaultMBB is not replaced.
11524   if (PeeledCaseProb != BranchProbability::getZero() &&
11525       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
11526     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
11527   WorkList.push_back(
11528       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
11529 
11530   while (!WorkList.empty()) {
11531     SwitchWorkListItem W = WorkList.pop_back_val();
11532     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
11533 
11534     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
11535         !DefaultMBB->getParent()->getFunction().hasMinSize()) {
11536       // For optimized builds, lower large range as a balanced binary tree.
11537       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
11538       continue;
11539     }
11540 
11541     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
11542   }
11543 }
11544 
11545 void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
11546   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11547   auto DL = getCurSDLoc();
11548   EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11549   setValue(&I, DAG.getStepVector(DL, ResultVT));
11550 }
11551 
11552 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
11553   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11554   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11555 
11556   SDLoc DL = getCurSDLoc();
11557   SDValue V = getValue(I.getOperand(0));
11558   assert(VT == V.getValueType() && "Malformed vector.reverse!");
11559 
11560   if (VT.isScalableVector()) {
11561     setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
11562     return;
11563   }
11564 
11565   // Use VECTOR_SHUFFLE for the fixed-length vector
11566   // to maintain existing behavior.
11567   SmallVector<int, 8> Mask;
11568   unsigned NumElts = VT.getVectorMinNumElements();
11569   for (unsigned i = 0; i != NumElts; ++i)
11570     Mask.push_back(NumElts - 1 - i);
11571 
11572   setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
11573 }
11574 
11575 void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
11576   auto DL = getCurSDLoc();
11577   SDValue InVec = getValue(I.getOperand(0));
11578   EVT OutVT =
11579       InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
11580 
11581   unsigned OutNumElts = OutVT.getVectorMinNumElements();
11582 
11583   // ISD Node needs the input vectors split into two equal parts
11584   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
11585                            DAG.getVectorIdxConstant(0, DL));
11586   SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
11587                            DAG.getVectorIdxConstant(OutNumElts, DL));
11588 
11589   // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
11590   // legalisation and combines.
11591   if (OutVT.isFixedLengthVector()) {
11592     SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
11593                                         createStrideMask(0, 2, OutNumElts));
11594     SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
11595                                        createStrideMask(1, 2, OutNumElts));
11596     SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
11597     setValue(&I, Res);
11598     return;
11599   }
11600 
11601   SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
11602                             DAG.getVTList(OutVT, OutVT), Lo, Hi);
11603   setValue(&I, Res);
11604   return;
11605 }
11606 
11607 void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
11608   auto DL = getCurSDLoc();
11609   EVT InVT = getValue(I.getOperand(0)).getValueType();
11610   SDValue InVec0 = getValue(I.getOperand(0));
11611   SDValue InVec1 = getValue(I.getOperand(1));
11612   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11613   EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11614 
11615   // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
11616   // legalisation and combines.
11617   if (OutVT.isFixedLengthVector()) {
11618     unsigned NumElts = InVT.getVectorMinNumElements();
11619     SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
11620     setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
11621                                       createInterleaveMask(NumElts, 2)));
11622     return;
11623   }
11624 
11625   SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
11626                             DAG.getVTList(InVT, InVT), InVec0, InVec1);
11627   Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
11628                     Res.getValue(1));
11629   setValue(&I, Res);
11630   return;
11631 }
11632 
11633 void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
11634   SmallVector<EVT, 4> ValueVTs;
11635   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
11636                   ValueVTs);
11637   unsigned NumValues = ValueVTs.size();
11638   if (NumValues == 0) return;
11639 
11640   SmallVector<SDValue, 4> Values(NumValues);
11641   SDValue Op = getValue(I.getOperand(0));
11642 
11643   for (unsigned i = 0; i != NumValues; ++i)
11644     Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
11645                             SDValue(Op.getNode(), Op.getResNo() + i));
11646 
11647   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
11648                            DAG.getVTList(ValueVTs), Values));
11649 }
11650 
11651 void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
11652   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11653   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11654 
11655   SDLoc DL = getCurSDLoc();
11656   SDValue V1 = getValue(I.getOperand(0));
11657   SDValue V2 = getValue(I.getOperand(1));
11658   int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
11659 
11660   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
11661   if (VT.isScalableVector()) {
11662     MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
11663     setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
11664                              DAG.getConstant(Imm, DL, IdxVT)));
11665     return;
11666   }
11667 
11668   unsigned NumElts = VT.getVectorNumElements();
11669 
11670   uint64_t Idx = (NumElts + Imm) % NumElts;
11671 
11672   // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
11673   SmallVector<int, 8> Mask;
11674   for (unsigned i = 0; i < NumElts; ++i)
11675     Mask.push_back(Idx + i);
11676   setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
11677 }
11678 
11679 // Consider the following MIR after SelectionDAG, which produces output in
11680 // phyregs in the first case or virtregs in the second case.
11681 //
11682 // INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
11683 // %5:gr32 = COPY $ebx
11684 // %6:gr32 = COPY $edx
11685 // %1:gr32 = COPY %6:gr32
11686 // %0:gr32 = COPY %5:gr32
11687 //
11688 // INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
11689 // %1:gr32 = COPY %6:gr32
11690 // %0:gr32 = COPY %5:gr32
11691 //
11692 // Given %0, we'd like to return $ebx in the first case and %5 in the second.
11693 // Given %1, we'd like to return $edx in the first case and %6 in the second.
11694 //
11695 // If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
11696 // to a single virtreg (such as %0). The remaining outputs monotonically
11697 // increase in virtreg number from there. If a callbr has no outputs, then it
11698 // should not have a corresponding callbr landingpad; in fact, the callbr
11699 // landingpad would not even be able to refer to such a callbr.
11700 static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
11701   MachineInstr *MI = MRI.def_begin(Reg)->getParent();
11702   // There is definitely at least one copy.
11703   assert(MI->getOpcode() == TargetOpcode::COPY &&
11704          "start of copy chain MUST be COPY");
11705   Reg = MI->getOperand(1).getReg();
11706   MI = MRI.def_begin(Reg)->getParent();
11707   // There may be an optional second copy.
11708   if (MI->getOpcode() == TargetOpcode::COPY) {
11709     assert(Reg.isVirtual() && "expected COPY of virtual register");
11710     Reg = MI->getOperand(1).getReg();
11711     assert(Reg.isPhysical() && "expected COPY of physical register");
11712     MI = MRI.def_begin(Reg)->getParent();
11713   }
11714   // The start of the chain must be an INLINEASM_BR.
11715   assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
11716          "end of copy chain MUST be INLINEASM_BR");
11717   return Reg;
11718 }
11719 
11720 // We must do this walk rather than the simpler
11721 //   setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
11722 // otherwise we will end up with copies of virtregs only valid along direct
11723 // edges.
11724 void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
11725   SmallVector<EVT, 8> ResultVTs;
11726   SmallVector<SDValue, 8> ResultValues;
11727   const auto *CBR =
11728       cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
11729 
11730   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11731   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
11732   MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
11733 
11734   unsigned InitialDef = FuncInfo.ValueMap[CBR];
11735   SDValue Chain = DAG.getRoot();
11736 
11737   // Re-parse the asm constraints string.
11738   TargetLowering::AsmOperandInfoVector TargetConstraints =
11739       TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
11740   for (auto &T : TargetConstraints) {
11741     SDISelAsmOperandInfo OpInfo(T);
11742     if (OpInfo.Type != InlineAsm::isOutput)
11743       continue;
11744 
11745     // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
11746     // individual constraint.
11747     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
11748 
11749     switch (OpInfo.ConstraintType) {
11750     case TargetLowering::C_Register:
11751     case TargetLowering::C_RegisterClass: {
11752       // Fill in OpInfo.AssignedRegs.Regs.
11753       getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
11754 
11755       // getRegistersForValue may produce 1 to many registers based on whether
11756       // the OpInfo.ConstraintVT is legal on the target or not.
11757       for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
11758         Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
11759         if (Register::isPhysicalRegister(OriginalDef))
11760           FuncInfo.MBB->addLiveIn(OriginalDef);
11761         // Update the assigned registers to use the original defs.
11762         OpInfo.AssignedRegs.Regs[i] = OriginalDef;
11763       }
11764 
11765       SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
11766           DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
11767       ResultValues.push_back(V);
11768       ResultVTs.push_back(OpInfo.ConstraintVT);
11769       break;
11770     }
11771     case TargetLowering::C_Other: {
11772       SDValue Flag;
11773       SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
11774                                                   OpInfo, DAG);
11775       ++InitialDef;
11776       ResultValues.push_back(V);
11777       ResultVTs.push_back(OpInfo.ConstraintVT);
11778       break;
11779     }
11780     default:
11781       break;
11782     }
11783   }
11784   SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
11785                           DAG.getVTList(ResultVTs), ResultValues);
11786   setValue(&I, V);
11787 }
11788