xref: /llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision 2a721374aef326d4668f750d341c86d1aa1a0309)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/Triple.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/AliasAnalysis.h"
27 #include "llvm/Analysis/BranchProbabilityInfo.h"
28 #include "llvm/Analysis/ConstantFolding.h"
29 #include "llvm/Analysis/EHPersonalities.h"
30 #include "llvm/Analysis/MemoryLocation.h"
31 #include "llvm/Analysis/TargetLibraryInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/Analysis.h"
34 #include "llvm/CodeGen/CodeGenCommonISel.h"
35 #include "llvm/CodeGen/FunctionLoweringInfo.h"
36 #include "llvm/CodeGen/GCMetadata.h"
37 #include "llvm/CodeGen/MachineBasicBlock.h"
38 #include "llvm/CodeGen/MachineFrameInfo.h"
39 #include "llvm/CodeGen/MachineFunction.h"
40 #include "llvm/CodeGen/MachineInstrBuilder.h"
41 #include "llvm/CodeGen/MachineInstrBundleIterator.h"
42 #include "llvm/CodeGen/MachineMemOperand.h"
43 #include "llvm/CodeGen/MachineModuleInfo.h"
44 #include "llvm/CodeGen/MachineOperand.h"
45 #include "llvm/CodeGen/MachineRegisterInfo.h"
46 #include "llvm/CodeGen/RuntimeLibcalls.h"
47 #include "llvm/CodeGen/SelectionDAG.h"
48 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
49 #include "llvm/CodeGen/StackMaps.h"
50 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
51 #include "llvm/CodeGen/TargetFrameLowering.h"
52 #include "llvm/CodeGen/TargetInstrInfo.h"
53 #include "llvm/CodeGen/TargetOpcodes.h"
54 #include "llvm/CodeGen/TargetRegisterInfo.h"
55 #include "llvm/CodeGen/TargetSubtargetInfo.h"
56 #include "llvm/CodeGen/WinEHFuncInfo.h"
57 #include "llvm/IR/Argument.h"
58 #include "llvm/IR/Attributes.h"
59 #include "llvm/IR/BasicBlock.h"
60 #include "llvm/IR/CFG.h"
61 #include "llvm/IR/CallingConv.h"
62 #include "llvm/IR/Constant.h"
63 #include "llvm/IR/ConstantRange.h"
64 #include "llvm/IR/Constants.h"
65 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/DebugInfoMetadata.h"
67 #include "llvm/IR/DerivedTypes.h"
68 #include "llvm/IR/DiagnosticInfo.h"
69 #include "llvm/IR/Function.h"
70 #include "llvm/IR/GetElementPtrTypeIterator.h"
71 #include "llvm/IR/InlineAsm.h"
72 #include "llvm/IR/InstrTypes.h"
73 #include "llvm/IR/Instructions.h"
74 #include "llvm/IR/IntrinsicInst.h"
75 #include "llvm/IR/Intrinsics.h"
76 #include "llvm/IR/IntrinsicsAArch64.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
78 #include "llvm/IR/LLVMContext.h"
79 #include "llvm/IR/Metadata.h"
80 #include "llvm/IR/Module.h"
81 #include "llvm/IR/Operator.h"
82 #include "llvm/IR/PatternMatch.h"
83 #include "llvm/IR/Statepoint.h"
84 #include "llvm/IR/Type.h"
85 #include "llvm/IR/User.h"
86 #include "llvm/IR/Value.h"
87 #include "llvm/MC/MCContext.h"
88 #include "llvm/Support/AtomicOrdering.h"
89 #include "llvm/Support/Casting.h"
90 #include "llvm/Support/CommandLine.h"
91 #include "llvm/Support/Compiler.h"
92 #include "llvm/Support/Debug.h"
93 #include "llvm/Support/MathExtras.h"
94 #include "llvm/Support/raw_ostream.h"
95 #include "llvm/Target/TargetIntrinsicInfo.h"
96 #include "llvm/Target/TargetMachine.h"
97 #include "llvm/Target/TargetOptions.h"
98 #include "llvm/Transforms/Utils/Local.h"
99 #include <cstddef>
100 #include <iterator>
101 #include <limits>
102 #include <tuple>
103 
104 using namespace llvm;
105 using namespace PatternMatch;
106 using namespace SwitchCG;
107 
108 #define DEBUG_TYPE "isel"
109 
110 /// LimitFloatPrecision - Generate low-precision inline sequences for
111 /// some float libcalls (6, 8 or 12 bits).
112 static unsigned LimitFloatPrecision;
113 
114 static cl::opt<bool>
115     InsertAssertAlign("insert-assert-align", cl::init(true),
116                       cl::desc("Insert the experimental `assertalign` node."),
117                       cl::ReallyHidden);
118 
119 static cl::opt<unsigned, true>
120     LimitFPPrecision("limit-float-precision",
121                      cl::desc("Generate low-precision inline sequences "
122                               "for some float libcalls"),
123                      cl::location(LimitFloatPrecision), cl::Hidden,
124                      cl::init(0));
125 
126 static cl::opt<unsigned> SwitchPeelThreshold(
127     "switch-peel-threshold", cl::Hidden, cl::init(66),
128     cl::desc("Set the case probability threshold for peeling the case from a "
129              "switch statement. A value greater than 100 will void this "
130              "optimization"));
131 
132 // Limit the width of DAG chains. This is important in general to prevent
133 // DAG-based analysis from blowing up. For example, alias analysis and
134 // load clustering may not complete in reasonable time. It is difficult to
135 // recognize and avoid this situation within each individual analysis, and
136 // future analyses are likely to have the same behavior. Limiting DAG width is
137 // the safe approach and will be especially important with global DAGs.
138 //
139 // MaxParallelChains default is arbitrarily high to avoid affecting
140 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
141 // sequence over this should have been converted to llvm.memcpy by the
142 // frontend. It is easy to induce this behavior with .ll code such as:
143 // %buffer = alloca [4096 x i8]
144 // %data = load [4096 x i8]* %argPtr
145 // store [4096 x i8] %data, [4096 x i8]* %buffer
146 static const unsigned MaxParallelChains = 64;
147 
148 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
149                                       const SDValue *Parts, unsigned NumParts,
150                                       MVT PartVT, EVT ValueVT, const Value *V,
151                                       Optional<CallingConv::ID> CC);
152 
153 /// getCopyFromParts - Create a value that contains the specified legal parts
154 /// combined into the value they represent.  If the parts combine to a type
155 /// larger than ValueVT then AssertOp can be used to specify whether the extra
156 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
157 /// (ISD::AssertSext).
158 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
159                                 const SDValue *Parts, unsigned NumParts,
160                                 MVT PartVT, EVT ValueVT, const Value *V,
161                                 Optional<CallingConv::ID> CC = None,
162                                 Optional<ISD::NodeType> AssertOp = None) {
163   // Let the target assemble the parts if it wants to
164   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
165   if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
166                                                    PartVT, ValueVT, CC))
167     return Val;
168 
169   if (ValueVT.isVector())
170     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
171                                   CC);
172 
173   assert(NumParts > 0 && "No parts to assemble!");
174   SDValue Val = Parts[0];
175 
176   if (NumParts > 1) {
177     // Assemble the value from multiple parts.
178     if (ValueVT.isInteger()) {
179       unsigned PartBits = PartVT.getSizeInBits();
180       unsigned ValueBits = ValueVT.getSizeInBits();
181 
182       // Assemble the power of 2 part.
183       unsigned RoundParts =
184           (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
185       unsigned RoundBits = PartBits * RoundParts;
186       EVT RoundVT = RoundBits == ValueBits ?
187         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
188       SDValue Lo, Hi;
189 
190       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
191 
192       if (RoundParts > 2) {
193         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
194                               PartVT, HalfVT, V);
195         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
196                               RoundParts / 2, PartVT, HalfVT, V);
197       } else {
198         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
199         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
200       }
201 
202       if (DAG.getDataLayout().isBigEndian())
203         std::swap(Lo, Hi);
204 
205       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
206 
207       if (RoundParts < NumParts) {
208         // Assemble the trailing non-power-of-2 part.
209         unsigned OddParts = NumParts - RoundParts;
210         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
211         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
212                               OddVT, V, CC);
213 
214         // Combine the round and odd parts.
215         Lo = Val;
216         if (DAG.getDataLayout().isBigEndian())
217           std::swap(Lo, Hi);
218         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
219         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
220         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
221                          DAG.getConstant(Lo.getValueSizeInBits(), DL,
222                                          TLI.getShiftAmountTy(
223                                              TotalVT, DAG.getDataLayout())));
224         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
225         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
226       }
227     } else if (PartVT.isFloatingPoint()) {
228       // FP split into multiple FP parts (for ppcf128)
229       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
230              "Unexpected split");
231       SDValue Lo, Hi;
232       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
233       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
234       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
235         std::swap(Lo, Hi);
236       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
237     } else {
238       // FP split into integer parts (soft fp)
239       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
240              !PartVT.isVector() && "Unexpected split");
241       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
242       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
243     }
244   }
245 
246   // There is now one part, held in Val.  Correct it to match ValueVT.
247   // PartEVT is the type of the register class that holds the value.
248   // ValueVT is the type of the inline asm operation.
249   EVT PartEVT = Val.getValueType();
250 
251   if (PartEVT == ValueVT)
252     return Val;
253 
254   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
255       ValueVT.bitsLT(PartEVT)) {
256     // For an FP value in an integer part, we need to truncate to the right
257     // width first.
258     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
259     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
260   }
261 
262   // Handle types that have the same size.
263   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
264     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
265 
266   // Handle types with different sizes.
267   if (PartEVT.isInteger() && ValueVT.isInteger()) {
268     if (ValueVT.bitsLT(PartEVT)) {
269       // For a truncate, see if we have any information to
270       // indicate whether the truncated bits will always be
271       // zero or sign-extension.
272       if (AssertOp)
273         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
274                           DAG.getValueType(ValueVT));
275       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
276     }
277     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
278   }
279 
280   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
281     // FP_ROUND's are always exact here.
282     if (ValueVT.bitsLT(Val.getValueType()))
283       return DAG.getNode(
284           ISD::FP_ROUND, DL, ValueVT, Val,
285           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
286 
287     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
288   }
289 
290   // Handle MMX to a narrower integer type by bitcasting MMX to integer and
291   // then truncating.
292   if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
293       ValueVT.bitsLT(PartEVT)) {
294     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
295     return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
296   }
297 
298   report_fatal_error("Unknown mismatch in getCopyFromParts!");
299 }
300 
301 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
302                                               const Twine &ErrMsg) {
303   const Instruction *I = dyn_cast_or_null<Instruction>(V);
304   if (!V)
305     return Ctx.emitError(ErrMsg);
306 
307   const char *AsmError = ", possible invalid constraint for vector type";
308   if (const CallInst *CI = dyn_cast<CallInst>(I))
309     if (CI->isInlineAsm())
310       return Ctx.emitError(I, ErrMsg + AsmError);
311 
312   return Ctx.emitError(I, ErrMsg);
313 }
314 
315 /// getCopyFromPartsVector - Create a value that contains the specified legal
316 /// parts combined into the value they represent.  If the parts combine to a
317 /// type larger than ValueVT then AssertOp can be used to specify whether the
318 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
319 /// ValueVT (ISD::AssertSext).
320 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
321                                       const SDValue *Parts, unsigned NumParts,
322                                       MVT PartVT, EVT ValueVT, const Value *V,
323                                       Optional<CallingConv::ID> CallConv) {
324   assert(ValueVT.isVector() && "Not a vector value");
325   assert(NumParts > 0 && "No parts to assemble!");
326   const bool IsABIRegCopy = CallConv.has_value();
327 
328   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
329   SDValue Val = Parts[0];
330 
331   // Handle a multi-element vector.
332   if (NumParts > 1) {
333     EVT IntermediateVT;
334     MVT RegisterVT;
335     unsigned NumIntermediates;
336     unsigned NumRegs;
337 
338     if (IsABIRegCopy) {
339       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
340           *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
341           NumIntermediates, RegisterVT);
342     } else {
343       NumRegs =
344           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
345                                      NumIntermediates, RegisterVT);
346     }
347 
348     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
349     NumParts = NumRegs; // Silence a compiler warning.
350     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
351     assert(RegisterVT.getSizeInBits() ==
352            Parts[0].getSimpleValueType().getSizeInBits() &&
353            "Part type sizes don't match!");
354 
355     // Assemble the parts into intermediate operands.
356     SmallVector<SDValue, 8> Ops(NumIntermediates);
357     if (NumIntermediates == NumParts) {
358       // If the register was not expanded, truncate or copy the value,
359       // as appropriate.
360       for (unsigned i = 0; i != NumParts; ++i)
361         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
362                                   PartVT, IntermediateVT, V, CallConv);
363     } else if (NumParts > 0) {
364       // If the intermediate type was expanded, build the intermediate
365       // operands from the parts.
366       assert(NumParts % NumIntermediates == 0 &&
367              "Must expand into a divisible number of parts!");
368       unsigned Factor = NumParts / NumIntermediates;
369       for (unsigned i = 0; i != NumIntermediates; ++i)
370         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
371                                   PartVT, IntermediateVT, V, CallConv);
372     }
373 
374     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
375     // intermediate operands.
376     EVT BuiltVectorTy =
377         IntermediateVT.isVector()
378             ? EVT::getVectorVT(
379                   *DAG.getContext(), IntermediateVT.getScalarType(),
380                   IntermediateVT.getVectorElementCount() * NumParts)
381             : EVT::getVectorVT(*DAG.getContext(),
382                                IntermediateVT.getScalarType(),
383                                NumIntermediates);
384     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
385                                                 : ISD::BUILD_VECTOR,
386                       DL, BuiltVectorTy, Ops);
387   }
388 
389   // There is now one part, held in Val.  Correct it to match ValueVT.
390   EVT PartEVT = Val.getValueType();
391 
392   if (PartEVT == ValueVT)
393     return Val;
394 
395   if (PartEVT.isVector()) {
396     // Vector/Vector bitcast.
397     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
398       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
399 
400     // If the element type of the source/dest vectors are the same, but the
401     // parts vector has more elements than the value vector, then we have a
402     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
403     // elements we want.
404     if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
405       assert((PartEVT.getVectorElementCount().getKnownMinValue() >
406               ValueVT.getVectorElementCount().getKnownMinValue()) &&
407              (PartEVT.getVectorElementCount().isScalable() ==
408               ValueVT.getVectorElementCount().isScalable()) &&
409              "Cannot narrow, it would be a lossy transformation");
410       PartEVT =
411           EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
412                            ValueVT.getVectorElementCount());
413       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
414                         DAG.getVectorIdxConstant(0, DL));
415       if (PartEVT == ValueVT)
416         return Val;
417     }
418 
419     // Promoted vector extract
420     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
421   }
422 
423   // Trivial bitcast if the types are the same size and the destination
424   // vector type is legal.
425   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
426       TLI.isTypeLegal(ValueVT))
427     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
428 
429   if (ValueVT.getVectorNumElements() != 1) {
430      // Certain ABIs require that vectors are passed as integers. For vectors
431      // are the same size, this is an obvious bitcast.
432      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
433        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
434      } else if (ValueVT.bitsLT(PartEVT)) {
435        const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
436        EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
437        // Drop the extra bits.
438        Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
439        return DAG.getBitcast(ValueVT, Val);
440      }
441 
442      diagnosePossiblyInvalidConstraint(
443          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
444      return DAG.getUNDEF(ValueVT);
445   }
446 
447   // Handle cases such as i8 -> <1 x i1>
448   EVT ValueSVT = ValueVT.getVectorElementType();
449   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
450     if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())
451       Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
452     else
453       Val = ValueVT.isFloatingPoint()
454                 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
455                 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
456   }
457 
458   return DAG.getBuildVector(ValueVT, DL, Val);
459 }
460 
461 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
462                                  SDValue Val, SDValue *Parts, unsigned NumParts,
463                                  MVT PartVT, const Value *V,
464                                  Optional<CallingConv::ID> CallConv);
465 
466 /// getCopyToParts - Create a series of nodes that contain the specified value
467 /// split into legal parts.  If the parts contain more bits than Val, then, for
468 /// integers, ExtendKind can be used to specify how to generate the extra bits.
469 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
470                            SDValue *Parts, unsigned NumParts, MVT PartVT,
471                            const Value *V,
472                            Optional<CallingConv::ID> CallConv = None,
473                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
474   // Let the target split the parts if it wants to
475   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
476   if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
477                                       CallConv))
478     return;
479   EVT ValueVT = Val.getValueType();
480 
481   // Handle the vector case separately.
482   if (ValueVT.isVector())
483     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
484                                 CallConv);
485 
486   unsigned PartBits = PartVT.getSizeInBits();
487   unsigned OrigNumParts = NumParts;
488   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
489          "Copying to an illegal type!");
490 
491   if (NumParts == 0)
492     return;
493 
494   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
495   EVT PartEVT = PartVT;
496   if (PartEVT == ValueVT) {
497     assert(NumParts == 1 && "No-op copy with multiple parts!");
498     Parts[0] = Val;
499     return;
500   }
501 
502   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
503     // If the parts cover more bits than the value has, promote the value.
504     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
505       assert(NumParts == 1 && "Do not know what to promote to!");
506       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
507     } else {
508       if (ValueVT.isFloatingPoint()) {
509         // FP values need to be bitcast, then extended if they are being put
510         // into a larger container.
511         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
512         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
513       }
514       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
515              ValueVT.isInteger() &&
516              "Unknown mismatch!");
517       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
518       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
519       if (PartVT == MVT::x86mmx)
520         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
521     }
522   } else if (PartBits == ValueVT.getSizeInBits()) {
523     // Different types of the same size.
524     assert(NumParts == 1 && PartEVT != ValueVT);
525     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
526   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
527     // If the parts cover less bits than value has, truncate the value.
528     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529            ValueVT.isInteger() &&
530            "Unknown mismatch!");
531     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
533     if (PartVT == MVT::x86mmx)
534       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535   }
536 
537   // The value may have changed - recompute ValueVT.
538   ValueVT = Val.getValueType();
539   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
540          "Failed to tile the value with PartVT!");
541 
542   if (NumParts == 1) {
543     if (PartEVT != ValueVT) {
544       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
545                                         "scalar-to-vector conversion failed");
546       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547     }
548 
549     Parts[0] = Val;
550     return;
551   }
552 
553   // Expand the value into multiple parts.
554   if (NumParts & (NumParts - 1)) {
555     // The number of parts is not a power of 2.  Split off and copy the tail.
556     assert(PartVT.isInteger() && ValueVT.isInteger() &&
557            "Do not know what to expand to!");
558     unsigned RoundParts = 1 << Log2_32(NumParts);
559     unsigned RoundBits = RoundParts * PartBits;
560     unsigned OddParts = NumParts - RoundParts;
561     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
562       DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
563 
564     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
565                    CallConv);
566 
567     if (DAG.getDataLayout().isBigEndian())
568       // The odd parts were reversed by getCopyToParts - unreverse them.
569       std::reverse(Parts + RoundParts, Parts + NumParts);
570 
571     NumParts = RoundParts;
572     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
573     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
574   }
575 
576   // The number of parts is a power of 2.  Repeatedly bisect the value using
577   // EXTRACT_ELEMENT.
578   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
579                          EVT::getIntegerVT(*DAG.getContext(),
580                                            ValueVT.getSizeInBits()),
581                          Val);
582 
583   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
584     for (unsigned i = 0; i < NumParts; i += StepSize) {
585       unsigned ThisBits = StepSize * PartBits / 2;
586       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
587       SDValue &Part0 = Parts[i];
588       SDValue &Part1 = Parts[i+StepSize/2];
589 
590       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
591                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
592       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
593                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
594 
595       if (ThisBits == PartBits && ThisVT != PartVT) {
596         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
597         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
598       }
599     }
600   }
601 
602   if (DAG.getDataLayout().isBigEndian())
603     std::reverse(Parts, Parts + OrigNumParts);
604 }
605 
606 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
607                                      const SDLoc &DL, EVT PartVT) {
608   if (!PartVT.isVector())
609     return SDValue();
610 
611   EVT ValueVT = Val.getValueType();
612   ElementCount PartNumElts = PartVT.getVectorElementCount();
613   ElementCount ValueNumElts = ValueVT.getVectorElementCount();
614 
615   // We only support widening vectors with equivalent element types and
616   // fixed/scalable properties. If a target needs to widen a fixed-length type
617   // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
618   if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
619       PartNumElts.isScalable() != ValueNumElts.isScalable() ||
620       PartVT.getVectorElementType() != ValueVT.getVectorElementType())
621     return SDValue();
622 
623   // Widening a scalable vector to another scalable vector is done by inserting
624   // the vector into a larger undef one.
625   if (PartNumElts.isScalable())
626     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
627                        Val, DAG.getVectorIdxConstant(0, DL));
628 
629   EVT ElementVT = PartVT.getVectorElementType();
630   // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
631   // undef elements.
632   SmallVector<SDValue, 16> Ops;
633   DAG.ExtractVectorElements(Val, Ops);
634   SDValue EltUndef = DAG.getUNDEF(ElementVT);
635   Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
636 
637   // FIXME: Use CONCAT for 2x -> 4x.
638   return DAG.getBuildVector(PartVT, DL, Ops);
639 }
640 
641 /// getCopyToPartsVector - Create a series of nodes that contain the specified
642 /// value split into legal parts.
643 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
644                                  SDValue Val, SDValue *Parts, unsigned NumParts,
645                                  MVT PartVT, const Value *V,
646                                  Optional<CallingConv::ID> CallConv) {
647   EVT ValueVT = Val.getValueType();
648   assert(ValueVT.isVector() && "Not a vector");
649   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
650   const bool IsABIRegCopy = CallConv.has_value();
651 
652   if (NumParts == 1) {
653     EVT PartEVT = PartVT;
654     if (PartEVT == ValueVT) {
655       // Nothing to do.
656     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
657       // Bitconvert vector->vector case.
658       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
659     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
660       Val = Widened;
661     } else if (PartVT.isVector() &&
662                PartEVT.getVectorElementType().bitsGE(
663                    ValueVT.getVectorElementType()) &&
664                PartEVT.getVectorElementCount() ==
665                    ValueVT.getVectorElementCount()) {
666 
667       // Promoted vector extract
668       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
669     } else if (PartEVT.isVector() &&
670                PartEVT.getVectorElementType() !=
671                    ValueVT.getVectorElementType() &&
672                TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
673                    TargetLowering::TypeWidenVector) {
674       // Combination of widening and promotion.
675       EVT WidenVT =
676           EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
677                            PartVT.getVectorElementCount());
678       SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
679       Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
680     } else {
681       if (ValueVT.getVectorElementCount().isScalar()) {
682         Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
683                           DAG.getVectorIdxConstant(0, DL));
684       } else {
685         uint64_t ValueSize = ValueVT.getFixedSizeInBits();
686         assert(PartVT.getFixedSizeInBits() > ValueSize &&
687                "lossy conversion of vector to scalar type");
688         EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
689         Val = DAG.getBitcast(IntermediateType, Val);
690         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
691       }
692     }
693 
694     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
695     Parts[0] = Val;
696     return;
697   }
698 
699   // Handle a multi-element vector.
700   EVT IntermediateVT;
701   MVT RegisterVT;
702   unsigned NumIntermediates;
703   unsigned NumRegs;
704   if (IsABIRegCopy) {
705     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
706         *DAG.getContext(), CallConv.value(), ValueVT, IntermediateVT,
707         NumIntermediates, RegisterVT);
708   } else {
709     NumRegs =
710         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
711                                    NumIntermediates, RegisterVT);
712   }
713 
714   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
715   NumParts = NumRegs; // Silence a compiler warning.
716   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
717 
718   assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
719          "Mixing scalable and fixed vectors when copying in parts");
720 
721   Optional<ElementCount> DestEltCnt;
722 
723   if (IntermediateVT.isVector())
724     DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
725   else
726     DestEltCnt = ElementCount::getFixed(NumIntermediates);
727 
728   EVT BuiltVectorTy = EVT::getVectorVT(
729       *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
730 
731   if (ValueVT == BuiltVectorTy) {
732     // Nothing to do.
733   } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
734     // Bitconvert vector->vector case.
735     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
736   } else {
737     if (BuiltVectorTy.getVectorElementType().bitsGT(
738             ValueVT.getVectorElementType())) {
739       // Integer promotion.
740       ValueVT = EVT::getVectorVT(*DAG.getContext(),
741                                  BuiltVectorTy.getVectorElementType(),
742                                  ValueVT.getVectorElementCount());
743       Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
744     }
745 
746     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
747       Val = Widened;
748     }
749   }
750 
751   assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
752 
753   // Split the vector into intermediate operands.
754   SmallVector<SDValue, 8> Ops(NumIntermediates);
755   for (unsigned i = 0; i != NumIntermediates; ++i) {
756     if (IntermediateVT.isVector()) {
757       // This does something sensible for scalable vectors - see the
758       // definition of EXTRACT_SUBVECTOR for further details.
759       unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
760       Ops[i] =
761           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
762                       DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
763     } else {
764       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
765                            DAG.getVectorIdxConstant(i, DL));
766     }
767   }
768 
769   // Split the intermediate operands into legal parts.
770   if (NumParts == NumIntermediates) {
771     // If the register was not expanded, promote or copy the value,
772     // as appropriate.
773     for (unsigned i = 0; i != NumParts; ++i)
774       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
775   } else if (NumParts > 0) {
776     // If the intermediate type was expanded, split each the value into
777     // legal parts.
778     assert(NumIntermediates != 0 && "division by zero");
779     assert(NumParts % NumIntermediates == 0 &&
780            "Must expand into a divisible number of parts!");
781     unsigned Factor = NumParts / NumIntermediates;
782     for (unsigned i = 0; i != NumIntermediates; ++i)
783       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
784                      CallConv);
785   }
786 }
787 
788 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
789                            EVT valuevt, Optional<CallingConv::ID> CC)
790     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
791       RegCount(1, regs.size()), CallConv(CC) {}
792 
793 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
794                            const DataLayout &DL, unsigned Reg, Type *Ty,
795                            Optional<CallingConv::ID> CC) {
796   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
797 
798   CallConv = CC;
799 
800   for (EVT ValueVT : ValueVTs) {
801     unsigned NumRegs =
802         isABIMangled()
803             ? TLI.getNumRegistersForCallingConv(Context, CC.value(), ValueVT)
804             : TLI.getNumRegisters(Context, ValueVT);
805     MVT RegisterVT =
806         isABIMangled()
807             ? TLI.getRegisterTypeForCallingConv(Context, CC.value(), ValueVT)
808             : TLI.getRegisterType(Context, ValueVT);
809     for (unsigned i = 0; i != NumRegs; ++i)
810       Regs.push_back(Reg + i);
811     RegVTs.push_back(RegisterVT);
812     RegCount.push_back(NumRegs);
813     Reg += NumRegs;
814   }
815 }
816 
817 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
818                                       FunctionLoweringInfo &FuncInfo,
819                                       const SDLoc &dl, SDValue &Chain,
820                                       SDValue *Flag, const Value *V) const {
821   // A Value with type {} or [0 x %t] needs no registers.
822   if (ValueVTs.empty())
823     return SDValue();
824 
825   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
826 
827   // Assemble the legal parts into the final values.
828   SmallVector<SDValue, 4> Values(ValueVTs.size());
829   SmallVector<SDValue, 8> Parts;
830   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
831     // Copy the legal parts from the registers.
832     EVT ValueVT = ValueVTs[Value];
833     unsigned NumRegs = RegCount[Value];
834     MVT RegisterVT =
835         isABIMangled() ? TLI.getRegisterTypeForCallingConv(
836                              *DAG.getContext(), CallConv.value(), RegVTs[Value])
837                        : RegVTs[Value];
838 
839     Parts.resize(NumRegs);
840     for (unsigned i = 0; i != NumRegs; ++i) {
841       SDValue P;
842       if (!Flag) {
843         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
844       } else {
845         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
846         *Flag = P.getValue(2);
847       }
848 
849       Chain = P.getValue(1);
850       Parts[i] = P;
851 
852       // If the source register was virtual and if we know something about it,
853       // add an assert node.
854       if (!Register::isVirtualRegister(Regs[Part + i]) ||
855           !RegisterVT.isInteger())
856         continue;
857 
858       const FunctionLoweringInfo::LiveOutInfo *LOI =
859         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
860       if (!LOI)
861         continue;
862 
863       unsigned RegSize = RegisterVT.getScalarSizeInBits();
864       unsigned NumSignBits = LOI->NumSignBits;
865       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
866 
867       if (NumZeroBits == RegSize) {
868         // The current value is a zero.
869         // Explicitly express that as it would be easier for
870         // optimizations to kick in.
871         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
872         continue;
873       }
874 
875       // FIXME: We capture more information than the dag can represent.  For
876       // now, just use the tightest assertzext/assertsext possible.
877       bool isSExt;
878       EVT FromVT(MVT::Other);
879       if (NumZeroBits) {
880         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
881         isSExt = false;
882       } else if (NumSignBits > 1) {
883         FromVT =
884             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
885         isSExt = true;
886       } else {
887         continue;
888       }
889       // Add an assertion node.
890       assert(FromVT != MVT::Other);
891       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
892                              RegisterVT, P, DAG.getValueType(FromVT));
893     }
894 
895     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
896                                      RegisterVT, ValueVT, V, CallConv);
897     Part += NumRegs;
898     Parts.clear();
899   }
900 
901   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
902 }
903 
904 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
905                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
906                                  const Value *V,
907                                  ISD::NodeType PreferredExtendType) const {
908   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
909   ISD::NodeType ExtendKind = PreferredExtendType;
910 
911   // Get the list of the values's legal parts.
912   unsigned NumRegs = Regs.size();
913   SmallVector<SDValue, 8> Parts(NumRegs);
914   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
915     unsigned NumParts = RegCount[Value];
916 
917     MVT RegisterVT =
918         isABIMangled() ? TLI.getRegisterTypeForCallingConv(
919                              *DAG.getContext(), CallConv.value(), RegVTs[Value])
920                        : RegVTs[Value];
921 
922     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
923       ExtendKind = ISD::ZERO_EXTEND;
924 
925     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
926                    NumParts, RegisterVT, V, CallConv, ExtendKind);
927     Part += NumParts;
928   }
929 
930   // Copy the parts into the registers.
931   SmallVector<SDValue, 8> Chains(NumRegs);
932   for (unsigned i = 0; i != NumRegs; ++i) {
933     SDValue Part;
934     if (!Flag) {
935       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
936     } else {
937       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
938       *Flag = Part.getValue(1);
939     }
940 
941     Chains[i] = Part.getValue(0);
942   }
943 
944   if (NumRegs == 1 || Flag)
945     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
946     // flagged to it. That is the CopyToReg nodes and the user are considered
947     // a single scheduling unit. If we create a TokenFactor and return it as
948     // chain, then the TokenFactor is both a predecessor (operand) of the
949     // user as well as a successor (the TF operands are flagged to the user).
950     // c1, f1 = CopyToReg
951     // c2, f2 = CopyToReg
952     // c3     = TokenFactor c1, c2
953     // ...
954     //        = op c3, ..., f2
955     Chain = Chains[NumRegs-1];
956   else
957     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
958 }
959 
960 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
961                                         unsigned MatchingIdx, const SDLoc &dl,
962                                         SelectionDAG &DAG,
963                                         std::vector<SDValue> &Ops) const {
964   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
965 
966   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
967   if (HasMatching)
968     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
969   else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
970     // Put the register class of the virtual registers in the flag word.  That
971     // way, later passes can recompute register class constraints for inline
972     // assembly as well as normal instructions.
973     // Don't do this for tied operands that can use the regclass information
974     // from the def.
975     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
976     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
977     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
978   }
979 
980   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
981   Ops.push_back(Res);
982 
983   if (Code == InlineAsm::Kind_Clobber) {
984     // Clobbers should always have a 1:1 mapping with registers, and may
985     // reference registers that have illegal (e.g. vector) types. Hence, we
986     // shouldn't try to apply any sort of splitting logic to them.
987     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
988            "No 1:1 mapping from clobbers to regs?");
989     Register SP = TLI.getStackPointerRegisterToSaveRestore();
990     (void)SP;
991     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
992       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
993       assert(
994           (Regs[I] != SP ||
995            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
996           "If we clobbered the stack pointer, MFI should know about it.");
997     }
998     return;
999   }
1000 
1001   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1002     MVT RegisterVT = RegVTs[Value];
1003     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1004                                            RegisterVT);
1005     for (unsigned i = 0; i != NumRegs; ++i) {
1006       assert(Reg < Regs.size() && "Mismatch in # registers expected");
1007       unsigned TheReg = Regs[Reg++];
1008       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1009     }
1010   }
1011 }
1012 
1013 SmallVector<std::pair<unsigned, TypeSize>, 4>
1014 RegsForValue::getRegsAndSizes() const {
1015   SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
1016   unsigned I = 0;
1017   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1018     unsigned RegCount = std::get<0>(CountAndVT);
1019     MVT RegisterVT = std::get<1>(CountAndVT);
1020     TypeSize RegisterSize = RegisterVT.getSizeInBits();
1021     for (unsigned E = I + RegCount; I != E; ++I)
1022       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1023   }
1024   return OutVec;
1025 }
1026 
1027 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1028                                const TargetLibraryInfo *li) {
1029   AA = aa;
1030   GFI = gfi;
1031   LibInfo = li;
1032   Context = DAG.getContext();
1033   LPadToCallSiteMap.clear();
1034   SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1035 }
1036 
1037 void SelectionDAGBuilder::clear() {
1038   NodeMap.clear();
1039   UnusedArgNodeMap.clear();
1040   PendingLoads.clear();
1041   PendingExports.clear();
1042   PendingConstrainedFP.clear();
1043   PendingConstrainedFPStrict.clear();
1044   CurInst = nullptr;
1045   HasTailCall = false;
1046   SDNodeOrder = LowestSDNodeOrder;
1047   StatepointLowering.clear();
1048 }
1049 
1050 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1051   DanglingDebugInfoMap.clear();
1052 }
1053 
1054 // Update DAG root to include dependencies on Pending chains.
1055 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1056   SDValue Root = DAG.getRoot();
1057 
1058   if (Pending.empty())
1059     return Root;
1060 
1061   // Add current root to PendingChains, unless we already indirectly
1062   // depend on it.
1063   if (Root.getOpcode() != ISD::EntryToken) {
1064     unsigned i = 0, e = Pending.size();
1065     for (; i != e; ++i) {
1066       assert(Pending[i].getNode()->getNumOperands() > 1);
1067       if (Pending[i].getNode()->getOperand(0) == Root)
1068         break;  // Don't add the root if we already indirectly depend on it.
1069     }
1070 
1071     if (i == e)
1072       Pending.push_back(Root);
1073   }
1074 
1075   if (Pending.size() == 1)
1076     Root = Pending[0];
1077   else
1078     Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1079 
1080   DAG.setRoot(Root);
1081   Pending.clear();
1082   return Root;
1083 }
1084 
1085 SDValue SelectionDAGBuilder::getMemoryRoot() {
1086   return updateRoot(PendingLoads);
1087 }
1088 
1089 SDValue SelectionDAGBuilder::getRoot() {
1090   // Chain up all pending constrained intrinsics together with all
1091   // pending loads, by simply appending them to PendingLoads and
1092   // then calling getMemoryRoot().
1093   PendingLoads.reserve(PendingLoads.size() +
1094                        PendingConstrainedFP.size() +
1095                        PendingConstrainedFPStrict.size());
1096   PendingLoads.append(PendingConstrainedFP.begin(),
1097                       PendingConstrainedFP.end());
1098   PendingLoads.append(PendingConstrainedFPStrict.begin(),
1099                       PendingConstrainedFPStrict.end());
1100   PendingConstrainedFP.clear();
1101   PendingConstrainedFPStrict.clear();
1102   return getMemoryRoot();
1103 }
1104 
1105 SDValue SelectionDAGBuilder::getControlRoot() {
1106   // We need to emit pending fpexcept.strict constrained intrinsics,
1107   // so append them to the PendingExports list.
1108   PendingExports.append(PendingConstrainedFPStrict.begin(),
1109                         PendingConstrainedFPStrict.end());
1110   PendingConstrainedFPStrict.clear();
1111   return updateRoot(PendingExports);
1112 }
1113 
1114 void SelectionDAGBuilder::visit(const Instruction &I) {
1115   // Set up outgoing PHI node register values before emitting the terminator.
1116   if (I.isTerminator()) {
1117     HandlePHINodesInSuccessorBlocks(I.getParent());
1118   }
1119 
1120   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1121   if (!isa<DbgInfoIntrinsic>(I))
1122     ++SDNodeOrder;
1123 
1124   CurInst = &I;
1125 
1126   visit(I.getOpcode(), I);
1127 
1128   if (!I.isTerminator() && !HasTailCall &&
1129       !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1130     CopyToExportRegsIfNeeded(&I);
1131 
1132   CurInst = nullptr;
1133 }
1134 
1135 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1136   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1137 }
1138 
1139 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1140   // Note: this doesn't use InstVisitor, because it has to work with
1141   // ConstantExpr's in addition to instructions.
1142   switch (Opcode) {
1143   default: llvm_unreachable("Unknown instruction type encountered!");
1144     // Build the switch statement using the Instruction.def file.
1145 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1146     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1147 #include "llvm/IR/Instruction.def"
1148   }
1149 }
1150 
1151 void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
1152                                                DebugLoc DL, unsigned Order) {
1153   // We treat variadic dbg_values differently at this stage.
1154   if (DI->hasArgList()) {
1155     // For variadic dbg_values we will now insert an undef.
1156     // FIXME: We can potentially recover these!
1157     SmallVector<SDDbgOperand, 2> Locs;
1158     for (const Value *V : DI->getValues()) {
1159       auto Undef = UndefValue::get(V->getType());
1160       Locs.push_back(SDDbgOperand::fromConst(Undef));
1161     }
1162     SDDbgValue *SDV = DAG.getDbgValueList(
1163         DI->getVariable(), DI->getExpression(), Locs, {},
1164         /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);
1165     DAG.AddDbgValue(SDV, /*isParameter=*/false);
1166   } else {
1167     // TODO: Dangling debug info will eventually either be resolved or produce
1168     // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1169     // between the original dbg.value location and its resolved DBG_VALUE,
1170     // which we should ideally fill with an extra Undef DBG_VALUE.
1171     assert(DI->getNumVariableLocationOps() == 1 &&
1172            "DbgValueInst without an ArgList should have a single location "
1173            "operand.");
1174     DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);
1175   }
1176 }
1177 
1178 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1179                                                 const DIExpression *Expr) {
1180   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1181     const DbgValueInst *DI = DDI.getDI();
1182     DIVariable *DanglingVariable = DI->getVariable();
1183     DIExpression *DanglingExpr = DI->getExpression();
1184     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1185       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1186       return true;
1187     }
1188     return false;
1189   };
1190 
1191   for (auto &DDIMI : DanglingDebugInfoMap) {
1192     DanglingDebugInfoVector &DDIV = DDIMI.second;
1193 
1194     // If debug info is to be dropped, run it through final checks to see
1195     // whether it can be salvaged.
1196     for (auto &DDI : DDIV)
1197       if (isMatchingDbgValue(DDI))
1198         salvageUnresolvedDbgValue(DDI);
1199 
1200     erase_if(DDIV, isMatchingDbgValue);
1201   }
1202 }
1203 
1204 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1205 // generate the debug data structures now that we've seen its definition.
1206 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1207                                                    SDValue Val) {
1208   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1209   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1210     return;
1211 
1212   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1213   for (auto &DDI : DDIV) {
1214     const DbgValueInst *DI = DDI.getDI();
1215     assert(!DI->hasArgList() && "Not implemented for variadic dbg_values");
1216     assert(DI && "Ill-formed DanglingDebugInfo");
1217     DebugLoc dl = DDI.getdl();
1218     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1219     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1220     DILocalVariable *Variable = DI->getVariable();
1221     DIExpression *Expr = DI->getExpression();
1222     assert(Variable->isValidLocationForIntrinsic(dl) &&
1223            "Expected inlined-at fields to agree");
1224     SDDbgValue *SDV;
1225     if (Val.getNode()) {
1226       // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1227       // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1228       // we couldn't resolve it directly when examining the DbgValue intrinsic
1229       // in the first place we should not be more successful here). Unless we
1230       // have some test case that prove this to be correct we should avoid
1231       // calling EmitFuncArgumentDbgValue here.
1232       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl,
1233                                     FuncArgumentDbgValueKind::Value, Val)) {
1234         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1235                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1236         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1237         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1238         // inserted after the definition of Val when emitting the instructions
1239         // after ISel. An alternative could be to teach
1240         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1241         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1242                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1243                    << ValSDNodeOrder << "\n");
1244         SDV = getDbgValue(Val, Variable, Expr, dl,
1245                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1246         DAG.AddDbgValue(SDV, false);
1247       } else
1248         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1249                           << "in EmitFuncArgumentDbgValue\n");
1250     } else {
1251       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1252       auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1253       auto SDV =
1254           DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1255       DAG.AddDbgValue(SDV, false);
1256     }
1257   }
1258   DDIV.clear();
1259 }
1260 
1261 void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1262   // TODO: For the variadic implementation, instead of only checking the fail
1263   // state of `handleDebugValue`, we need know specifically which values were
1264   // invalid, so that we attempt to salvage only those values when processing
1265   // a DIArgList.
1266   assert(!DDI.getDI()->hasArgList() &&
1267          "Not implemented for variadic dbg_values");
1268   Value *V = DDI.getDI()->getValue(0);
1269   DILocalVariable *Var = DDI.getDI()->getVariable();
1270   DIExpression *Expr = DDI.getDI()->getExpression();
1271   DebugLoc DL = DDI.getdl();
1272   DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1273   unsigned SDOrder = DDI.getSDNodeOrder();
1274   // Currently we consider only dbg.value intrinsics -- we tell the salvager
1275   // that DW_OP_stack_value is desired.
1276   assert(isa<DbgValueInst>(DDI.getDI()));
1277   bool StackValue = true;
1278 
1279   // Can this Value can be encoded without any further work?
1280   if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))
1281     return;
1282 
1283   // Attempt to salvage back through as many instructions as possible. Bail if
1284   // a non-instruction is seen, such as a constant expression or global
1285   // variable. FIXME: Further work could recover those too.
1286   while (isa<Instruction>(V)) {
1287     Instruction &VAsInst = *cast<Instruction>(V);
1288     // Temporary "0", awaiting real implementation.
1289     SmallVector<uint64_t, 16> Ops;
1290     SmallVector<Value *, 4> AdditionalValues;
1291     V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1292                              AdditionalValues);
1293     // If we cannot salvage any further, and haven't yet found a suitable debug
1294     // expression, bail out.
1295     if (!V)
1296       break;
1297 
1298     // TODO: If AdditionalValues isn't empty, then the salvage can only be
1299     // represented with a DBG_VALUE_LIST, so we give up. When we have support
1300     // here for variadic dbg_values, remove that condition.
1301     if (!AdditionalValues.empty())
1302       break;
1303 
1304     // New value and expr now represent this debuginfo.
1305     Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1306 
1307     // Some kind of simplification occurred: check whether the operand of the
1308     // salvaged debug expression can be encoded in this DAG.
1309     if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,
1310                          /*IsVariadic=*/false)) {
1311       LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n  "
1312                         << *DDI.getDI() << "\nBy stripping back to:\n  " << *V);
1313       return;
1314     }
1315   }
1316 
1317   // This was the final opportunity to salvage this debug information, and it
1318   // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1319   // any earlier variable location.
1320   auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1321   auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1322   DAG.AddDbgValue(SDV, false);
1323 
1324   LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << *DDI.getDI()
1325                     << "\n");
1326   LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *DDI.getDI()->getOperand(0)
1327                     << "\n");
1328 }
1329 
1330 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1331                                            DILocalVariable *Var,
1332                                            DIExpression *Expr, DebugLoc dl,
1333                                            DebugLoc InstDL, unsigned Order,
1334                                            bool IsVariadic) {
1335   if (Values.empty())
1336     return true;
1337   SmallVector<SDDbgOperand> LocationOps;
1338   SmallVector<SDNode *> Dependencies;
1339   for (const Value *V : Values) {
1340     // Constant value.
1341     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1342         isa<ConstantPointerNull>(V)) {
1343       LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1344       continue;
1345     }
1346 
1347     // If the Value is a frame index, we can create a FrameIndex debug value
1348     // without relying on the DAG at all.
1349     if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1350       auto SI = FuncInfo.StaticAllocaMap.find(AI);
1351       if (SI != FuncInfo.StaticAllocaMap.end()) {
1352         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1353         continue;
1354       }
1355     }
1356 
1357     // Do not use getValue() in here; we don't want to generate code at
1358     // this point if it hasn't been done yet.
1359     SDValue N = NodeMap[V];
1360     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1361       N = UnusedArgNodeMap[V];
1362     if (N.getNode()) {
1363       // Only emit func arg dbg value for non-variadic dbg.values for now.
1364       if (!IsVariadic &&
1365           EmitFuncArgumentDbgValue(V, Var, Expr, dl,
1366                                    FuncArgumentDbgValueKind::Value, N))
1367         return true;
1368       if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1369         // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1370         // describe stack slot locations.
1371         //
1372         // Consider "int x = 0; int *px = &x;". There are two kinds of
1373         // interesting debug values here after optimization:
1374         //
1375         //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
1376         //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1377         //
1378         // Both describe the direct values of their associated variables.
1379         Dependencies.push_back(N.getNode());
1380         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1381         continue;
1382       }
1383       LocationOps.emplace_back(
1384           SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1385       continue;
1386     }
1387 
1388     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1389     // Special rules apply for the first dbg.values of parameter variables in a
1390     // function. Identify them by the fact they reference Argument Values, that
1391     // they're parameters, and they are parameters of the current function. We
1392     // need to let them dangle until they get an SDNode.
1393     bool IsParamOfFunc =
1394         isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();
1395     if (IsParamOfFunc)
1396       return false;
1397 
1398     // The value is not used in this block yet (or it would have an SDNode).
1399     // We still want the value to appear for the user if possible -- if it has
1400     // an associated VReg, we can refer to that instead.
1401     auto VMI = FuncInfo.ValueMap.find(V);
1402     if (VMI != FuncInfo.ValueMap.end()) {
1403       unsigned Reg = VMI->second;
1404       // If this is a PHI node, it may be split up into several MI PHI nodes
1405       // (in FunctionLoweringInfo::set).
1406       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1407                        V->getType(), None);
1408       if (RFV.occupiesMultipleRegs()) {
1409         // FIXME: We could potentially support variadic dbg_values here.
1410         if (IsVariadic)
1411           return false;
1412         unsigned Offset = 0;
1413         unsigned BitsToDescribe = 0;
1414         if (auto VarSize = Var->getSizeInBits())
1415           BitsToDescribe = *VarSize;
1416         if (auto Fragment = Expr->getFragmentInfo())
1417           BitsToDescribe = Fragment->SizeInBits;
1418         for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1419           // Bail out if all bits are described already.
1420           if (Offset >= BitsToDescribe)
1421             break;
1422           // TODO: handle scalable vectors.
1423           unsigned RegisterSize = RegAndSize.second;
1424           unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1425                                       ? BitsToDescribe - Offset
1426                                       : RegisterSize;
1427           auto FragmentExpr = DIExpression::createFragmentExpression(
1428               Expr, Offset, FragmentSize);
1429           if (!FragmentExpr)
1430             continue;
1431           SDDbgValue *SDV = DAG.getVRegDbgValue(
1432               Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);
1433           DAG.AddDbgValue(SDV, false);
1434           Offset += RegisterSize;
1435         }
1436         return true;
1437       }
1438       // We can use simple vreg locations for variadic dbg_values as well.
1439       LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1440       continue;
1441     }
1442     // We failed to create a SDDbgOperand for V.
1443     return false;
1444   }
1445 
1446   // We have created a SDDbgOperand for each Value in Values.
1447   // Should use Order instead of SDNodeOrder?
1448   assert(!LocationOps.empty());
1449   SDDbgValue *SDV =
1450       DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1451                           /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);
1452   DAG.AddDbgValue(SDV, /*isParameter=*/false);
1453   return true;
1454 }
1455 
1456 void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1457   // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1458   for (auto &Pair : DanglingDebugInfoMap)
1459     for (auto &DDI : Pair.second)
1460       salvageUnresolvedDbgValue(DDI);
1461   clearDanglingDebugInfo();
1462 }
1463 
1464 /// getCopyFromRegs - If there was virtual register allocated for the value V
1465 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1466 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1467   DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
1468   SDValue Result;
1469 
1470   if (It != FuncInfo.ValueMap.end()) {
1471     Register InReg = It->second;
1472 
1473     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1474                      DAG.getDataLayout(), InReg, Ty,
1475                      None); // This is not an ABI copy.
1476     SDValue Chain = DAG.getEntryNode();
1477     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1478                                  V);
1479     resolveDanglingDebugInfo(V, Result);
1480   }
1481 
1482   return Result;
1483 }
1484 
1485 /// getValue - Return an SDValue for the given Value.
1486 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1487   // If we already have an SDValue for this value, use it. It's important
1488   // to do this first, so that we don't create a CopyFromReg if we already
1489   // have a regular SDValue.
1490   SDValue &N = NodeMap[V];
1491   if (N.getNode()) return N;
1492 
1493   // If there's a virtual register allocated and initialized for this
1494   // value, use it.
1495   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1496     return copyFromReg;
1497 
1498   // Otherwise create a new SDValue and remember it.
1499   SDValue Val = getValueImpl(V);
1500   NodeMap[V] = Val;
1501   resolveDanglingDebugInfo(V, Val);
1502   return Val;
1503 }
1504 
1505 /// getNonRegisterValue - Return an SDValue for the given Value, but
1506 /// don't look in FuncInfo.ValueMap for a virtual register.
1507 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1508   // If we already have an SDValue for this value, use it.
1509   SDValue &N = NodeMap[V];
1510   if (N.getNode()) {
1511     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1512       // Remove the debug location from the node as the node is about to be used
1513       // in a location which may differ from the original debug location.  This
1514       // is relevant to Constant and ConstantFP nodes because they can appear
1515       // as constant expressions inside PHI nodes.
1516       N->setDebugLoc(DebugLoc());
1517     }
1518     return N;
1519   }
1520 
1521   // Otherwise create a new SDValue and remember it.
1522   SDValue Val = getValueImpl(V);
1523   NodeMap[V] = Val;
1524   resolveDanglingDebugInfo(V, Val);
1525   return Val;
1526 }
1527 
1528 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1529 /// Create an SDValue for the given value.
1530 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1531   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1532 
1533   if (const Constant *C = dyn_cast<Constant>(V)) {
1534     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1535 
1536     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1537       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1538 
1539     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1540       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1541 
1542     if (isa<ConstantPointerNull>(C)) {
1543       unsigned AS = V->getType()->getPointerAddressSpace();
1544       return DAG.getConstant(0, getCurSDLoc(),
1545                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1546     }
1547 
1548     if (match(C, m_VScale(DAG.getDataLayout())))
1549       return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1550 
1551     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1552       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1553 
1554     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1555       return DAG.getUNDEF(VT);
1556 
1557     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1558       visit(CE->getOpcode(), *CE);
1559       SDValue N1 = NodeMap[V];
1560       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1561       return N1;
1562     }
1563 
1564     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1565       SmallVector<SDValue, 4> Constants;
1566       for (const Use &U : C->operands()) {
1567         SDNode *Val = getValue(U).getNode();
1568         // If the operand is an empty aggregate, there are no values.
1569         if (!Val) continue;
1570         // Add each leaf value from the operand to the Constants list
1571         // to form a flattened list of all the values.
1572         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1573           Constants.push_back(SDValue(Val, i));
1574       }
1575 
1576       return DAG.getMergeValues(Constants, getCurSDLoc());
1577     }
1578 
1579     if (const ConstantDataSequential *CDS =
1580           dyn_cast<ConstantDataSequential>(C)) {
1581       SmallVector<SDValue, 4> Ops;
1582       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1583         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1584         // Add each leaf value from the operand to the Constants list
1585         // to form a flattened list of all the values.
1586         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1587           Ops.push_back(SDValue(Val, i));
1588       }
1589 
1590       if (isa<ArrayType>(CDS->getType()))
1591         return DAG.getMergeValues(Ops, getCurSDLoc());
1592       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1593     }
1594 
1595     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1596       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1597              "Unknown struct or array constant!");
1598 
1599       SmallVector<EVT, 4> ValueVTs;
1600       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1601       unsigned NumElts = ValueVTs.size();
1602       if (NumElts == 0)
1603         return SDValue(); // empty struct
1604       SmallVector<SDValue, 4> Constants(NumElts);
1605       for (unsigned i = 0; i != NumElts; ++i) {
1606         EVT EltVT = ValueVTs[i];
1607         if (isa<UndefValue>(C))
1608           Constants[i] = DAG.getUNDEF(EltVT);
1609         else if (EltVT.isFloatingPoint())
1610           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1611         else
1612           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1613       }
1614 
1615       return DAG.getMergeValues(Constants, getCurSDLoc());
1616     }
1617 
1618     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1619       return DAG.getBlockAddress(BA, VT);
1620 
1621     if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1622       return getValue(Equiv->getGlobalValue());
1623 
1624     if (const auto *NC = dyn_cast<NoCFIValue>(C))
1625       return getValue(NC->getGlobalValue());
1626 
1627     VectorType *VecTy = cast<VectorType>(V->getType());
1628 
1629     // Now that we know the number and type of the elements, get that number of
1630     // elements into the Ops array based on what kind of constant it is.
1631     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1632       SmallVector<SDValue, 16> Ops;
1633       unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1634       for (unsigned i = 0; i != NumElements; ++i)
1635         Ops.push_back(getValue(CV->getOperand(i)));
1636 
1637       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1638     }
1639 
1640     if (isa<ConstantAggregateZero>(C)) {
1641       EVT EltVT =
1642           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1643 
1644       SDValue Op;
1645       if (EltVT.isFloatingPoint())
1646         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1647       else
1648         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1649 
1650       if (isa<ScalableVectorType>(VecTy))
1651         return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);
1652 
1653       SmallVector<SDValue, 16> Ops;
1654       Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);
1655       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1656     }
1657 
1658     llvm_unreachable("Unknown vector constant");
1659   }
1660 
1661   // If this is a static alloca, generate it as the frameindex instead of
1662   // computation.
1663   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1664     DenseMap<const AllocaInst*, int>::iterator SI =
1665       FuncInfo.StaticAllocaMap.find(AI);
1666     if (SI != FuncInfo.StaticAllocaMap.end())
1667       return DAG.getFrameIndex(SI->second,
1668                                TLI.getFrameIndexTy(DAG.getDataLayout()));
1669   }
1670 
1671   // If this is an instruction which fast-isel has deferred, select it now.
1672   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1673     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1674 
1675     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1676                      Inst->getType(), None);
1677     SDValue Chain = DAG.getEntryNode();
1678     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1679   }
1680 
1681   if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1682     return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1683 
1684   if (const auto *BB = dyn_cast<BasicBlock>(V))
1685     return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1686 
1687   llvm_unreachable("Can't get register for value!");
1688 }
1689 
1690 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1691   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1692   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1693   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1694   bool IsSEH = isAsynchronousEHPersonality(Pers);
1695   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1696   if (!IsSEH)
1697     CatchPadMBB->setIsEHScopeEntry();
1698   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1699   if (IsMSVCCXX || IsCoreCLR)
1700     CatchPadMBB->setIsEHFuncletEntry();
1701 }
1702 
1703 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1704   // Update machine-CFG edge.
1705   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1706   FuncInfo.MBB->addSuccessor(TargetMBB);
1707   TargetMBB->setIsEHCatchretTarget(true);
1708   DAG.getMachineFunction().setHasEHCatchret(true);
1709 
1710   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1711   bool IsSEH = isAsynchronousEHPersonality(Pers);
1712   if (IsSEH) {
1713     // If this is not a fall-through branch or optimizations are switched off,
1714     // emit the branch.
1715     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1716         TM.getOptLevel() == CodeGenOpt::None)
1717       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1718                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1719     return;
1720   }
1721 
1722   // Figure out the funclet membership for the catchret's successor.
1723   // This will be used by the FuncletLayout pass to determine how to order the
1724   // BB's.
1725   // A 'catchret' returns to the outer scope's color.
1726   Value *ParentPad = I.getCatchSwitchParentPad();
1727   const BasicBlock *SuccessorColor;
1728   if (isa<ConstantTokenNone>(ParentPad))
1729     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1730   else
1731     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1732   assert(SuccessorColor && "No parent funclet for catchret!");
1733   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1734   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1735 
1736   // Create the terminator node.
1737   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1738                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1739                             DAG.getBasicBlock(SuccessorColorMBB));
1740   DAG.setRoot(Ret);
1741 }
1742 
1743 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1744   // Don't emit any special code for the cleanuppad instruction. It just marks
1745   // the start of an EH scope/funclet.
1746   FuncInfo.MBB->setIsEHScopeEntry();
1747   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1748   if (Pers != EHPersonality::Wasm_CXX) {
1749     FuncInfo.MBB->setIsEHFuncletEntry();
1750     FuncInfo.MBB->setIsCleanupFuncletEntry();
1751   }
1752 }
1753 
1754 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1755 // not match, it is OK to add only the first unwind destination catchpad to the
1756 // successors, because there will be at least one invoke instruction within the
1757 // catch scope that points to the next unwind destination, if one exists, so
1758 // CFGSort cannot mess up with BB sorting order.
1759 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1760 // call within them, and catchpads only consisting of 'catch (...)' have a
1761 // '__cxa_end_catch' call within them, both of which generate invokes in case
1762 // the next unwind destination exists, i.e., the next unwind destination is not
1763 // the caller.)
1764 //
1765 // Having at most one EH pad successor is also simpler and helps later
1766 // transformations.
1767 //
1768 // For example,
1769 // current:
1770 //   invoke void @foo to ... unwind label %catch.dispatch
1771 // catch.dispatch:
1772 //   %0 = catchswitch within ... [label %catch.start] unwind label %next
1773 // catch.start:
1774 //   ...
1775 //   ... in this BB or some other child BB dominated by this BB there will be an
1776 //   invoke that points to 'next' BB as an unwind destination
1777 //
1778 // next: ; We don't need to add this to 'current' BB's successor
1779 //   ...
1780 static void findWasmUnwindDestinations(
1781     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1782     BranchProbability Prob,
1783     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1784         &UnwindDests) {
1785   while (EHPadBB) {
1786     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1787     if (isa<CleanupPadInst>(Pad)) {
1788       // Stop on cleanup pads.
1789       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1790       UnwindDests.back().first->setIsEHScopeEntry();
1791       break;
1792     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1793       // Add the catchpad handlers to the possible destinations. We don't
1794       // continue to the unwind destination of the catchswitch for wasm.
1795       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1796         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1797         UnwindDests.back().first->setIsEHScopeEntry();
1798       }
1799       break;
1800     } else {
1801       continue;
1802     }
1803   }
1804 }
1805 
1806 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1807 /// many places it could ultimately go. In the IR, we have a single unwind
1808 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1809 /// This function skips over imaginary basic blocks that hold catchswitch
1810 /// instructions, and finds all the "real" machine
1811 /// basic block destinations. As those destinations may not be successors of
1812 /// EHPadBB, here we also calculate the edge probability to those destinations.
1813 /// The passed-in Prob is the edge probability to EHPadBB.
1814 static void findUnwindDestinations(
1815     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1816     BranchProbability Prob,
1817     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1818         &UnwindDests) {
1819   EHPersonality Personality =
1820     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1821   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1822   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1823   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1824   bool IsSEH = isAsynchronousEHPersonality(Personality);
1825 
1826   if (IsWasmCXX) {
1827     findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1828     assert(UnwindDests.size() <= 1 &&
1829            "There should be at most one unwind destination for wasm");
1830     return;
1831   }
1832 
1833   while (EHPadBB) {
1834     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1835     BasicBlock *NewEHPadBB = nullptr;
1836     if (isa<LandingPadInst>(Pad)) {
1837       // Stop on landingpads. They are not funclets.
1838       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1839       break;
1840     } else if (isa<CleanupPadInst>(Pad)) {
1841       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1842       // personalities.
1843       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1844       UnwindDests.back().first->setIsEHScopeEntry();
1845       UnwindDests.back().first->setIsEHFuncletEntry();
1846       break;
1847     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1848       // Add the catchpad handlers to the possible destinations.
1849       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1850         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1851         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1852         if (IsMSVCCXX || IsCoreCLR)
1853           UnwindDests.back().first->setIsEHFuncletEntry();
1854         if (!IsSEH)
1855           UnwindDests.back().first->setIsEHScopeEntry();
1856       }
1857       NewEHPadBB = CatchSwitch->getUnwindDest();
1858     } else {
1859       continue;
1860     }
1861 
1862     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1863     if (BPI && NewEHPadBB)
1864       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1865     EHPadBB = NewEHPadBB;
1866   }
1867 }
1868 
1869 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1870   // Update successor info.
1871   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1872   auto UnwindDest = I.getUnwindDest();
1873   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1874   BranchProbability UnwindDestProb =
1875       (BPI && UnwindDest)
1876           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1877           : BranchProbability::getZero();
1878   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1879   for (auto &UnwindDest : UnwindDests) {
1880     UnwindDest.first->setIsEHPad();
1881     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1882   }
1883   FuncInfo.MBB->normalizeSuccProbs();
1884 
1885   // Create the terminator node.
1886   SDValue Ret =
1887       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1888   DAG.setRoot(Ret);
1889 }
1890 
1891 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1892   report_fatal_error("visitCatchSwitch not yet implemented!");
1893 }
1894 
1895 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1896   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1897   auto &DL = DAG.getDataLayout();
1898   SDValue Chain = getControlRoot();
1899   SmallVector<ISD::OutputArg, 8> Outs;
1900   SmallVector<SDValue, 8> OutVals;
1901 
1902   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1903   // lower
1904   //
1905   //   %val = call <ty> @llvm.experimental.deoptimize()
1906   //   ret <ty> %val
1907   //
1908   // differently.
1909   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1910     LowerDeoptimizingReturn();
1911     return;
1912   }
1913 
1914   if (!FuncInfo.CanLowerReturn) {
1915     unsigned DemoteReg = FuncInfo.DemoteRegister;
1916     const Function *F = I.getParent()->getParent();
1917 
1918     // Emit a store of the return value through the virtual register.
1919     // Leave Outs empty so that LowerReturn won't try to load return
1920     // registers the usual way.
1921     SmallVector<EVT, 1> PtrValueVTs;
1922     ComputeValueVTs(TLI, DL,
1923                     F->getReturnType()->getPointerTo(
1924                         DAG.getDataLayout().getAllocaAddrSpace()),
1925                     PtrValueVTs);
1926 
1927     SDValue RetPtr =
1928         DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
1929     SDValue RetOp = getValue(I.getOperand(0));
1930 
1931     SmallVector<EVT, 4> ValueVTs, MemVTs;
1932     SmallVector<uint64_t, 4> Offsets;
1933     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1934                     &Offsets);
1935     unsigned NumValues = ValueVTs.size();
1936 
1937     SmallVector<SDValue, 4> Chains(NumValues);
1938     Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1939     for (unsigned i = 0; i != NumValues; ++i) {
1940       // An aggregate return value cannot wrap around the address space, so
1941       // offsets to its parts don't wrap either.
1942       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
1943                                            TypeSize::Fixed(Offsets[i]));
1944 
1945       SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1946       if (MemVTs[i] != ValueVTs[i])
1947         Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1948       Chains[i] = DAG.getStore(
1949           Chain, getCurSDLoc(), Val,
1950           // FIXME: better loc info would be nice.
1951           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
1952           commonAlignment(BaseAlign, Offsets[i]));
1953     }
1954 
1955     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1956                         MVT::Other, Chains);
1957   } else if (I.getNumOperands() != 0) {
1958     SmallVector<EVT, 4> ValueVTs;
1959     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1960     unsigned NumValues = ValueVTs.size();
1961     if (NumValues) {
1962       SDValue RetOp = getValue(I.getOperand(0));
1963 
1964       const Function *F = I.getParent()->getParent();
1965 
1966       bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1967           I.getOperand(0)->getType(), F->getCallingConv(),
1968           /*IsVarArg*/ false, DL);
1969 
1970       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1971       if (F->getAttributes().hasRetAttr(Attribute::SExt))
1972         ExtendKind = ISD::SIGN_EXTEND;
1973       else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
1974         ExtendKind = ISD::ZERO_EXTEND;
1975 
1976       LLVMContext &Context = F->getContext();
1977       bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
1978 
1979       for (unsigned j = 0; j != NumValues; ++j) {
1980         EVT VT = ValueVTs[j];
1981 
1982         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1983           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1984 
1985         CallingConv::ID CC = F->getCallingConv();
1986 
1987         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1988         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1989         SmallVector<SDValue, 4> Parts(NumParts);
1990         getCopyToParts(DAG, getCurSDLoc(),
1991                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1992                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1993 
1994         // 'inreg' on function refers to return value
1995         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1996         if (RetInReg)
1997           Flags.setInReg();
1998 
1999         if (I.getOperand(0)->getType()->isPointerTy()) {
2000           Flags.setPointer();
2001           Flags.setPointerAddrSpace(
2002               cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2003         }
2004 
2005         if (NeedsRegBlock) {
2006           Flags.setInConsecutiveRegs();
2007           if (j == NumValues - 1)
2008             Flags.setInConsecutiveRegsLast();
2009         }
2010 
2011         // Propagate extension type if any
2012         if (ExtendKind == ISD::SIGN_EXTEND)
2013           Flags.setSExt();
2014         else if (ExtendKind == ISD::ZERO_EXTEND)
2015           Flags.setZExt();
2016 
2017         for (unsigned i = 0; i < NumParts; ++i) {
2018           Outs.push_back(ISD::OutputArg(Flags,
2019                                         Parts[i].getValueType().getSimpleVT(),
2020                                         VT, /*isfixed=*/true, 0, 0));
2021           OutVals.push_back(Parts[i]);
2022         }
2023       }
2024     }
2025   }
2026 
2027   // Push in swifterror virtual register as the last element of Outs. This makes
2028   // sure swifterror virtual register will be returned in the swifterror
2029   // physical register.
2030   const Function *F = I.getParent()->getParent();
2031   if (TLI.supportSwiftError() &&
2032       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2033     assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2034     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2035     Flags.setSwiftError();
2036     Outs.push_back(ISD::OutputArg(
2037         Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2038         /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2039     // Create SDNode for the swifterror virtual register.
2040     OutVals.push_back(
2041         DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2042                             &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2043                         EVT(TLI.getPointerTy(DL))));
2044   }
2045 
2046   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2047   CallingConv::ID CallConv =
2048     DAG.getMachineFunction().getFunction().getCallingConv();
2049   Chain = DAG.getTargetLoweringInfo().LowerReturn(
2050       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2051 
2052   // Verify that the target's LowerReturn behaved as expected.
2053   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2054          "LowerReturn didn't return a valid chain!");
2055 
2056   // Update the DAG with the new chain value resulting from return lowering.
2057   DAG.setRoot(Chain);
2058 }
2059 
2060 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2061 /// created for it, emit nodes to copy the value into the virtual
2062 /// registers.
2063 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2064   // Skip empty types
2065   if (V->getType()->isEmptyTy())
2066     return;
2067 
2068   DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
2069   if (VMI != FuncInfo.ValueMap.end()) {
2070     assert(!V->use_empty() && "Unused value assigned virtual registers!");
2071     CopyValueToVirtualRegister(V, VMI->second);
2072   }
2073 }
2074 
2075 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2076 /// the current basic block, add it to ValueMap now so that we'll get a
2077 /// CopyTo/FromReg.
2078 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2079   // No need to export constants.
2080   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2081 
2082   // Already exported?
2083   if (FuncInfo.isExportedInst(V)) return;
2084 
2085   unsigned Reg = FuncInfo.InitializeRegForValue(V);
2086   CopyValueToVirtualRegister(V, Reg);
2087 }
2088 
2089 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2090                                                      const BasicBlock *FromBB) {
2091   // The operands of the setcc have to be in this block.  We don't know
2092   // how to export them from some other block.
2093   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2094     // Can export from current BB.
2095     if (VI->getParent() == FromBB)
2096       return true;
2097 
2098     // Is already exported, noop.
2099     return FuncInfo.isExportedInst(V);
2100   }
2101 
2102   // If this is an argument, we can export it if the BB is the entry block or
2103   // if it is already exported.
2104   if (isa<Argument>(V)) {
2105     if (FromBB->isEntryBlock())
2106       return true;
2107 
2108     // Otherwise, can only export this if it is already exported.
2109     return FuncInfo.isExportedInst(V);
2110   }
2111 
2112   // Otherwise, constants can always be exported.
2113   return true;
2114 }
2115 
2116 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2117 BranchProbability
2118 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2119                                         const MachineBasicBlock *Dst) const {
2120   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2121   const BasicBlock *SrcBB = Src->getBasicBlock();
2122   const BasicBlock *DstBB = Dst->getBasicBlock();
2123   if (!BPI) {
2124     // If BPI is not available, set the default probability as 1 / N, where N is
2125     // the number of successors.
2126     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2127     return BranchProbability(1, SuccSize);
2128   }
2129   return BPI->getEdgeProbability(SrcBB, DstBB);
2130 }
2131 
2132 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2133                                                MachineBasicBlock *Dst,
2134                                                BranchProbability Prob) {
2135   if (!FuncInfo.BPI)
2136     Src->addSuccessorWithoutProb(Dst);
2137   else {
2138     if (Prob.isUnknown())
2139       Prob = getEdgeProbability(Src, Dst);
2140     Src->addSuccessor(Dst, Prob);
2141   }
2142 }
2143 
2144 static bool InBlock(const Value *V, const BasicBlock *BB) {
2145   if (const Instruction *I = dyn_cast<Instruction>(V))
2146     return I->getParent() == BB;
2147   return true;
2148 }
2149 
2150 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2151 /// This function emits a branch and is used at the leaves of an OR or an
2152 /// AND operator tree.
2153 void
2154 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2155                                                   MachineBasicBlock *TBB,
2156                                                   MachineBasicBlock *FBB,
2157                                                   MachineBasicBlock *CurBB,
2158                                                   MachineBasicBlock *SwitchBB,
2159                                                   BranchProbability TProb,
2160                                                   BranchProbability FProb,
2161                                                   bool InvertCond) {
2162   const BasicBlock *BB = CurBB->getBasicBlock();
2163 
2164   // If the leaf of the tree is a comparison, merge the condition into
2165   // the caseblock.
2166   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2167     // The operands of the cmp have to be in this block.  We don't know
2168     // how to export them from some other block.  If this is the first block
2169     // of the sequence, no exporting is needed.
2170     if (CurBB == SwitchBB ||
2171         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2172          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2173       ISD::CondCode Condition;
2174       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2175         ICmpInst::Predicate Pred =
2176             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2177         Condition = getICmpCondCode(Pred);
2178       } else {
2179         const FCmpInst *FC = cast<FCmpInst>(Cond);
2180         FCmpInst::Predicate Pred =
2181             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2182         Condition = getFCmpCondCode(Pred);
2183         if (TM.Options.NoNaNsFPMath)
2184           Condition = getFCmpCodeWithoutNaN(Condition);
2185       }
2186 
2187       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2188                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2189       SL->SwitchCases.push_back(CB);
2190       return;
2191     }
2192   }
2193 
2194   // Create a CaseBlock record representing this branch.
2195   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2196   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2197                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2198   SL->SwitchCases.push_back(CB);
2199 }
2200 
2201 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2202                                                MachineBasicBlock *TBB,
2203                                                MachineBasicBlock *FBB,
2204                                                MachineBasicBlock *CurBB,
2205                                                MachineBasicBlock *SwitchBB,
2206                                                Instruction::BinaryOps Opc,
2207                                                BranchProbability TProb,
2208                                                BranchProbability FProb,
2209                                                bool InvertCond) {
2210   // Skip over not part of the tree and remember to invert op and operands at
2211   // next level.
2212   Value *NotCond;
2213   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2214       InBlock(NotCond, CurBB->getBasicBlock())) {
2215     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2216                          !InvertCond);
2217     return;
2218   }
2219 
2220   const Instruction *BOp = dyn_cast<Instruction>(Cond);
2221   const Value *BOpOp0, *BOpOp1;
2222   // Compute the effective opcode for Cond, taking into account whether it needs
2223   // to be inverted, e.g.
2224   //   and (not (or A, B)), C
2225   // gets lowered as
2226   //   and (and (not A, not B), C)
2227   Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2228   if (BOp) {
2229     BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2230                ? Instruction::And
2231                : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2232                       ? Instruction::Or
2233                       : (Instruction::BinaryOps)0);
2234     if (InvertCond) {
2235       if (BOpc == Instruction::And)
2236         BOpc = Instruction::Or;
2237       else if (BOpc == Instruction::Or)
2238         BOpc = Instruction::And;
2239     }
2240   }
2241 
2242   // If this node is not part of the or/and tree, emit it as a branch.
2243   // Note that all nodes in the tree should have same opcode.
2244   bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2245   if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2246       !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2247       !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2248     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2249                                  TProb, FProb, InvertCond);
2250     return;
2251   }
2252 
2253   //  Create TmpBB after CurBB.
2254   MachineFunction::iterator BBI(CurBB);
2255   MachineFunction &MF = DAG.getMachineFunction();
2256   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2257   CurBB->getParent()->insert(++BBI, TmpBB);
2258 
2259   if (Opc == Instruction::Or) {
2260     // Codegen X | Y as:
2261     // BB1:
2262     //   jmp_if_X TBB
2263     //   jmp TmpBB
2264     // TmpBB:
2265     //   jmp_if_Y TBB
2266     //   jmp FBB
2267     //
2268 
2269     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2270     // The requirement is that
2271     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2272     //     = TrueProb for original BB.
2273     // Assuming the original probabilities are A and B, one choice is to set
2274     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2275     // A/(1+B) and 2B/(1+B). This choice assumes that
2276     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2277     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2278     // TmpBB, but the math is more complicated.
2279 
2280     auto NewTrueProb = TProb / 2;
2281     auto NewFalseProb = TProb / 2 + FProb;
2282     // Emit the LHS condition.
2283     FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2284                          NewFalseProb, InvertCond);
2285 
2286     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2287     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2288     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2289     // Emit the RHS condition into TmpBB.
2290     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2291                          Probs[1], InvertCond);
2292   } else {
2293     assert(Opc == Instruction::And && "Unknown merge op!");
2294     // Codegen X & Y as:
2295     // BB1:
2296     //   jmp_if_X TmpBB
2297     //   jmp FBB
2298     // TmpBB:
2299     //   jmp_if_Y TBB
2300     //   jmp FBB
2301     //
2302     //  This requires creation of TmpBB after CurBB.
2303 
2304     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2305     // The requirement is that
2306     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2307     //     = FalseProb for original BB.
2308     // Assuming the original probabilities are A and B, one choice is to set
2309     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2310     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2311     // TrueProb for BB1 * FalseProb for TmpBB.
2312 
2313     auto NewTrueProb = TProb + FProb / 2;
2314     auto NewFalseProb = FProb / 2;
2315     // Emit the LHS condition.
2316     FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2317                          NewFalseProb, InvertCond);
2318 
2319     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2320     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2321     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2322     // Emit the RHS condition into TmpBB.
2323     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2324                          Probs[1], InvertCond);
2325   }
2326 }
2327 
2328 /// If the set of cases should be emitted as a series of branches, return true.
2329 /// If we should emit this as a bunch of and/or'd together conditions, return
2330 /// false.
2331 bool
2332 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2333   if (Cases.size() != 2) return true;
2334 
2335   // If this is two comparisons of the same values or'd or and'd together, they
2336   // will get folded into a single comparison, so don't emit two blocks.
2337   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2338        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2339       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2340        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2341     return false;
2342   }
2343 
2344   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2345   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2346   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2347       Cases[0].CC == Cases[1].CC &&
2348       isa<Constant>(Cases[0].CmpRHS) &&
2349       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2350     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2351       return false;
2352     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2353       return false;
2354   }
2355 
2356   return true;
2357 }
2358 
2359 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2360   MachineBasicBlock *BrMBB = FuncInfo.MBB;
2361 
2362   // Update machine-CFG edges.
2363   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2364 
2365   if (I.isUnconditional()) {
2366     // Update machine-CFG edges.
2367     BrMBB->addSuccessor(Succ0MBB);
2368 
2369     // If this is not a fall-through branch or optimizations are switched off,
2370     // emit the branch.
2371     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2372       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2373                               MVT::Other, getControlRoot(),
2374                               DAG.getBasicBlock(Succ0MBB)));
2375 
2376     return;
2377   }
2378 
2379   // If this condition is one of the special cases we handle, do special stuff
2380   // now.
2381   const Value *CondVal = I.getCondition();
2382   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2383 
2384   // If this is a series of conditions that are or'd or and'd together, emit
2385   // this as a sequence of branches instead of setcc's with and/or operations.
2386   // As long as jumps are not expensive (exceptions for multi-use logic ops,
2387   // unpredictable branches, and vector extracts because those jumps are likely
2388   // expensive for any target), this should improve performance.
2389   // For example, instead of something like:
2390   //     cmp A, B
2391   //     C = seteq
2392   //     cmp D, E
2393   //     F = setle
2394   //     or C, F
2395   //     jnz foo
2396   // Emit:
2397   //     cmp A, B
2398   //     je foo
2399   //     cmp D, E
2400   //     jle foo
2401   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2402   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2403       BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2404     Value *Vec;
2405     const Value *BOp0, *BOp1;
2406     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2407     if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2408       Opcode = Instruction::And;
2409     else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2410       Opcode = Instruction::Or;
2411 
2412     if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2413                     match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2414       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2415                            getEdgeProbability(BrMBB, Succ0MBB),
2416                            getEdgeProbability(BrMBB, Succ1MBB),
2417                            /*InvertCond=*/false);
2418       // If the compares in later blocks need to use values not currently
2419       // exported from this block, export them now.  This block should always
2420       // be the first entry.
2421       assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2422 
2423       // Allow some cases to be rejected.
2424       if (ShouldEmitAsBranches(SL->SwitchCases)) {
2425         for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2426           ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2427           ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2428         }
2429 
2430         // Emit the branch for this block.
2431         visitSwitchCase(SL->SwitchCases[0], BrMBB);
2432         SL->SwitchCases.erase(SL->SwitchCases.begin());
2433         return;
2434       }
2435 
2436       // Okay, we decided not to do this, remove any inserted MBB's and clear
2437       // SwitchCases.
2438       for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2439         FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2440 
2441       SL->SwitchCases.clear();
2442     }
2443   }
2444 
2445   // Create a CaseBlock record representing this branch.
2446   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2447                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2448 
2449   // Use visitSwitchCase to actually insert the fast branch sequence for this
2450   // cond branch.
2451   visitSwitchCase(CB, BrMBB);
2452 }
2453 
2454 /// visitSwitchCase - Emits the necessary code to represent a single node in
2455 /// the binary search tree resulting from lowering a switch instruction.
2456 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2457                                           MachineBasicBlock *SwitchBB) {
2458   SDValue Cond;
2459   SDValue CondLHS = getValue(CB.CmpLHS);
2460   SDLoc dl = CB.DL;
2461 
2462   if (CB.CC == ISD::SETTRUE) {
2463     // Branch or fall through to TrueBB.
2464     addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2465     SwitchBB->normalizeSuccProbs();
2466     if (CB.TrueBB != NextBlock(SwitchBB)) {
2467       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2468                               DAG.getBasicBlock(CB.TrueBB)));
2469     }
2470     return;
2471   }
2472 
2473   auto &TLI = DAG.getTargetLoweringInfo();
2474   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2475 
2476   // Build the setcc now.
2477   if (!CB.CmpMHS) {
2478     // Fold "(X == true)" to X and "(X == false)" to !X to
2479     // handle common cases produced by branch lowering.
2480     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2481         CB.CC == ISD::SETEQ)
2482       Cond = CondLHS;
2483     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2484              CB.CC == ISD::SETEQ) {
2485       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2486       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2487     } else {
2488       SDValue CondRHS = getValue(CB.CmpRHS);
2489 
2490       // If a pointer's DAG type is larger than its memory type then the DAG
2491       // values are zero-extended. This breaks signed comparisons so truncate
2492       // back to the underlying type before doing the compare.
2493       if (CondLHS.getValueType() != MemVT) {
2494         CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2495         CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2496       }
2497       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2498     }
2499   } else {
2500     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2501 
2502     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2503     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2504 
2505     SDValue CmpOp = getValue(CB.CmpMHS);
2506     EVT VT = CmpOp.getValueType();
2507 
2508     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2509       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2510                           ISD::SETLE);
2511     } else {
2512       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2513                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2514       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2515                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2516     }
2517   }
2518 
2519   // Update successor info
2520   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2521   // TrueBB and FalseBB are always different unless the incoming IR is
2522   // degenerate. This only happens when running llc on weird IR.
2523   if (CB.TrueBB != CB.FalseBB)
2524     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2525   SwitchBB->normalizeSuccProbs();
2526 
2527   // If the lhs block is the next block, invert the condition so that we can
2528   // fall through to the lhs instead of the rhs block.
2529   if (CB.TrueBB == NextBlock(SwitchBB)) {
2530     std::swap(CB.TrueBB, CB.FalseBB);
2531     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2532     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2533   }
2534 
2535   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2536                                MVT::Other, getControlRoot(), Cond,
2537                                DAG.getBasicBlock(CB.TrueBB));
2538 
2539   // Insert the false branch. Do this even if it's a fall through branch,
2540   // this makes it easier to do DAG optimizations which require inverting
2541   // the branch condition.
2542   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2543                        DAG.getBasicBlock(CB.FalseBB));
2544 
2545   DAG.setRoot(BrCond);
2546 }
2547 
2548 /// visitJumpTable - Emit JumpTable node in the current MBB
2549 void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2550   // Emit the code for the jump table
2551   assert(JT.Reg != -1U && "Should lower JT Header first!");
2552   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2553   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2554                                      JT.Reg, PTy);
2555   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2556   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2557                                     MVT::Other, Index.getValue(1),
2558                                     Table, Index);
2559   DAG.setRoot(BrJumpTable);
2560 }
2561 
2562 /// visitJumpTableHeader - This function emits necessary code to produce index
2563 /// in the JumpTable from switch case.
2564 void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2565                                                JumpTableHeader &JTH,
2566                                                MachineBasicBlock *SwitchBB) {
2567   SDLoc dl = getCurSDLoc();
2568 
2569   // Subtract the lowest switch case value from the value being switched on.
2570   SDValue SwitchOp = getValue(JTH.SValue);
2571   EVT VT = SwitchOp.getValueType();
2572   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2573                             DAG.getConstant(JTH.First, dl, VT));
2574 
2575   // The SDNode we just created, which holds the value being switched on minus
2576   // the smallest case value, needs to be copied to a virtual register so it
2577   // can be used as an index into the jump table in a subsequent basic block.
2578   // This value may be smaller or larger than the target's pointer type, and
2579   // therefore require extension or truncating.
2580   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2581   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2582 
2583   unsigned JumpTableReg =
2584       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2585   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2586                                     JumpTableReg, SwitchOp);
2587   JT.Reg = JumpTableReg;
2588 
2589   if (!JTH.FallthroughUnreachable) {
2590     // Emit the range check for the jump table, and branch to the default block
2591     // for the switch statement if the value being switched on exceeds the
2592     // largest case in the switch.
2593     SDValue CMP = DAG.getSetCC(
2594         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2595                                    Sub.getValueType()),
2596         Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2597 
2598     SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2599                                  MVT::Other, CopyTo, CMP,
2600                                  DAG.getBasicBlock(JT.Default));
2601 
2602     // Avoid emitting unnecessary branches to the next block.
2603     if (JT.MBB != NextBlock(SwitchBB))
2604       BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2605                            DAG.getBasicBlock(JT.MBB));
2606 
2607     DAG.setRoot(BrCond);
2608   } else {
2609     // Avoid emitting unnecessary branches to the next block.
2610     if (JT.MBB != NextBlock(SwitchBB))
2611       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2612                               DAG.getBasicBlock(JT.MBB)));
2613     else
2614       DAG.setRoot(CopyTo);
2615   }
2616 }
2617 
2618 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2619 /// variable if there exists one.
2620 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2621                                  SDValue &Chain) {
2622   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2623   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2624   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2625   MachineFunction &MF = DAG.getMachineFunction();
2626   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2627   MachineSDNode *Node =
2628       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2629   if (Global) {
2630     MachinePointerInfo MPInfo(Global);
2631     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2632                  MachineMemOperand::MODereferenceable;
2633     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2634         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2635     DAG.setNodeMemRefs(Node, {MemRef});
2636   }
2637   if (PtrTy != PtrMemTy)
2638     return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2639   return SDValue(Node, 0);
2640 }
2641 
2642 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2643 /// tail spliced into a stack protector check success bb.
2644 ///
2645 /// For a high level explanation of how this fits into the stack protector
2646 /// generation see the comment on the declaration of class
2647 /// StackProtectorDescriptor.
2648 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2649                                                   MachineBasicBlock *ParentBB) {
2650 
2651   // First create the loads to the guard/stack slot for the comparison.
2652   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2653   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2654   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2655 
2656   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2657   int FI = MFI.getStackProtectorIndex();
2658 
2659   SDValue Guard;
2660   SDLoc dl = getCurSDLoc();
2661   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2662   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2663   Align Align =
2664       DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2665 
2666   // Generate code to load the content of the guard slot.
2667   SDValue GuardVal = DAG.getLoad(
2668       PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2669       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2670       MachineMemOperand::MOVolatile);
2671 
2672   if (TLI.useStackGuardXorFP())
2673     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2674 
2675   // Retrieve guard check function, nullptr if instrumentation is inlined.
2676   if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2677     // The target provides a guard check function to validate the guard value.
2678     // Generate a call to that function with the content of the guard slot as
2679     // argument.
2680     FunctionType *FnTy = GuardCheckFn->getFunctionType();
2681     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2682 
2683     TargetLowering::ArgListTy Args;
2684     TargetLowering::ArgListEntry Entry;
2685     Entry.Node = GuardVal;
2686     Entry.Ty = FnTy->getParamType(0);
2687     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2688       Entry.IsInReg = true;
2689     Args.push_back(Entry);
2690 
2691     TargetLowering::CallLoweringInfo CLI(DAG);
2692     CLI.setDebugLoc(getCurSDLoc())
2693         .setChain(DAG.getEntryNode())
2694         .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2695                    getValue(GuardCheckFn), std::move(Args));
2696 
2697     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2698     DAG.setRoot(Result.second);
2699     return;
2700   }
2701 
2702   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2703   // Otherwise, emit a volatile load to retrieve the stack guard value.
2704   SDValue Chain = DAG.getEntryNode();
2705   if (TLI.useLoadStackGuardNode()) {
2706     Guard = getLoadStackGuard(DAG, dl, Chain);
2707   } else {
2708     const Value *IRGuard = TLI.getSDagStackGuard(M);
2709     SDValue GuardPtr = getValue(IRGuard);
2710 
2711     Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2712                         MachinePointerInfo(IRGuard, 0), Align,
2713                         MachineMemOperand::MOVolatile);
2714   }
2715 
2716   // Perform the comparison via a getsetcc.
2717   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2718                                                         *DAG.getContext(),
2719                                                         Guard.getValueType()),
2720                              Guard, GuardVal, ISD::SETNE);
2721 
2722   // If the guard/stackslot do not equal, branch to failure MBB.
2723   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2724                                MVT::Other, GuardVal.getOperand(0),
2725                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2726   // Otherwise branch to success MBB.
2727   SDValue Br = DAG.getNode(ISD::BR, dl,
2728                            MVT::Other, BrCond,
2729                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2730 
2731   DAG.setRoot(Br);
2732 }
2733 
2734 /// Codegen the failure basic block for a stack protector check.
2735 ///
2736 /// A failure stack protector machine basic block consists simply of a call to
2737 /// __stack_chk_fail().
2738 ///
2739 /// For a high level explanation of how this fits into the stack protector
2740 /// generation see the comment on the declaration of class
2741 /// StackProtectorDescriptor.
2742 void
2743 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2744   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2745   TargetLowering::MakeLibCallOptions CallOptions;
2746   CallOptions.setDiscardResult(true);
2747   SDValue Chain =
2748       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2749                       None, CallOptions, getCurSDLoc()).second;
2750   // On PS4/PS5, the "return address" must still be within the calling
2751   // function, even if it's at the very end, so emit an explicit TRAP here.
2752   // Passing 'true' for doesNotReturn above won't generate the trap for us.
2753   if (TM.getTargetTriple().isPS())
2754     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2755   // WebAssembly needs an unreachable instruction after a non-returning call,
2756   // because the function return type can be different from __stack_chk_fail's
2757   // return type (void).
2758   if (TM.getTargetTriple().isWasm())
2759     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2760 
2761   DAG.setRoot(Chain);
2762 }
2763 
2764 /// visitBitTestHeader - This function emits necessary code to produce value
2765 /// suitable for "bit tests"
2766 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2767                                              MachineBasicBlock *SwitchBB) {
2768   SDLoc dl = getCurSDLoc();
2769 
2770   // Subtract the minimum value.
2771   SDValue SwitchOp = getValue(B.SValue);
2772   EVT VT = SwitchOp.getValueType();
2773   SDValue RangeSub =
2774       DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2775 
2776   // Determine the type of the test operands.
2777   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2778   bool UsePtrType = false;
2779   if (!TLI.isTypeLegal(VT)) {
2780     UsePtrType = true;
2781   } else {
2782     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2783       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2784         // Switch table case range are encoded into series of masks.
2785         // Just use pointer type, it's guaranteed to fit.
2786         UsePtrType = true;
2787         break;
2788       }
2789   }
2790   SDValue Sub = RangeSub;
2791   if (UsePtrType) {
2792     VT = TLI.getPointerTy(DAG.getDataLayout());
2793     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2794   }
2795 
2796   B.RegVT = VT.getSimpleVT();
2797   B.Reg = FuncInfo.CreateReg(B.RegVT);
2798   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2799 
2800   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2801 
2802   if (!B.FallthroughUnreachable)
2803     addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2804   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2805   SwitchBB->normalizeSuccProbs();
2806 
2807   SDValue Root = CopyTo;
2808   if (!B.FallthroughUnreachable) {
2809     // Conditional branch to the default block.
2810     SDValue RangeCmp = DAG.getSetCC(dl,
2811         TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2812                                RangeSub.getValueType()),
2813         RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2814         ISD::SETUGT);
2815 
2816     Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2817                        DAG.getBasicBlock(B.Default));
2818   }
2819 
2820   // Avoid emitting unnecessary branches to the next block.
2821   if (MBB != NextBlock(SwitchBB))
2822     Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2823 
2824   DAG.setRoot(Root);
2825 }
2826 
2827 /// visitBitTestCase - this function produces one "bit test"
2828 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2829                                            MachineBasicBlock* NextMBB,
2830                                            BranchProbability BranchProbToNext,
2831                                            unsigned Reg,
2832                                            BitTestCase &B,
2833                                            MachineBasicBlock *SwitchBB) {
2834   SDLoc dl = getCurSDLoc();
2835   MVT VT = BB.RegVT;
2836   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2837   SDValue Cmp;
2838   unsigned PopCount = countPopulation(B.Mask);
2839   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2840   if (PopCount == 1) {
2841     // Testing for a single bit; just compare the shift count with what it
2842     // would need to be to shift a 1 bit in that position.
2843     Cmp = DAG.getSetCC(
2844         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2845         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2846         ISD::SETEQ);
2847   } else if (PopCount == BB.Range) {
2848     // There is only one zero bit in the range, test for it directly.
2849     Cmp = DAG.getSetCC(
2850         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2851         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2852         ISD::SETNE);
2853   } else {
2854     // Make desired shift
2855     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2856                                     DAG.getConstant(1, dl, VT), ShiftOp);
2857 
2858     // Emit bit tests and jumps
2859     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2860                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2861     Cmp = DAG.getSetCC(
2862         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2863         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2864   }
2865 
2866   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2867   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2868   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2869   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2870   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2871   // one as they are relative probabilities (and thus work more like weights),
2872   // and hence we need to normalize them to let the sum of them become one.
2873   SwitchBB->normalizeSuccProbs();
2874 
2875   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2876                               MVT::Other, getControlRoot(),
2877                               Cmp, DAG.getBasicBlock(B.TargetBB));
2878 
2879   // Avoid emitting unnecessary branches to the next block.
2880   if (NextMBB != NextBlock(SwitchBB))
2881     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2882                         DAG.getBasicBlock(NextMBB));
2883 
2884   DAG.setRoot(BrAnd);
2885 }
2886 
2887 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2888   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2889 
2890   // Retrieve successors. Look through artificial IR level blocks like
2891   // catchswitch for successors.
2892   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2893   const BasicBlock *EHPadBB = I.getSuccessor(1);
2894 
2895   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2896   // have to do anything here to lower funclet bundles.
2897   assert(!I.hasOperandBundlesOtherThan(
2898              {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2899               LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2900               LLVMContext::OB_cfguardtarget,
2901               LLVMContext::OB_clang_arc_attachedcall}) &&
2902          "Cannot lower invokes with arbitrary operand bundles yet!");
2903 
2904   const Value *Callee(I.getCalledOperand());
2905   const Function *Fn = dyn_cast<Function>(Callee);
2906   if (isa<InlineAsm>(Callee))
2907     visitInlineAsm(I, EHPadBB);
2908   else if (Fn && Fn->isIntrinsic()) {
2909     switch (Fn->getIntrinsicID()) {
2910     default:
2911       llvm_unreachable("Cannot invoke this intrinsic");
2912     case Intrinsic::donothing:
2913       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2914     case Intrinsic::seh_try_begin:
2915     case Intrinsic::seh_scope_begin:
2916     case Intrinsic::seh_try_end:
2917     case Intrinsic::seh_scope_end:
2918       break;
2919     case Intrinsic::experimental_patchpoint_void:
2920     case Intrinsic::experimental_patchpoint_i64:
2921       visitPatchpoint(I, EHPadBB);
2922       break;
2923     case Intrinsic::experimental_gc_statepoint:
2924       LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2925       break;
2926     case Intrinsic::wasm_rethrow: {
2927       // This is usually done in visitTargetIntrinsic, but this intrinsic is
2928       // special because it can be invoked, so we manually lower it to a DAG
2929       // node here.
2930       SmallVector<SDValue, 8> Ops;
2931       Ops.push_back(getRoot()); // inchain
2932       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2933       Ops.push_back(
2934           DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2935                                 TLI.getPointerTy(DAG.getDataLayout())));
2936       SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2937       DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2938       break;
2939     }
2940     }
2941   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2942     // Currently we do not lower any intrinsic calls with deopt operand bundles.
2943     // Eventually we will support lowering the @llvm.experimental.deoptimize
2944     // intrinsic, and right now there are no plans to support other intrinsics
2945     // with deopt state.
2946     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2947   } else {
2948     LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2949   }
2950 
2951   // If the value of the invoke is used outside of its defining block, make it
2952   // available as a virtual register.
2953   // We already took care of the exported value for the statepoint instruction
2954   // during call to the LowerStatepoint.
2955   if (!isa<GCStatepointInst>(I)) {
2956     CopyToExportRegsIfNeeded(&I);
2957   }
2958 
2959   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2960   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2961   BranchProbability EHPadBBProb =
2962       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2963           : BranchProbability::getZero();
2964   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2965 
2966   // Update successor info.
2967   addSuccessorWithProb(InvokeMBB, Return);
2968   for (auto &UnwindDest : UnwindDests) {
2969     UnwindDest.first->setIsEHPad();
2970     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2971   }
2972   InvokeMBB->normalizeSuccProbs();
2973 
2974   // Drop into normal successor.
2975   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2976                           DAG.getBasicBlock(Return)));
2977 }
2978 
2979 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2980   MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2981 
2982   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2983   // have to do anything here to lower funclet bundles.
2984   assert(!I.hasOperandBundlesOtherThan(
2985              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2986          "Cannot lower callbrs with arbitrary operand bundles yet!");
2987 
2988   assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
2989   visitInlineAsm(I);
2990   CopyToExportRegsIfNeeded(&I);
2991 
2992   // Retrieve successors.
2993   SmallPtrSet<BasicBlock *, 8> Dests;
2994   Dests.insert(I.getDefaultDest());
2995   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2996 
2997   // Update successor info.
2998   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
2999   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3000     BasicBlock *Dest = I.getIndirectDest(i);
3001     MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];
3002     Target->setIsInlineAsmBrIndirectTarget();
3003     Target->setHasAddressTaken();
3004     // Don't add duplicate machine successors.
3005     if (Dests.insert(Dest).second)
3006       addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3007   }
3008   CallBrMBB->normalizeSuccProbs();
3009 
3010   // Drop into default successor.
3011   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3012                           MVT::Other, getControlRoot(),
3013                           DAG.getBasicBlock(Return)));
3014 }
3015 
3016 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3017   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3018 }
3019 
3020 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3021   assert(FuncInfo.MBB->isEHPad() &&
3022          "Call to landingpad not in landing pad!");
3023 
3024   // If there aren't registers to copy the values into (e.g., during SjLj
3025   // exceptions), then don't bother to create these DAG nodes.
3026   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3027   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3028   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3029       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3030     return;
3031 
3032   // If landingpad's return type is token type, we don't create DAG nodes
3033   // for its exception pointer and selector value. The extraction of exception
3034   // pointer or selector value from token type landingpads is not currently
3035   // supported.
3036   if (LP.getType()->isTokenTy())
3037     return;
3038 
3039   SmallVector<EVT, 2> ValueVTs;
3040   SDLoc dl = getCurSDLoc();
3041   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3042   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3043 
3044   // Get the two live-in registers as SDValues. The physregs have already been
3045   // copied into virtual registers.
3046   SDValue Ops[2];
3047   if (FuncInfo.ExceptionPointerVirtReg) {
3048     Ops[0] = DAG.getZExtOrTrunc(
3049         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3050                            FuncInfo.ExceptionPointerVirtReg,
3051                            TLI.getPointerTy(DAG.getDataLayout())),
3052         dl, ValueVTs[0]);
3053   } else {
3054     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3055   }
3056   Ops[1] = DAG.getZExtOrTrunc(
3057       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3058                          FuncInfo.ExceptionSelectorVirtReg,
3059                          TLI.getPointerTy(DAG.getDataLayout())),
3060       dl, ValueVTs[1]);
3061 
3062   // Merge into one.
3063   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3064                             DAG.getVTList(ValueVTs), Ops);
3065   setValue(&LP, Res);
3066 }
3067 
3068 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3069                                            MachineBasicBlock *Last) {
3070   // Update JTCases.
3071   for (JumpTableBlock &JTB : SL->JTCases)
3072     if (JTB.first.HeaderBB == First)
3073       JTB.first.HeaderBB = Last;
3074 
3075   // Update BitTestCases.
3076   for (BitTestBlock &BTB : SL->BitTestCases)
3077     if (BTB.Parent == First)
3078       BTB.Parent = Last;
3079 }
3080 
3081 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3082   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3083 
3084   // Update machine-CFG edges with unique successors.
3085   SmallSet<BasicBlock*, 32> Done;
3086   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3087     BasicBlock *BB = I.getSuccessor(i);
3088     bool Inserted = Done.insert(BB).second;
3089     if (!Inserted)
3090         continue;
3091 
3092     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3093     addSuccessorWithProb(IndirectBrMBB, Succ);
3094   }
3095   IndirectBrMBB->normalizeSuccProbs();
3096 
3097   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3098                           MVT::Other, getControlRoot(),
3099                           getValue(I.getAddress())));
3100 }
3101 
3102 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3103   if (!DAG.getTarget().Options.TrapUnreachable)
3104     return;
3105 
3106   // We may be able to ignore unreachable behind a noreturn call.
3107   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
3108     const BasicBlock &BB = *I.getParent();
3109     if (&I != &BB.front()) {
3110       BasicBlock::const_iterator PredI =
3111         std::prev(BasicBlock::const_iterator(&I));
3112       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3113         if (Call->doesNotReturn())
3114           return;
3115       }
3116     }
3117   }
3118 
3119   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3120 }
3121 
3122 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3123   SDNodeFlags Flags;
3124   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3125     Flags.copyFMF(*FPOp);
3126 
3127   SDValue Op = getValue(I.getOperand(0));
3128   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3129                                     Op, Flags);
3130   setValue(&I, UnNodeValue);
3131 }
3132 
3133 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3134   SDNodeFlags Flags;
3135   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3136     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3137     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3138   }
3139   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3140     Flags.setExact(ExactOp->isExact());
3141   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3142     Flags.copyFMF(*FPOp);
3143 
3144   SDValue Op1 = getValue(I.getOperand(0));
3145   SDValue Op2 = getValue(I.getOperand(1));
3146   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3147                                      Op1, Op2, Flags);
3148   setValue(&I, BinNodeValue);
3149 }
3150 
3151 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3152   SDValue Op1 = getValue(I.getOperand(0));
3153   SDValue Op2 = getValue(I.getOperand(1));
3154 
3155   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3156       Op1.getValueType(), DAG.getDataLayout());
3157 
3158   // Coerce the shift amount to the right type if we can. This exposes the
3159   // truncate or zext to optimization early.
3160   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3161     assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3162            "Unexpected shift type");
3163     Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3164   }
3165 
3166   bool nuw = false;
3167   bool nsw = false;
3168   bool exact = false;
3169 
3170   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3171 
3172     if (const OverflowingBinaryOperator *OFBinOp =
3173             dyn_cast<const OverflowingBinaryOperator>(&I)) {
3174       nuw = OFBinOp->hasNoUnsignedWrap();
3175       nsw = OFBinOp->hasNoSignedWrap();
3176     }
3177     if (const PossiblyExactOperator *ExactOp =
3178             dyn_cast<const PossiblyExactOperator>(&I))
3179       exact = ExactOp->isExact();
3180   }
3181   SDNodeFlags Flags;
3182   Flags.setExact(exact);
3183   Flags.setNoSignedWrap(nsw);
3184   Flags.setNoUnsignedWrap(nuw);
3185   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3186                             Flags);
3187   setValue(&I, Res);
3188 }
3189 
3190 void SelectionDAGBuilder::visitSDiv(const User &I) {
3191   SDValue Op1 = getValue(I.getOperand(0));
3192   SDValue Op2 = getValue(I.getOperand(1));
3193 
3194   SDNodeFlags Flags;
3195   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3196                  cast<PossiblyExactOperator>(&I)->isExact());
3197   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3198                            Op2, Flags));
3199 }
3200 
3201 void SelectionDAGBuilder::visitICmp(const User &I) {
3202   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3203   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3204     predicate = IC->getPredicate();
3205   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3206     predicate = ICmpInst::Predicate(IC->getPredicate());
3207   SDValue Op1 = getValue(I.getOperand(0));
3208   SDValue Op2 = getValue(I.getOperand(1));
3209   ISD::CondCode Opcode = getICmpCondCode(predicate);
3210 
3211   auto &TLI = DAG.getTargetLoweringInfo();
3212   EVT MemVT =
3213       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3214 
3215   // If a pointer's DAG type is larger than its memory type then the DAG values
3216   // are zero-extended. This breaks signed comparisons so truncate back to the
3217   // underlying type before doing the compare.
3218   if (Op1.getValueType() != MemVT) {
3219     Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3220     Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3221   }
3222 
3223   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3224                                                         I.getType());
3225   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3226 }
3227 
3228 void SelectionDAGBuilder::visitFCmp(const User &I) {
3229   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3230   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3231     predicate = FC->getPredicate();
3232   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3233     predicate = FCmpInst::Predicate(FC->getPredicate());
3234   SDValue Op1 = getValue(I.getOperand(0));
3235   SDValue Op2 = getValue(I.getOperand(1));
3236 
3237   ISD::CondCode Condition = getFCmpCondCode(predicate);
3238   auto *FPMO = cast<FPMathOperator>(&I);
3239   if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3240     Condition = getFCmpCodeWithoutNaN(Condition);
3241 
3242   SDNodeFlags Flags;
3243   Flags.copyFMF(*FPMO);
3244   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3245 
3246   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3247                                                         I.getType());
3248   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3249 }
3250 
3251 // Check if the condition of the select has one use or two users that are both
3252 // selects with the same condition.
3253 static bool hasOnlySelectUsers(const Value *Cond) {
3254   return llvm::all_of(Cond->users(), [](const Value *V) {
3255     return isa<SelectInst>(V);
3256   });
3257 }
3258 
3259 void SelectionDAGBuilder::visitSelect(const User &I) {
3260   SmallVector<EVT, 4> ValueVTs;
3261   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3262                   ValueVTs);
3263   unsigned NumValues = ValueVTs.size();
3264   if (NumValues == 0) return;
3265 
3266   SmallVector<SDValue, 4> Values(NumValues);
3267   SDValue Cond     = getValue(I.getOperand(0));
3268   SDValue LHSVal   = getValue(I.getOperand(1));
3269   SDValue RHSVal   = getValue(I.getOperand(2));
3270   SmallVector<SDValue, 1> BaseOps(1, Cond);
3271   ISD::NodeType OpCode =
3272       Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3273 
3274   bool IsUnaryAbs = false;
3275   bool Negate = false;
3276 
3277   SDNodeFlags Flags;
3278   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3279     Flags.copyFMF(*FPOp);
3280 
3281   // Min/max matching is only viable if all output VTs are the same.
3282   if (is_splat(ValueVTs)) {
3283     EVT VT = ValueVTs[0];
3284     LLVMContext &Ctx = *DAG.getContext();
3285     auto &TLI = DAG.getTargetLoweringInfo();
3286 
3287     // We care about the legality of the operation after it has been type
3288     // legalized.
3289     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3290       VT = TLI.getTypeToTransformTo(Ctx, VT);
3291 
3292     // If the vselect is legal, assume we want to leave this as a vector setcc +
3293     // vselect. Otherwise, if this is going to be scalarized, we want to see if
3294     // min/max is legal on the scalar type.
3295     bool UseScalarMinMax = VT.isVector() &&
3296       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3297 
3298     Value *LHS, *RHS;
3299     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3300     ISD::NodeType Opc = ISD::DELETED_NODE;
3301     switch (SPR.Flavor) {
3302     case SPF_UMAX:    Opc = ISD::UMAX; break;
3303     case SPF_UMIN:    Opc = ISD::UMIN; break;
3304     case SPF_SMAX:    Opc = ISD::SMAX; break;
3305     case SPF_SMIN:    Opc = ISD::SMIN; break;
3306     case SPF_FMINNUM:
3307       switch (SPR.NaNBehavior) {
3308       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3309       case SPNB_RETURNS_NAN:   Opc = ISD::FMINIMUM; break;
3310       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3311       case SPNB_RETURNS_ANY: {
3312         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3313           Opc = ISD::FMINNUM;
3314         else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3315           Opc = ISD::FMINIMUM;
3316         else if (UseScalarMinMax)
3317           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3318             ISD::FMINNUM : ISD::FMINIMUM;
3319         break;
3320       }
3321       }
3322       break;
3323     case SPF_FMAXNUM:
3324       switch (SPR.NaNBehavior) {
3325       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3326       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXIMUM; break;
3327       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3328       case SPNB_RETURNS_ANY:
3329 
3330         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3331           Opc = ISD::FMAXNUM;
3332         else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3333           Opc = ISD::FMAXIMUM;
3334         else if (UseScalarMinMax)
3335           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3336             ISD::FMAXNUM : ISD::FMAXIMUM;
3337         break;
3338       }
3339       break;
3340     case SPF_NABS:
3341       Negate = true;
3342       LLVM_FALLTHROUGH;
3343     case SPF_ABS:
3344       IsUnaryAbs = true;
3345       Opc = ISD::ABS;
3346       break;
3347     default: break;
3348     }
3349 
3350     if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3351         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3352          (UseScalarMinMax &&
3353           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3354         // If the underlying comparison instruction is used by any other
3355         // instruction, the consumed instructions won't be destroyed, so it is
3356         // not profitable to convert to a min/max.
3357         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3358       OpCode = Opc;
3359       LHSVal = getValue(LHS);
3360       RHSVal = getValue(RHS);
3361       BaseOps.clear();
3362     }
3363 
3364     if (IsUnaryAbs) {
3365       OpCode = Opc;
3366       LHSVal = getValue(LHS);
3367       BaseOps.clear();
3368     }
3369   }
3370 
3371   if (IsUnaryAbs) {
3372     for (unsigned i = 0; i != NumValues; ++i) {
3373       SDLoc dl = getCurSDLoc();
3374       EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3375       Values[i] =
3376           DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3377       if (Negate)
3378         Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
3379                                 Values[i]);
3380     }
3381   } else {
3382     for (unsigned i = 0; i != NumValues; ++i) {
3383       SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3384       Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3385       Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3386       Values[i] = DAG.getNode(
3387           OpCode, getCurSDLoc(),
3388           LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3389     }
3390   }
3391 
3392   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3393                            DAG.getVTList(ValueVTs), Values));
3394 }
3395 
3396 void SelectionDAGBuilder::visitTrunc(const User &I) {
3397   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3398   SDValue N = getValue(I.getOperand(0));
3399   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3400                                                         I.getType());
3401   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3402 }
3403 
3404 void SelectionDAGBuilder::visitZExt(const User &I) {
3405   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3406   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3407   SDValue N = getValue(I.getOperand(0));
3408   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3409                                                         I.getType());
3410   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3411 }
3412 
3413 void SelectionDAGBuilder::visitSExt(const User &I) {
3414   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3415   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3416   SDValue N = getValue(I.getOperand(0));
3417   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3418                                                         I.getType());
3419   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3420 }
3421 
3422 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3423   // FPTrunc is never a no-op cast, no need to check
3424   SDValue N = getValue(I.getOperand(0));
3425   SDLoc dl = getCurSDLoc();
3426   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3427   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3428   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3429                            DAG.getTargetConstant(
3430                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3431 }
3432 
3433 void SelectionDAGBuilder::visitFPExt(const User &I) {
3434   // FPExt is never a no-op cast, no need to check
3435   SDValue N = getValue(I.getOperand(0));
3436   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3437                                                         I.getType());
3438   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3439 }
3440 
3441 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3442   // FPToUI is never a no-op cast, no need to check
3443   SDValue N = getValue(I.getOperand(0));
3444   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3445                                                         I.getType());
3446   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3447 }
3448 
3449 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3450   // FPToSI is never a no-op cast, no need to check
3451   SDValue N = getValue(I.getOperand(0));
3452   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3453                                                         I.getType());
3454   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3455 }
3456 
3457 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3458   // UIToFP is never a no-op cast, no need to check
3459   SDValue N = getValue(I.getOperand(0));
3460   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3461                                                         I.getType());
3462   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3463 }
3464 
3465 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3466   // SIToFP is never a no-op cast, no need to check
3467   SDValue N = getValue(I.getOperand(0));
3468   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3469                                                         I.getType());
3470   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3471 }
3472 
3473 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3474   // What to do depends on the size of the integer and the size of the pointer.
3475   // We can either truncate, zero extend, or no-op, accordingly.
3476   SDValue N = getValue(I.getOperand(0));
3477   auto &TLI = DAG.getTargetLoweringInfo();
3478   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3479                                                         I.getType());
3480   EVT PtrMemVT =
3481       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3482   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3483   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3484   setValue(&I, N);
3485 }
3486 
3487 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3488   // What to do depends on the size of the integer and the size of the pointer.
3489   // We can either truncate, zero extend, or no-op, accordingly.
3490   SDValue N = getValue(I.getOperand(0));
3491   auto &TLI = DAG.getTargetLoweringInfo();
3492   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3493   EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3494   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3495   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3496   setValue(&I, N);
3497 }
3498 
3499 void SelectionDAGBuilder::visitBitCast(const User &I) {
3500   SDValue N = getValue(I.getOperand(0));
3501   SDLoc dl = getCurSDLoc();
3502   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3503                                                         I.getType());
3504 
3505   // BitCast assures us that source and destination are the same size so this is
3506   // either a BITCAST or a no-op.
3507   if (DestVT != N.getValueType())
3508     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3509                              DestVT, N)); // convert types.
3510   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3511   // might fold any kind of constant expression to an integer constant and that
3512   // is not what we are looking for. Only recognize a bitcast of a genuine
3513   // constant integer as an opaque constant.
3514   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3515     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3516                                  /*isOpaque*/true));
3517   else
3518     setValue(&I, N);            // noop cast.
3519 }
3520 
3521 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3522   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3523   const Value *SV = I.getOperand(0);
3524   SDValue N = getValue(SV);
3525   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3526 
3527   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3528   unsigned DestAS = I.getType()->getPointerAddressSpace();
3529 
3530   if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3531     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3532 
3533   setValue(&I, N);
3534 }
3535 
3536 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3537   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3538   SDValue InVec = getValue(I.getOperand(0));
3539   SDValue InVal = getValue(I.getOperand(1));
3540   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3541                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3542   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3543                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3544                            InVec, InVal, InIdx));
3545 }
3546 
3547 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3548   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3549   SDValue InVec = getValue(I.getOperand(0));
3550   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3551                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3552   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3553                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3554                            InVec, InIdx));
3555 }
3556 
3557 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3558   SDValue Src1 = getValue(I.getOperand(0));
3559   SDValue Src2 = getValue(I.getOperand(1));
3560   ArrayRef<int> Mask;
3561   if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3562     Mask = SVI->getShuffleMask();
3563   else
3564     Mask = cast<ConstantExpr>(I).getShuffleMask();
3565   SDLoc DL = getCurSDLoc();
3566   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3567   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3568   EVT SrcVT = Src1.getValueType();
3569 
3570   if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3571       VT.isScalableVector()) {
3572     // Canonical splat form of first element of first input vector.
3573     SDValue FirstElt =
3574         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
3575                     DAG.getVectorIdxConstant(0, DL));
3576     setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3577     return;
3578   }
3579 
3580   // For now, we only handle splats for scalable vectors.
3581   // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3582   // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3583   assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3584 
3585   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3586   unsigned MaskNumElts = Mask.size();
3587 
3588   if (SrcNumElts == MaskNumElts) {
3589     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3590     return;
3591   }
3592 
3593   // Normalize the shuffle vector since mask and vector length don't match.
3594   if (SrcNumElts < MaskNumElts) {
3595     // Mask is longer than the source vectors. We can use concatenate vector to
3596     // make the mask and vectors lengths match.
3597 
3598     if (MaskNumElts % SrcNumElts == 0) {
3599       // Mask length is a multiple of the source vector length.
3600       // Check if the shuffle is some kind of concatenation of the input
3601       // vectors.
3602       unsigned NumConcat = MaskNumElts / SrcNumElts;
3603       bool IsConcat = true;
3604       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3605       for (unsigned i = 0; i != MaskNumElts; ++i) {
3606         int Idx = Mask[i];
3607         if (Idx < 0)
3608           continue;
3609         // Ensure the indices in each SrcVT sized piece are sequential and that
3610         // the same source is used for the whole piece.
3611         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3612             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3613              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3614           IsConcat = false;
3615           break;
3616         }
3617         // Remember which source this index came from.
3618         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3619       }
3620 
3621       // The shuffle is concatenating multiple vectors together. Just emit
3622       // a CONCAT_VECTORS operation.
3623       if (IsConcat) {
3624         SmallVector<SDValue, 8> ConcatOps;
3625         for (auto Src : ConcatSrcs) {
3626           if (Src < 0)
3627             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3628           else if (Src == 0)
3629             ConcatOps.push_back(Src1);
3630           else
3631             ConcatOps.push_back(Src2);
3632         }
3633         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3634         return;
3635       }
3636     }
3637 
3638     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3639     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3640     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3641                                     PaddedMaskNumElts);
3642 
3643     // Pad both vectors with undefs to make them the same length as the mask.
3644     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3645 
3646     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3647     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3648     MOps1[0] = Src1;
3649     MOps2[0] = Src2;
3650 
3651     Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3652     Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3653 
3654     // Readjust mask for new input vector length.
3655     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3656     for (unsigned i = 0; i != MaskNumElts; ++i) {
3657       int Idx = Mask[i];
3658       if (Idx >= (int)SrcNumElts)
3659         Idx -= SrcNumElts - PaddedMaskNumElts;
3660       MappedOps[i] = Idx;
3661     }
3662 
3663     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3664 
3665     // If the concatenated vector was padded, extract a subvector with the
3666     // correct number of elements.
3667     if (MaskNumElts != PaddedMaskNumElts)
3668       Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3669                            DAG.getVectorIdxConstant(0, DL));
3670 
3671     setValue(&I, Result);
3672     return;
3673   }
3674 
3675   if (SrcNumElts > MaskNumElts) {
3676     // Analyze the access pattern of the vector to see if we can extract
3677     // two subvectors and do the shuffle.
3678     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3679     bool CanExtract = true;
3680     for (int Idx : Mask) {
3681       unsigned Input = 0;
3682       if (Idx < 0)
3683         continue;
3684 
3685       if (Idx >= (int)SrcNumElts) {
3686         Input = 1;
3687         Idx -= SrcNumElts;
3688       }
3689 
3690       // If all the indices come from the same MaskNumElts sized portion of
3691       // the sources we can use extract. Also make sure the extract wouldn't
3692       // extract past the end of the source.
3693       int NewStartIdx = alignDown(Idx, MaskNumElts);
3694       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3695           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3696         CanExtract = false;
3697       // Make sure we always update StartIdx as we use it to track if all
3698       // elements are undef.
3699       StartIdx[Input] = NewStartIdx;
3700     }
3701 
3702     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3703       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3704       return;
3705     }
3706     if (CanExtract) {
3707       // Extract appropriate subvector and generate a vector shuffle
3708       for (unsigned Input = 0; Input < 2; ++Input) {
3709         SDValue &Src = Input == 0 ? Src1 : Src2;
3710         if (StartIdx[Input] < 0)
3711           Src = DAG.getUNDEF(VT);
3712         else {
3713           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3714                             DAG.getVectorIdxConstant(StartIdx[Input], DL));
3715         }
3716       }
3717 
3718       // Calculate new mask.
3719       SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3720       for (int &Idx : MappedOps) {
3721         if (Idx >= (int)SrcNumElts)
3722           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3723         else if (Idx >= 0)
3724           Idx -= StartIdx[0];
3725       }
3726 
3727       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3728       return;
3729     }
3730   }
3731 
3732   // We can't use either concat vectors or extract subvectors so fall back to
3733   // replacing the shuffle with extract and build vector.
3734   // to insert and build vector.
3735   EVT EltVT = VT.getVectorElementType();
3736   SmallVector<SDValue,8> Ops;
3737   for (int Idx : Mask) {
3738     SDValue Res;
3739 
3740     if (Idx < 0) {
3741       Res = DAG.getUNDEF(EltVT);
3742     } else {
3743       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3744       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3745 
3746       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3747                         DAG.getVectorIdxConstant(Idx, DL));
3748     }
3749 
3750     Ops.push_back(Res);
3751   }
3752 
3753   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3754 }
3755 
3756 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3757   ArrayRef<unsigned> Indices = I.getIndices();
3758   const Value *Op0 = I.getOperand(0);
3759   const Value *Op1 = I.getOperand(1);
3760   Type *AggTy = I.getType();
3761   Type *ValTy = Op1->getType();
3762   bool IntoUndef = isa<UndefValue>(Op0);
3763   bool FromUndef = isa<UndefValue>(Op1);
3764 
3765   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3766 
3767   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3768   SmallVector<EVT, 4> AggValueVTs;
3769   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3770   SmallVector<EVT, 4> ValValueVTs;
3771   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3772 
3773   unsigned NumAggValues = AggValueVTs.size();
3774   unsigned NumValValues = ValValueVTs.size();
3775   SmallVector<SDValue, 4> Values(NumAggValues);
3776 
3777   // Ignore an insertvalue that produces an empty object
3778   if (!NumAggValues) {
3779     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3780     return;
3781   }
3782 
3783   SDValue Agg = getValue(Op0);
3784   unsigned i = 0;
3785   // Copy the beginning value(s) from the original aggregate.
3786   for (; i != LinearIndex; ++i)
3787     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3788                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3789   // Copy values from the inserted value(s).
3790   if (NumValValues) {
3791     SDValue Val = getValue(Op1);
3792     for (; i != LinearIndex + NumValValues; ++i)
3793       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3794                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3795   }
3796   // Copy remaining value(s) from the original aggregate.
3797   for (; i != NumAggValues; ++i)
3798     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3799                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3800 
3801   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3802                            DAG.getVTList(AggValueVTs), Values));
3803 }
3804 
3805 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3806   ArrayRef<unsigned> Indices = I.getIndices();
3807   const Value *Op0 = I.getOperand(0);
3808   Type *AggTy = Op0->getType();
3809   Type *ValTy = I.getType();
3810   bool OutOfUndef = isa<UndefValue>(Op0);
3811 
3812   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3813 
3814   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3815   SmallVector<EVT, 4> ValValueVTs;
3816   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3817 
3818   unsigned NumValValues = ValValueVTs.size();
3819 
3820   // Ignore a extractvalue that produces an empty object
3821   if (!NumValValues) {
3822     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3823     return;
3824   }
3825 
3826   SmallVector<SDValue, 4> Values(NumValValues);
3827 
3828   SDValue Agg = getValue(Op0);
3829   // Copy out the selected value(s).
3830   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3831     Values[i - LinearIndex] =
3832       OutOfUndef ?
3833         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3834         SDValue(Agg.getNode(), Agg.getResNo() + i);
3835 
3836   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3837                            DAG.getVTList(ValValueVTs), Values));
3838 }
3839 
3840 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3841   Value *Op0 = I.getOperand(0);
3842   // Note that the pointer operand may be a vector of pointers. Take the scalar
3843   // element which holds a pointer.
3844   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3845   SDValue N = getValue(Op0);
3846   SDLoc dl = getCurSDLoc();
3847   auto &TLI = DAG.getTargetLoweringInfo();
3848 
3849   // Normalize Vector GEP - all scalar operands should be converted to the
3850   // splat vector.
3851   bool IsVectorGEP = I.getType()->isVectorTy();
3852   ElementCount VectorElementCount =
3853       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3854                   : ElementCount::getFixed(0);
3855 
3856   if (IsVectorGEP && !N.getValueType().isVector()) {
3857     LLVMContext &Context = *DAG.getContext();
3858     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3859     if (VectorElementCount.isScalable())
3860       N = DAG.getSplatVector(VT, dl, N);
3861     else
3862       N = DAG.getSplatBuildVector(VT, dl, N);
3863   }
3864 
3865   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3866        GTI != E; ++GTI) {
3867     const Value *Idx = GTI.getOperand();
3868     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3869       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3870       if (Field) {
3871         // N = N + Offset
3872         uint64_t Offset =
3873             DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
3874 
3875         // In an inbounds GEP with an offset that is nonnegative even when
3876         // interpreted as signed, assume there is no unsigned overflow.
3877         SDNodeFlags Flags;
3878         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3879           Flags.setNoUnsignedWrap(true);
3880 
3881         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3882                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3883       }
3884     } else {
3885       // IdxSize is the width of the arithmetic according to IR semantics.
3886       // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3887       // (and fix up the result later).
3888       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3889       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3890       TypeSize ElementSize =
3891           DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3892       // We intentionally mask away the high bits here; ElementSize may not
3893       // fit in IdxTy.
3894       APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3895       bool ElementScalable = ElementSize.isScalable();
3896 
3897       // If this is a scalar constant or a splat vector of constants,
3898       // handle it quickly.
3899       const auto *C = dyn_cast<Constant>(Idx);
3900       if (C && isa<VectorType>(C->getType()))
3901         C = C->getSplatValue();
3902 
3903       const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3904       if (CI && CI->isZero())
3905         continue;
3906       if (CI && !ElementScalable) {
3907         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3908         LLVMContext &Context = *DAG.getContext();
3909         SDValue OffsVal;
3910         if (IsVectorGEP)
3911           OffsVal = DAG.getConstant(
3912               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3913         else
3914           OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3915 
3916         // In an inbounds GEP with an offset that is nonnegative even when
3917         // interpreted as signed, assume there is no unsigned overflow.
3918         SDNodeFlags Flags;
3919         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3920           Flags.setNoUnsignedWrap(true);
3921 
3922         OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3923 
3924         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3925         continue;
3926       }
3927 
3928       // N = N + Idx * ElementMul;
3929       SDValue IdxN = getValue(Idx);
3930 
3931       if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3932         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3933                                   VectorElementCount);
3934         if (VectorElementCount.isScalable())
3935           IdxN = DAG.getSplatVector(VT, dl, IdxN);
3936         else
3937           IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3938       }
3939 
3940       // If the index is smaller or larger than intptr_t, truncate or extend
3941       // it.
3942       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3943 
3944       if (ElementScalable) {
3945         EVT VScaleTy = N.getValueType().getScalarType();
3946         SDValue VScale = DAG.getNode(
3947             ISD::VSCALE, dl, VScaleTy,
3948             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3949         if (IsVectorGEP)
3950           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3951         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3952       } else {
3953         // If this is a multiply by a power of two, turn it into a shl
3954         // immediately.  This is a very common case.
3955         if (ElementMul != 1) {
3956           if (ElementMul.isPowerOf2()) {
3957             unsigned Amt = ElementMul.logBase2();
3958             IdxN = DAG.getNode(ISD::SHL, dl,
3959                                N.getValueType(), IdxN,
3960                                DAG.getConstant(Amt, dl, IdxN.getValueType()));
3961           } else {
3962             SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3963                                             IdxN.getValueType());
3964             IdxN = DAG.getNode(ISD::MUL, dl,
3965                                N.getValueType(), IdxN, Scale);
3966           }
3967         }
3968       }
3969 
3970       N = DAG.getNode(ISD::ADD, dl,
3971                       N.getValueType(), N, IdxN);
3972     }
3973   }
3974 
3975   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3976   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3977   if (IsVectorGEP) {
3978     PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
3979     PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
3980   }
3981 
3982   if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3983     N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3984 
3985   setValue(&I, N);
3986 }
3987 
3988 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3989   // If this is a fixed sized alloca in the entry block of the function,
3990   // allocate it statically on the stack.
3991   if (FuncInfo.StaticAllocaMap.count(&I))
3992     return;   // getValue will auto-populate this.
3993 
3994   SDLoc dl = getCurSDLoc();
3995   Type *Ty = I.getAllocatedType();
3996   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3997   auto &DL = DAG.getDataLayout();
3998   TypeSize TySize = DL.getTypeAllocSize(Ty);
3999   MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4000 
4001   SDValue AllocSize = getValue(I.getArraySize());
4002 
4003   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
4004   if (AllocSize.getValueType() != IntPtr)
4005     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4006 
4007   if (TySize.isScalable())
4008     AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4009                             DAG.getVScale(dl, IntPtr,
4010                                           APInt(IntPtr.getScalarSizeInBits(),
4011                                                 TySize.getKnownMinValue())));
4012   else
4013     AllocSize =
4014         DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4015                     DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4016 
4017   // Handle alignment.  If the requested alignment is less than or equal to
4018   // the stack alignment, ignore it.  If the size is greater than or equal to
4019   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4020   Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4021   if (*Alignment <= StackAlign)
4022     Alignment = None;
4023 
4024   const uint64_t StackAlignMask = StackAlign.value() - 1U;
4025   // Round the size of the allocation up to the stack alignment size
4026   // by add SA-1 to the size. This doesn't overflow because we're computing
4027   // an address inside an alloca.
4028   SDNodeFlags Flags;
4029   Flags.setNoUnsignedWrap(true);
4030   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4031                           DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4032 
4033   // Mask out the low bits for alignment purposes.
4034   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4035                           DAG.getConstant(~StackAlignMask, dl, IntPtr));
4036 
4037   SDValue Ops[] = {
4038       getRoot(), AllocSize,
4039       DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4040   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4041   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4042   setValue(&I, DSA);
4043   DAG.setRoot(DSA.getValue(1));
4044 
4045   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4046 }
4047 
4048 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4049   if (I.isAtomic())
4050     return visitAtomicLoad(I);
4051 
4052   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4053   const Value *SV = I.getOperand(0);
4054   if (TLI.supportSwiftError()) {
4055     // Swifterror values can come from either a function parameter with
4056     // swifterror attribute or an alloca with swifterror attribute.
4057     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4058       if (Arg->hasSwiftErrorAttr())
4059         return visitLoadFromSwiftError(I);
4060     }
4061 
4062     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4063       if (Alloca->isSwiftError())
4064         return visitLoadFromSwiftError(I);
4065     }
4066   }
4067 
4068   SDValue Ptr = getValue(SV);
4069 
4070   Type *Ty = I.getType();
4071   Align Alignment = I.getAlign();
4072 
4073   AAMDNodes AAInfo = I.getAAMetadata();
4074   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4075 
4076   SmallVector<EVT, 4> ValueVTs, MemVTs;
4077   SmallVector<uint64_t, 4> Offsets;
4078   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4079   unsigned NumValues = ValueVTs.size();
4080   if (NumValues == 0)
4081     return;
4082 
4083   bool isVolatile = I.isVolatile();
4084 
4085   SDValue Root;
4086   bool ConstantMemory = false;
4087   if (isVolatile)
4088     // Serialize volatile loads with other side effects.
4089     Root = getRoot();
4090   else if (NumValues > MaxParallelChains)
4091     Root = getMemoryRoot();
4092   else if (AA &&
4093            AA->pointsToConstantMemory(MemoryLocation(
4094                SV,
4095                LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4096                AAInfo))) {
4097     // Do not serialize (non-volatile) loads of constant memory with anything.
4098     Root = DAG.getEntryNode();
4099     ConstantMemory = true;
4100   } else {
4101     // Do not serialize non-volatile loads against each other.
4102     Root = DAG.getRoot();
4103   }
4104 
4105   SDLoc dl = getCurSDLoc();
4106 
4107   if (isVolatile)
4108     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4109 
4110   // An aggregate load cannot wrap around the address space, so offsets to its
4111   // parts don't wrap either.
4112   SDNodeFlags Flags;
4113   Flags.setNoUnsignedWrap(true);
4114 
4115   SmallVector<SDValue, 4> Values(NumValues);
4116   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4117   EVT PtrVT = Ptr.getValueType();
4118 
4119   MachineMemOperand::Flags MMOFlags
4120     = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4121 
4122   unsigned ChainI = 0;
4123   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4124     // Serializing loads here may result in excessive register pressure, and
4125     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4126     // could recover a bit by hoisting nodes upward in the chain by recognizing
4127     // they are side-effect free or do not alias. The optimizer should really
4128     // avoid this case by converting large object/array copies to llvm.memcpy
4129     // (MaxParallelChains should always remain as failsafe).
4130     if (ChainI == MaxParallelChains) {
4131       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4132       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4133                                   makeArrayRef(Chains.data(), ChainI));
4134       Root = Chain;
4135       ChainI = 0;
4136     }
4137     SDValue A = DAG.getNode(ISD::ADD, dl,
4138                             PtrVT, Ptr,
4139                             DAG.getConstant(Offsets[i], dl, PtrVT),
4140                             Flags);
4141 
4142     SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4143                             MachinePointerInfo(SV, Offsets[i]), Alignment,
4144                             MMOFlags, AAInfo, Ranges);
4145     Chains[ChainI] = L.getValue(1);
4146 
4147     if (MemVTs[i] != ValueVTs[i])
4148       L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4149 
4150     Values[i] = L;
4151   }
4152 
4153   if (!ConstantMemory) {
4154     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4155                                 makeArrayRef(Chains.data(), ChainI));
4156     if (isVolatile)
4157       DAG.setRoot(Chain);
4158     else
4159       PendingLoads.push_back(Chain);
4160   }
4161 
4162   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4163                            DAG.getVTList(ValueVTs), Values));
4164 }
4165 
4166 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4167   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4168          "call visitStoreToSwiftError when backend supports swifterror");
4169 
4170   SmallVector<EVT, 4> ValueVTs;
4171   SmallVector<uint64_t, 4> Offsets;
4172   const Value *SrcV = I.getOperand(0);
4173   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4174                   SrcV->getType(), ValueVTs, &Offsets);
4175   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4176          "expect a single EVT for swifterror");
4177 
4178   SDValue Src = getValue(SrcV);
4179   // Create a virtual register, then update the virtual register.
4180   Register VReg =
4181       SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4182   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4183   // Chain can be getRoot or getControlRoot.
4184   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4185                                       SDValue(Src.getNode(), Src.getResNo()));
4186   DAG.setRoot(CopyNode);
4187 }
4188 
4189 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4190   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4191          "call visitLoadFromSwiftError when backend supports swifterror");
4192 
4193   assert(!I.isVolatile() &&
4194          !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4195          !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4196          "Support volatile, non temporal, invariant for load_from_swift_error");
4197 
4198   const Value *SV = I.getOperand(0);
4199   Type *Ty = I.getType();
4200   assert(
4201       (!AA ||
4202        !AA->pointsToConstantMemory(MemoryLocation(
4203            SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4204            I.getAAMetadata()))) &&
4205       "load_from_swift_error should not be constant memory");
4206 
4207   SmallVector<EVT, 4> ValueVTs;
4208   SmallVector<uint64_t, 4> Offsets;
4209   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4210                   ValueVTs, &Offsets);
4211   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4212          "expect a single EVT for swifterror");
4213 
4214   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4215   SDValue L = DAG.getCopyFromReg(
4216       getRoot(), getCurSDLoc(),
4217       SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4218 
4219   setValue(&I, L);
4220 }
4221 
4222 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4223   if (I.isAtomic())
4224     return visitAtomicStore(I);
4225 
4226   const Value *SrcV = I.getOperand(0);
4227   const Value *PtrV = I.getOperand(1);
4228 
4229   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4230   if (TLI.supportSwiftError()) {
4231     // Swifterror values can come from either a function parameter with
4232     // swifterror attribute or an alloca with swifterror attribute.
4233     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4234       if (Arg->hasSwiftErrorAttr())
4235         return visitStoreToSwiftError(I);
4236     }
4237 
4238     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4239       if (Alloca->isSwiftError())
4240         return visitStoreToSwiftError(I);
4241     }
4242   }
4243 
4244   SmallVector<EVT, 4> ValueVTs, MemVTs;
4245   SmallVector<uint64_t, 4> Offsets;
4246   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4247                   SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4248   unsigned NumValues = ValueVTs.size();
4249   if (NumValues == 0)
4250     return;
4251 
4252   // Get the lowered operands. Note that we do this after
4253   // checking if NumResults is zero, because with zero results
4254   // the operands won't have values in the map.
4255   SDValue Src = getValue(SrcV);
4256   SDValue Ptr = getValue(PtrV);
4257 
4258   SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4259   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4260   SDLoc dl = getCurSDLoc();
4261   Align Alignment = I.getAlign();
4262   AAMDNodes AAInfo = I.getAAMetadata();
4263 
4264   auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4265 
4266   // An aggregate load cannot wrap around the address space, so offsets to its
4267   // parts don't wrap either.
4268   SDNodeFlags Flags;
4269   Flags.setNoUnsignedWrap(true);
4270 
4271   unsigned ChainI = 0;
4272   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4273     // See visitLoad comments.
4274     if (ChainI == MaxParallelChains) {
4275       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4276                                   makeArrayRef(Chains.data(), ChainI));
4277       Root = Chain;
4278       ChainI = 0;
4279     }
4280     SDValue Add =
4281         DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4282     SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4283     if (MemVTs[i] != ValueVTs[i])
4284       Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4285     SDValue St =
4286         DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4287                      Alignment, MMOFlags, AAInfo);
4288     Chains[ChainI] = St;
4289   }
4290 
4291   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4292                                   makeArrayRef(Chains.data(), ChainI));
4293   DAG.setRoot(StoreNode);
4294 }
4295 
4296 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4297                                            bool IsCompressing) {
4298   SDLoc sdl = getCurSDLoc();
4299 
4300   auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4301                                MaybeAlign &Alignment) {
4302     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4303     Src0 = I.getArgOperand(0);
4304     Ptr = I.getArgOperand(1);
4305     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4306     Mask = I.getArgOperand(3);
4307   };
4308   auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4309                                     MaybeAlign &Alignment) {
4310     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4311     Src0 = I.getArgOperand(0);
4312     Ptr = I.getArgOperand(1);
4313     Mask = I.getArgOperand(2);
4314     Alignment = None;
4315   };
4316 
4317   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4318   MaybeAlign Alignment;
4319   if (IsCompressing)
4320     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4321   else
4322     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4323 
4324   SDValue Ptr = getValue(PtrOperand);
4325   SDValue Src0 = getValue(Src0Operand);
4326   SDValue Mask = getValue(MaskOperand);
4327   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4328 
4329   EVT VT = Src0.getValueType();
4330   if (!Alignment)
4331     Alignment = DAG.getEVTAlign(VT);
4332 
4333   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4334       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
4335       MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
4336   SDValue StoreNode =
4337       DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4338                          ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4339   DAG.setRoot(StoreNode);
4340   setValue(&I, StoreNode);
4341 }
4342 
4343 // Get a uniform base for the Gather/Scatter intrinsic.
4344 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4345 // We try to represent it as a base pointer + vector of indices.
4346 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4347 // The first operand of the GEP may be a single pointer or a vector of pointers
4348 // Example:
4349 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4350 //  or
4351 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4352 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4353 //
4354 // When the first GEP operand is a single pointer - it is the uniform base we
4355 // are looking for. If first operand of the GEP is a splat vector - we
4356 // extract the splat value and use it as a uniform base.
4357 // In all other cases the function returns 'false'.
4358 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4359                            ISD::MemIndexType &IndexType, SDValue &Scale,
4360                            SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4361                            uint64_t ElemSize) {
4362   SelectionDAG& DAG = SDB->DAG;
4363   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4364   const DataLayout &DL = DAG.getDataLayout();
4365 
4366   assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4367 
4368   // Handle splat constant pointer.
4369   if (auto *C = dyn_cast<Constant>(Ptr)) {
4370     C = C->getSplatValue();
4371     if (!C)
4372       return false;
4373 
4374     Base = SDB->getValue(C);
4375 
4376     ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4377     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4378     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4379     IndexType = ISD::SIGNED_SCALED;
4380     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4381     return true;
4382   }
4383 
4384   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4385   if (!GEP || GEP->getParent() != CurBB)
4386     return false;
4387 
4388   if (GEP->getNumOperands() != 2)
4389     return false;
4390 
4391   const Value *BasePtr = GEP->getPointerOperand();
4392   const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4393 
4394   // Make sure the base is scalar and the index is a vector.
4395   if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4396     return false;
4397 
4398   Base = SDB->getValue(BasePtr);
4399   Index = SDB->getValue(IndexVal);
4400   IndexType = ISD::SIGNED_SCALED;
4401 
4402   // MGATHER/MSCATTER are only required to support scaling by one or by the
4403   // element size. Other scales may be produced using target-specific DAG
4404   // combines.
4405   uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4406   if (ScaleVal != ElemSize && ScaleVal != 1)
4407     return false;
4408 
4409   Scale =
4410       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4411   return true;
4412 }
4413 
4414 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4415   SDLoc sdl = getCurSDLoc();
4416 
4417   // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4418   const Value *Ptr = I.getArgOperand(1);
4419   SDValue Src0 = getValue(I.getArgOperand(0));
4420   SDValue Mask = getValue(I.getArgOperand(3));
4421   EVT VT = Src0.getValueType();
4422   Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4423                         ->getMaybeAlignValue()
4424                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4425   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4426 
4427   SDValue Base;
4428   SDValue Index;
4429   ISD::MemIndexType IndexType;
4430   SDValue Scale;
4431   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4432                                     I.getParent(), VT.getScalarStoreSize());
4433 
4434   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4435   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4436       MachinePointerInfo(AS), MachineMemOperand::MOStore,
4437       // TODO: Make MachineMemOperands aware of scalable
4438       // vectors.
4439       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4440   if (!UniformBase) {
4441     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4442     Index = getValue(Ptr);
4443     IndexType = ISD::SIGNED_SCALED;
4444     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4445   }
4446 
4447   EVT IdxVT = Index.getValueType();
4448   EVT EltTy = IdxVT.getVectorElementType();
4449   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4450     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4451     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4452   }
4453 
4454   SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4455   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4456                                          Ops, MMO, IndexType, false);
4457   DAG.setRoot(Scatter);
4458   setValue(&I, Scatter);
4459 }
4460 
4461 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4462   SDLoc sdl = getCurSDLoc();
4463 
4464   auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4465                               MaybeAlign &Alignment) {
4466     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4467     Ptr = I.getArgOperand(0);
4468     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4469     Mask = I.getArgOperand(2);
4470     Src0 = I.getArgOperand(3);
4471   };
4472   auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4473                                  MaybeAlign &Alignment) {
4474     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4475     Ptr = I.getArgOperand(0);
4476     Alignment = None;
4477     Mask = I.getArgOperand(1);
4478     Src0 = I.getArgOperand(2);
4479   };
4480 
4481   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4482   MaybeAlign Alignment;
4483   if (IsExpanding)
4484     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4485   else
4486     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4487 
4488   SDValue Ptr = getValue(PtrOperand);
4489   SDValue Src0 = getValue(Src0Operand);
4490   SDValue Mask = getValue(MaskOperand);
4491   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4492 
4493   EVT VT = Src0.getValueType();
4494   if (!Alignment)
4495     Alignment = DAG.getEVTAlign(VT);
4496 
4497   AAMDNodes AAInfo = I.getAAMetadata();
4498   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4499 
4500   // Do not serialize masked loads of constant memory with anything.
4501   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4502   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4503 
4504   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4505 
4506   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4507       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
4508       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
4509 
4510   SDValue Load =
4511       DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4512                         ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4513   if (AddToChain)
4514     PendingLoads.push_back(Load.getValue(1));
4515   setValue(&I, Load);
4516 }
4517 
4518 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4519   SDLoc sdl = getCurSDLoc();
4520 
4521   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4522   const Value *Ptr = I.getArgOperand(0);
4523   SDValue Src0 = getValue(I.getArgOperand(3));
4524   SDValue Mask = getValue(I.getArgOperand(2));
4525 
4526   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4527   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4528   Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4529                         ->getMaybeAlignValue()
4530                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4531 
4532   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4533 
4534   SDValue Root = DAG.getRoot();
4535   SDValue Base;
4536   SDValue Index;
4537   ISD::MemIndexType IndexType;
4538   SDValue Scale;
4539   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4540                                     I.getParent(), VT.getScalarStoreSize());
4541   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4542   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4543       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
4544       // TODO: Make MachineMemOperands aware of scalable
4545       // vectors.
4546       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4547 
4548   if (!UniformBase) {
4549     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4550     Index = getValue(Ptr);
4551     IndexType = ISD::SIGNED_SCALED;
4552     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4553   }
4554 
4555   EVT IdxVT = Index.getValueType();
4556   EVT EltTy = IdxVT.getVectorElementType();
4557   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4558     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4559     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4560   }
4561 
4562   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4563   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4564                                        Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4565 
4566   PendingLoads.push_back(Gather.getValue(1));
4567   setValue(&I, Gather);
4568 }
4569 
4570 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4571   SDLoc dl = getCurSDLoc();
4572   AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4573   AtomicOrdering FailureOrdering = I.getFailureOrdering();
4574   SyncScope::ID SSID = I.getSyncScopeID();
4575 
4576   SDValue InChain = getRoot();
4577 
4578   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4579   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4580 
4581   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4582   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4583 
4584   MachineFunction &MF = DAG.getMachineFunction();
4585   MachineMemOperand *MMO = MF.getMachineMemOperand(
4586       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4587       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
4588       FailureOrdering);
4589 
4590   SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4591                                    dl, MemVT, VTs, InChain,
4592                                    getValue(I.getPointerOperand()),
4593                                    getValue(I.getCompareOperand()),
4594                                    getValue(I.getNewValOperand()), MMO);
4595 
4596   SDValue OutChain = L.getValue(2);
4597 
4598   setValue(&I, L);
4599   DAG.setRoot(OutChain);
4600 }
4601 
4602 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4603   SDLoc dl = getCurSDLoc();
4604   ISD::NodeType NT;
4605   switch (I.getOperation()) {
4606   default: llvm_unreachable("Unknown atomicrmw operation");
4607   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4608   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4609   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4610   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4611   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4612   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4613   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4614   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4615   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4616   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4617   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4618   case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4619   case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4620   case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
4621   case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
4622   }
4623   AtomicOrdering Ordering = I.getOrdering();
4624   SyncScope::ID SSID = I.getSyncScopeID();
4625 
4626   SDValue InChain = getRoot();
4627 
4628   auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4629   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4630   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4631 
4632   MachineFunction &MF = DAG.getMachineFunction();
4633   MachineMemOperand *MMO = MF.getMachineMemOperand(
4634       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4635       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
4636 
4637   SDValue L =
4638     DAG.getAtomic(NT, dl, MemVT, InChain,
4639                   getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4640                   MMO);
4641 
4642   SDValue OutChain = L.getValue(1);
4643 
4644   setValue(&I, L);
4645   DAG.setRoot(OutChain);
4646 }
4647 
4648 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4649   SDLoc dl = getCurSDLoc();
4650   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4651   SDValue Ops[3];
4652   Ops[0] = getRoot();
4653   Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
4654                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4655   Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
4656                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4657   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4658 }
4659 
4660 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4661   SDLoc dl = getCurSDLoc();
4662   AtomicOrdering Order = I.getOrdering();
4663   SyncScope::ID SSID = I.getSyncScopeID();
4664 
4665   SDValue InChain = getRoot();
4666 
4667   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4668   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4669   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4670 
4671   if (!TLI.supportsUnalignedAtomics() &&
4672       I.getAlign().value() < MemVT.getSizeInBits() / 8)
4673     report_fatal_error("Cannot generate unaligned atomic load");
4674 
4675   auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4676 
4677   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4678       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4679       I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
4680 
4681   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4682 
4683   SDValue Ptr = getValue(I.getPointerOperand());
4684 
4685   if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
4686     // TODO: Once this is better exercised by tests, it should be merged with
4687     // the normal path for loads to prevent future divergence.
4688     SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4689     if (MemVT != VT)
4690       L = DAG.getPtrExtOrTrunc(L, dl, VT);
4691 
4692     setValue(&I, L);
4693     SDValue OutChain = L.getValue(1);
4694     if (!I.isUnordered())
4695       DAG.setRoot(OutChain);
4696     else
4697       PendingLoads.push_back(OutChain);
4698     return;
4699   }
4700 
4701   SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4702                             Ptr, MMO);
4703 
4704   SDValue OutChain = L.getValue(1);
4705   if (MemVT != VT)
4706     L = DAG.getPtrExtOrTrunc(L, dl, VT);
4707 
4708   setValue(&I, L);
4709   DAG.setRoot(OutChain);
4710 }
4711 
4712 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4713   SDLoc dl = getCurSDLoc();
4714 
4715   AtomicOrdering Ordering = I.getOrdering();
4716   SyncScope::ID SSID = I.getSyncScopeID();
4717 
4718   SDValue InChain = getRoot();
4719 
4720   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4721   EVT MemVT =
4722       TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4723 
4724   if (I.getAlign().value() < MemVT.getSizeInBits() / 8)
4725     report_fatal_error("Cannot generate unaligned atomic store");
4726 
4727   auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4728 
4729   MachineFunction &MF = DAG.getMachineFunction();
4730   MachineMemOperand *MMO = MF.getMachineMemOperand(
4731       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4732       I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
4733 
4734   SDValue Val = getValue(I.getValueOperand());
4735   if (Val.getValueType() != MemVT)
4736     Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4737   SDValue Ptr = getValue(I.getPointerOperand());
4738 
4739   if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
4740     // TODO: Once this is better exercised by tests, it should be merged with
4741     // the normal path for stores to prevent future divergence.
4742     SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4743     DAG.setRoot(S);
4744     return;
4745   }
4746   SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4747                                    Ptr, Val, MMO);
4748 
4749 
4750   DAG.setRoot(OutChain);
4751 }
4752 
4753 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4754 /// node.
4755 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4756                                                unsigned Intrinsic) {
4757   // Ignore the callsite's attributes. A specific call site may be marked with
4758   // readnone, but the lowering code will expect the chain based on the
4759   // definition.
4760   const Function *F = I.getCalledFunction();
4761   bool HasChain = !F->doesNotAccessMemory();
4762   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4763 
4764   // Build the operand list.
4765   SmallVector<SDValue, 8> Ops;
4766   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4767     if (OnlyLoad) {
4768       // We don't need to serialize loads against other loads.
4769       Ops.push_back(DAG.getRoot());
4770     } else {
4771       Ops.push_back(getRoot());
4772     }
4773   }
4774 
4775   // Info is set by getTgtMemIntrinsic
4776   TargetLowering::IntrinsicInfo Info;
4777   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4778   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4779                                                DAG.getMachineFunction(),
4780                                                Intrinsic);
4781 
4782   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4783   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4784       Info.opc == ISD::INTRINSIC_W_CHAIN)
4785     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4786                                         TLI.getPointerTy(DAG.getDataLayout())));
4787 
4788   // Add all operands of the call to the operand list.
4789   for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
4790     const Value *Arg = I.getArgOperand(i);
4791     if (!I.paramHasAttr(i, Attribute::ImmArg)) {
4792       Ops.push_back(getValue(Arg));
4793       continue;
4794     }
4795 
4796     // Use TargetConstant instead of a regular constant for immarg.
4797     EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
4798     if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4799       assert(CI->getBitWidth() <= 64 &&
4800              "large intrinsic immediates not handled");
4801       Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
4802     } else {
4803       Ops.push_back(
4804           DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
4805     }
4806   }
4807 
4808   SmallVector<EVT, 4> ValueVTs;
4809   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4810 
4811   if (HasChain)
4812     ValueVTs.push_back(MVT::Other);
4813 
4814   SDVTList VTs = DAG.getVTList(ValueVTs);
4815 
4816   // Propagate fast-math-flags from IR to node(s).
4817   SDNodeFlags Flags;
4818   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
4819     Flags.copyFMF(*FPMO);
4820   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
4821 
4822   // Create the node.
4823   SDValue Result;
4824   if (IsTgtIntrinsic) {
4825     // This is target intrinsic that touches memory
4826     Result =
4827         DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4828                                 MachinePointerInfo(Info.ptrVal, Info.offset),
4829                                 Info.align, Info.flags, Info.size,
4830                                 I.getAAMetadata());
4831   } else if (!HasChain) {
4832     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4833   } else if (!I.getType()->isVoidTy()) {
4834     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4835   } else {
4836     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4837   }
4838 
4839   if (HasChain) {
4840     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4841     if (OnlyLoad)
4842       PendingLoads.push_back(Chain);
4843     else
4844       DAG.setRoot(Chain);
4845   }
4846 
4847   if (!I.getType()->isVoidTy()) {
4848     if (!isa<VectorType>(I.getType()))
4849       Result = lowerRangeToAssertZExt(DAG, I, Result);
4850 
4851     MaybeAlign Alignment = I.getRetAlign();
4852     if (!Alignment)
4853       Alignment = F->getAttributes().getRetAlignment();
4854     // Insert `assertalign` node if there's an alignment.
4855     if (InsertAssertAlign && Alignment) {
4856       Result =
4857           DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
4858     }
4859 
4860     setValue(&I, Result);
4861   }
4862 }
4863 
4864 /// GetSignificand - Get the significand and build it into a floating-point
4865 /// number with exponent of 1:
4866 ///
4867 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4868 ///
4869 /// where Op is the hexadecimal representation of floating point value.
4870 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4871   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4872                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4873   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4874                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4875   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4876 }
4877 
4878 /// GetExponent - Get the exponent:
4879 ///
4880 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4881 ///
4882 /// where Op is the hexadecimal representation of floating point value.
4883 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4884                            const TargetLowering &TLI, const SDLoc &dl) {
4885   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4886                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4887   SDValue t1 = DAG.getNode(
4888       ISD::SRL, dl, MVT::i32, t0,
4889       DAG.getConstant(23, dl,
4890                       TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
4891   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4892                            DAG.getConstant(127, dl, MVT::i32));
4893   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4894 }
4895 
4896 /// getF32Constant - Get 32-bit floating point constant.
4897 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4898                               const SDLoc &dl) {
4899   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4900                            MVT::f32);
4901 }
4902 
4903 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4904                                        SelectionDAG &DAG) {
4905   // TODO: What fast-math-flags should be set on the floating-point nodes?
4906 
4907   //   IntegerPartOfX = ((int32_t)(t0);
4908   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4909 
4910   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4911   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4912   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4913 
4914   //   IntegerPartOfX <<= 23;
4915   IntegerPartOfX =
4916       DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4917                   DAG.getConstant(23, dl,
4918                                   DAG.getTargetLoweringInfo().getShiftAmountTy(
4919                                       MVT::i32, DAG.getDataLayout())));
4920 
4921   SDValue TwoToFractionalPartOfX;
4922   if (LimitFloatPrecision <= 6) {
4923     // For floating-point precision of 6:
4924     //
4925     //   TwoToFractionalPartOfX =
4926     //     0.997535578f +
4927     //       (0.735607626f + 0.252464424f * x) * x;
4928     //
4929     // error 0.0144103317, which is 6 bits
4930     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4931                              getF32Constant(DAG, 0x3e814304, dl));
4932     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4933                              getF32Constant(DAG, 0x3f3c50c8, dl));
4934     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4935     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4936                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
4937   } else if (LimitFloatPrecision <= 12) {
4938     // For floating-point precision of 12:
4939     //
4940     //   TwoToFractionalPartOfX =
4941     //     0.999892986f +
4942     //       (0.696457318f +
4943     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4944     //
4945     // error 0.000107046256, which is 13 to 14 bits
4946     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4947                              getF32Constant(DAG, 0x3da235e3, dl));
4948     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4949                              getF32Constant(DAG, 0x3e65b8f3, dl));
4950     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4951     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4952                              getF32Constant(DAG, 0x3f324b07, dl));
4953     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4954     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4955                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
4956   } else { // LimitFloatPrecision <= 18
4957     // For floating-point precision of 18:
4958     //
4959     //   TwoToFractionalPartOfX =
4960     //     0.999999982f +
4961     //       (0.693148872f +
4962     //         (0.240227044f +
4963     //           (0.554906021e-1f +
4964     //             (0.961591928e-2f +
4965     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4966     // error 2.47208000*10^(-7), which is better than 18 bits
4967     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4968                              getF32Constant(DAG, 0x3924b03e, dl));
4969     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4970                              getF32Constant(DAG, 0x3ab24b87, dl));
4971     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4972     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4973                              getF32Constant(DAG, 0x3c1d8c17, dl));
4974     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4975     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4976                              getF32Constant(DAG, 0x3d634a1d, dl));
4977     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4978     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4979                              getF32Constant(DAG, 0x3e75fe14, dl));
4980     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4981     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4982                               getF32Constant(DAG, 0x3f317234, dl));
4983     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4984     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4985                                          getF32Constant(DAG, 0x3f800000, dl));
4986   }
4987 
4988   // Add the exponent into the result in integer domain.
4989   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4990   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4991                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4992 }
4993 
4994 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4995 /// limited-precision mode.
4996 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4997                          const TargetLowering &TLI, SDNodeFlags Flags) {
4998   if (Op.getValueType() == MVT::f32 &&
4999       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5000 
5001     // Put the exponent in the right bit position for later addition to the
5002     // final result:
5003     //
5004     // t0 = Op * log2(e)
5005 
5006     // TODO: What fast-math-flags should be set here?
5007     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5008                              DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5009     return getLimitedPrecisionExp2(t0, dl, DAG);
5010   }
5011 
5012   // No special expansion.
5013   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5014 }
5015 
5016 /// expandLog - Lower a log intrinsic. Handles the special sequences for
5017 /// limited-precision mode.
5018 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5019                          const TargetLowering &TLI, SDNodeFlags Flags) {
5020   // TODO: What fast-math-flags should be set on the floating-point nodes?
5021 
5022   if (Op.getValueType() == MVT::f32 &&
5023       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5024     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5025 
5026     // Scale the exponent by log(2).
5027     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5028     SDValue LogOfExponent =
5029         DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5030                     DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5031 
5032     // Get the significand and build it into a floating-point number with
5033     // exponent of 1.
5034     SDValue X = GetSignificand(DAG, Op1, dl);
5035 
5036     SDValue LogOfMantissa;
5037     if (LimitFloatPrecision <= 6) {
5038       // For floating-point precision of 6:
5039       //
5040       //   LogofMantissa =
5041       //     -1.1609546f +
5042       //       (1.4034025f - 0.23903021f * x) * x;
5043       //
5044       // error 0.0034276066, which is better than 8 bits
5045       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5046                                getF32Constant(DAG, 0xbe74c456, dl));
5047       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5048                                getF32Constant(DAG, 0x3fb3a2b1, dl));
5049       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5050       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5051                                   getF32Constant(DAG, 0x3f949a29, dl));
5052     } else if (LimitFloatPrecision <= 12) {
5053       // For floating-point precision of 12:
5054       //
5055       //   LogOfMantissa =
5056       //     -1.7417939f +
5057       //       (2.8212026f +
5058       //         (-1.4699568f +
5059       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5060       //
5061       // error 0.000061011436, which is 14 bits
5062       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5063                                getF32Constant(DAG, 0xbd67b6d6, dl));
5064       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5065                                getF32Constant(DAG, 0x3ee4f4b8, dl));
5066       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5067       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5068                                getF32Constant(DAG, 0x3fbc278b, dl));
5069       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5070       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5071                                getF32Constant(DAG, 0x40348e95, dl));
5072       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5073       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5074                                   getF32Constant(DAG, 0x3fdef31a, dl));
5075     } else { // LimitFloatPrecision <= 18
5076       // For floating-point precision of 18:
5077       //
5078       //   LogOfMantissa =
5079       //     -2.1072184f +
5080       //       (4.2372794f +
5081       //         (-3.7029485f +
5082       //           (2.2781945f +
5083       //             (-0.87823314f +
5084       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5085       //
5086       // error 0.0000023660568, which is better than 18 bits
5087       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5088                                getF32Constant(DAG, 0xbc91e5ac, dl));
5089       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5090                                getF32Constant(DAG, 0x3e4350aa, dl));
5091       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5092       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5093                                getF32Constant(DAG, 0x3f60d3e3, dl));
5094       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5095       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5096                                getF32Constant(DAG, 0x4011cdf0, dl));
5097       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5098       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5099                                getF32Constant(DAG, 0x406cfd1c, dl));
5100       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5101       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5102                                getF32Constant(DAG, 0x408797cb, dl));
5103       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5104       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5105                                   getF32Constant(DAG, 0x4006dcab, dl));
5106     }
5107 
5108     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5109   }
5110 
5111   // No special expansion.
5112   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5113 }
5114 
5115 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5116 /// limited-precision mode.
5117 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5118                           const TargetLowering &TLI, SDNodeFlags Flags) {
5119   // TODO: What fast-math-flags should be set on the floating-point nodes?
5120 
5121   if (Op.getValueType() == MVT::f32 &&
5122       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5123     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5124 
5125     // Get the exponent.
5126     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5127 
5128     // Get the significand and build it into a floating-point number with
5129     // exponent of 1.
5130     SDValue X = GetSignificand(DAG, Op1, dl);
5131 
5132     // Different possible minimax approximations of significand in
5133     // floating-point for various degrees of accuracy over [1,2].
5134     SDValue Log2ofMantissa;
5135     if (LimitFloatPrecision <= 6) {
5136       // For floating-point precision of 6:
5137       //
5138       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5139       //
5140       // error 0.0049451742, which is more than 7 bits
5141       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5142                                getF32Constant(DAG, 0xbeb08fe0, dl));
5143       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5144                                getF32Constant(DAG, 0x40019463, dl));
5145       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5146       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5147                                    getF32Constant(DAG, 0x3fd6633d, dl));
5148     } else if (LimitFloatPrecision <= 12) {
5149       // For floating-point precision of 12:
5150       //
5151       //   Log2ofMantissa =
5152       //     -2.51285454f +
5153       //       (4.07009056f +
5154       //         (-2.12067489f +
5155       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5156       //
5157       // error 0.0000876136000, which is better than 13 bits
5158       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5159                                getF32Constant(DAG, 0xbda7262e, dl));
5160       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5161                                getF32Constant(DAG, 0x3f25280b, dl));
5162       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5163       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5164                                getF32Constant(DAG, 0x4007b923, dl));
5165       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5166       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5167                                getF32Constant(DAG, 0x40823e2f, dl));
5168       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5169       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5170                                    getF32Constant(DAG, 0x4020d29c, dl));
5171     } else { // LimitFloatPrecision <= 18
5172       // For floating-point precision of 18:
5173       //
5174       //   Log2ofMantissa =
5175       //     -3.0400495f +
5176       //       (6.1129976f +
5177       //         (-5.3420409f +
5178       //           (3.2865683f +
5179       //             (-1.2669343f +
5180       //               (0.27515199f -
5181       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5182       //
5183       // error 0.0000018516, which is better than 18 bits
5184       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5185                                getF32Constant(DAG, 0xbcd2769e, dl));
5186       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5187                                getF32Constant(DAG, 0x3e8ce0b9, dl));
5188       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5189       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5190                                getF32Constant(DAG, 0x3fa22ae7, dl));
5191       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5192       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5193                                getF32Constant(DAG, 0x40525723, dl));
5194       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5195       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5196                                getF32Constant(DAG, 0x40aaf200, dl));
5197       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5198       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5199                                getF32Constant(DAG, 0x40c39dad, dl));
5200       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5201       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5202                                    getF32Constant(DAG, 0x4042902c, dl));
5203     }
5204 
5205     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5206   }
5207 
5208   // No special expansion.
5209   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5210 }
5211 
5212 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5213 /// limited-precision mode.
5214 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5215                            const TargetLowering &TLI, SDNodeFlags Flags) {
5216   // TODO: What fast-math-flags should be set on the floating-point nodes?
5217 
5218   if (Op.getValueType() == MVT::f32 &&
5219       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5220     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5221 
5222     // Scale the exponent by log10(2) [0.30102999f].
5223     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5224     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5225                                         getF32Constant(DAG, 0x3e9a209a, dl));
5226 
5227     // Get the significand and build it into a floating-point number with
5228     // exponent of 1.
5229     SDValue X = GetSignificand(DAG, Op1, dl);
5230 
5231     SDValue Log10ofMantissa;
5232     if (LimitFloatPrecision <= 6) {
5233       // For floating-point precision of 6:
5234       //
5235       //   Log10ofMantissa =
5236       //     -0.50419619f +
5237       //       (0.60948995f - 0.10380950f * x) * x;
5238       //
5239       // error 0.0014886165, which is 6 bits
5240       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5241                                getF32Constant(DAG, 0xbdd49a13, dl));
5242       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5243                                getF32Constant(DAG, 0x3f1c0789, dl));
5244       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5245       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5246                                     getF32Constant(DAG, 0x3f011300, dl));
5247     } else if (LimitFloatPrecision <= 12) {
5248       // For floating-point precision of 12:
5249       //
5250       //   Log10ofMantissa =
5251       //     -0.64831180f +
5252       //       (0.91751397f +
5253       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5254       //
5255       // error 0.00019228036, which is better than 12 bits
5256       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5257                                getF32Constant(DAG, 0x3d431f31, dl));
5258       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5259                                getF32Constant(DAG, 0x3ea21fb2, dl));
5260       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5261       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5262                                getF32Constant(DAG, 0x3f6ae232, dl));
5263       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5264       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5265                                     getF32Constant(DAG, 0x3f25f7c3, dl));
5266     } else { // LimitFloatPrecision <= 18
5267       // For floating-point precision of 18:
5268       //
5269       //   Log10ofMantissa =
5270       //     -0.84299375f +
5271       //       (1.5327582f +
5272       //         (-1.0688956f +
5273       //           (0.49102474f +
5274       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5275       //
5276       // error 0.0000037995730, which is better than 18 bits
5277       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5278                                getF32Constant(DAG, 0x3c5d51ce, dl));
5279       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5280                                getF32Constant(DAG, 0x3e00685a, dl));
5281       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5282       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5283                                getF32Constant(DAG, 0x3efb6798, dl));
5284       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5285       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5286                                getF32Constant(DAG, 0x3f88d192, dl));
5287       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5288       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5289                                getF32Constant(DAG, 0x3fc4316c, dl));
5290       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5291       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5292                                     getF32Constant(DAG, 0x3f57ce70, dl));
5293     }
5294 
5295     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5296   }
5297 
5298   // No special expansion.
5299   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5300 }
5301 
5302 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5303 /// limited-precision mode.
5304 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5305                           const TargetLowering &TLI, SDNodeFlags Flags) {
5306   if (Op.getValueType() == MVT::f32 &&
5307       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5308     return getLimitedPrecisionExp2(Op, dl, DAG);
5309 
5310   // No special expansion.
5311   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5312 }
5313 
5314 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
5315 /// limited-precision mode with x == 10.0f.
5316 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5317                          SelectionDAG &DAG, const TargetLowering &TLI,
5318                          SDNodeFlags Flags) {
5319   bool IsExp10 = false;
5320   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5321       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5322     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5323       APFloat Ten(10.0f);
5324       IsExp10 = LHSC->isExactlyValue(Ten);
5325     }
5326   }
5327 
5328   // TODO: What fast-math-flags should be set on the FMUL node?
5329   if (IsExp10) {
5330     // Put the exponent in the right bit position for later addition to the
5331     // final result:
5332     //
5333     //   #define LOG2OF10 3.3219281f
5334     //   t0 = Op * LOG2OF10;
5335     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5336                              getF32Constant(DAG, 0x40549a78, dl));
5337     return getLimitedPrecisionExp2(t0, dl, DAG);
5338   }
5339 
5340   // No special expansion.
5341   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5342 }
5343 
5344 /// ExpandPowI - Expand a llvm.powi intrinsic.
5345 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5346                           SelectionDAG &DAG) {
5347   // If RHS is a constant, we can expand this out to a multiplication tree if
5348   // it's beneficial on the target, otherwise we end up lowering to a call to
5349   // __powidf2 (for example).
5350   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5351     unsigned Val = RHSC->getSExtValue();
5352 
5353     // powi(x, 0) -> 1.0
5354     if (Val == 0)
5355       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5356 
5357     if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
5358             Val, DAG.shouldOptForSize())) {
5359       // Get the exponent as a positive value.
5360       if ((int)Val < 0)
5361         Val = -Val;
5362       // We use the simple binary decomposition method to generate the multiply
5363       // sequence.  There are more optimal ways to do this (for example,
5364       // powi(x,15) generates one more multiply than it should), but this has
5365       // the benefit of being both really simple and much better than a libcall.
5366       SDValue Res; // Logically starts equal to 1.0
5367       SDValue CurSquare = LHS;
5368       // TODO: Intrinsics should have fast-math-flags that propagate to these
5369       // nodes.
5370       while (Val) {
5371         if (Val & 1) {
5372           if (Res.getNode())
5373             Res =
5374                 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5375           else
5376             Res = CurSquare; // 1.0*CurSquare.
5377         }
5378 
5379         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5380                                 CurSquare, CurSquare);
5381         Val >>= 1;
5382       }
5383 
5384       // If the original was negative, invert the result, producing 1/(x*x*x).
5385       if (RHSC->getSExtValue() < 0)
5386         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5387                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5388       return Res;
5389     }
5390   }
5391 
5392   // Otherwise, expand to a libcall.
5393   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5394 }
5395 
5396 static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5397                             SDValue LHS, SDValue RHS, SDValue Scale,
5398                             SelectionDAG &DAG, const TargetLowering &TLI) {
5399   EVT VT = LHS.getValueType();
5400   bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5401   bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5402   LLVMContext &Ctx = *DAG.getContext();
5403 
5404   // If the type is legal but the operation isn't, this node might survive all
5405   // the way to operation legalization. If we end up there and we do not have
5406   // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5407   // node.
5408 
5409   // Coax the legalizer into expanding the node during type legalization instead
5410   // by bumping the size by one bit. This will force it to Promote, enabling the
5411   // early expansion and avoiding the need to expand later.
5412 
5413   // We don't have to do this if Scale is 0; that can always be expanded, unless
5414   // it's a saturating signed operation. Those can experience true integer
5415   // division overflow, a case which we must avoid.
5416 
5417   // FIXME: We wouldn't have to do this (or any of the early
5418   // expansion/promotion) if it was possible to expand a libcall of an
5419   // illegal type during operation legalization. But it's not, so things
5420   // get a bit hacky.
5421   unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
5422   if ((ScaleInt > 0 || (Saturating && Signed)) &&
5423       (TLI.isTypeLegal(VT) ||
5424        (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5425     TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5426         Opcode, VT, ScaleInt);
5427     if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5428       EVT PromVT;
5429       if (VT.isScalarInteger())
5430         PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5431       else if (VT.isVector()) {
5432         PromVT = VT.getVectorElementType();
5433         PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5434         PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5435       } else
5436         llvm_unreachable("Wrong VT for DIVFIX?");
5437       if (Signed) {
5438         LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
5439         RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
5440       } else {
5441         LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
5442         RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
5443       }
5444       EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5445       // For saturating operations, we need to shift up the LHS to get the
5446       // proper saturation width, and then shift down again afterwards.
5447       if (Saturating)
5448         LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5449                           DAG.getConstant(1, DL, ShiftTy));
5450       SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5451       if (Saturating)
5452         Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5453                           DAG.getConstant(1, DL, ShiftTy));
5454       return DAG.getZExtOrTrunc(Res, DL, VT);
5455     }
5456   }
5457 
5458   return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5459 }
5460 
5461 // getUnderlyingArgRegs - Find underlying registers used for a truncated,
5462 // bitcasted, or split argument. Returns a list of <Register, size in bits>
5463 static void
5464 getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5465                      const SDValue &N) {
5466   switch (N.getOpcode()) {
5467   case ISD::CopyFromReg: {
5468     SDValue Op = N.getOperand(1);
5469     Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5470                       Op.getValueType().getSizeInBits());
5471     return;
5472   }
5473   case ISD::BITCAST:
5474   case ISD::AssertZext:
5475   case ISD::AssertSext:
5476   case ISD::TRUNCATE:
5477     getUnderlyingArgRegs(Regs, N.getOperand(0));
5478     return;
5479   case ISD::BUILD_PAIR:
5480   case ISD::BUILD_VECTOR:
5481   case ISD::CONCAT_VECTORS:
5482     for (SDValue Op : N->op_values())
5483       getUnderlyingArgRegs(Regs, Op);
5484     return;
5485   default:
5486     return;
5487   }
5488 }
5489 
5490 /// If the DbgValueInst is a dbg_value of a function argument, create the
5491 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
5492 /// instruction selection, they will be inserted to the entry BB.
5493 /// We don't currently support this for variadic dbg_values, as they shouldn't
5494 /// appear for function arguments or in the prologue.
5495 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5496     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5497     DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5498   const Argument *Arg = dyn_cast<Argument>(V);
5499   if (!Arg)
5500     return false;
5501 
5502   MachineFunction &MF = DAG.getMachineFunction();
5503   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5504 
5505   // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5506   // we've been asked to pursue.
5507   auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5508                               bool Indirect) {
5509     if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5510       // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5511       // pointing at the VReg, which will be patched up later.
5512       auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5513       auto MIB = BuildMI(MF, DL, Inst);
5514       MIB.addReg(Reg);
5515       MIB.addImm(0);
5516       MIB.addMetadata(Variable);
5517       auto *NewDIExpr = FragExpr;
5518       // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5519       // the DIExpression.
5520       if (Indirect)
5521         NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5522       MIB.addMetadata(NewDIExpr);
5523       return MIB;
5524     } else {
5525       // Create a completely standard DBG_VALUE.
5526       auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5527       return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5528     }
5529   };
5530 
5531   if (Kind == FuncArgumentDbgValueKind::Value) {
5532     // ArgDbgValues are hoisted to the beginning of the entry block. So we
5533     // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5534     // the entry block.
5535     bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5536     if (!IsInEntryBlock)
5537       return false;
5538 
5539     // ArgDbgValues are hoisted to the beginning of the entry block.  So we
5540     // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5541     // variable that also is a param.
5542     //
5543     // Although, if we are at the top of the entry block already, we can still
5544     // emit using ArgDbgValue. This might catch some situations when the
5545     // dbg.value refers to an argument that isn't used in the entry block, so
5546     // any CopyToReg node would be optimized out and the only way to express
5547     // this DBG_VALUE is by using the physical reg (or FI) as done in this
5548     // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
5549     // we should only emit as ArgDbgValue if the Variable is an argument to the
5550     // current function, and the dbg.value intrinsic is found in the entry
5551     // block.
5552     bool VariableIsFunctionInputArg = Variable->isParameter() &&
5553         !DL->getInlinedAt();
5554     bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5555     if (!IsInPrologue && !VariableIsFunctionInputArg)
5556       return false;
5557 
5558     // Here we assume that a function argument on IR level only can be used to
5559     // describe one input parameter on source level. If we for example have
5560     // source code like this
5561     //
5562     //    struct A { long x, y; };
5563     //    void foo(struct A a, long b) {
5564     //      ...
5565     //      b = a.x;
5566     //      ...
5567     //    }
5568     //
5569     // and IR like this
5570     //
5571     //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
5572     //  entry:
5573     //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5574     //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5575     //    call void @llvm.dbg.value(metadata i32 %b, "b",
5576     //    ...
5577     //    call void @llvm.dbg.value(metadata i32 %a1, "b"
5578     //    ...
5579     //
5580     // then the last dbg.value is describing a parameter "b" using a value that
5581     // is an argument. But since we already has used %a1 to describe a parameter
5582     // we should not handle that last dbg.value here (that would result in an
5583     // incorrect hoisting of the DBG_VALUE to the function entry).
5584     // Notice that we allow one dbg.value per IR level argument, to accommodate
5585     // for the situation with fragments above.
5586     if (VariableIsFunctionInputArg) {
5587       unsigned ArgNo = Arg->getArgNo();
5588       if (ArgNo >= FuncInfo.DescribedArgs.size())
5589         FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
5590       else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
5591         return false;
5592       FuncInfo.DescribedArgs.set(ArgNo);
5593     }
5594   }
5595 
5596   bool IsIndirect = false;
5597   Optional<MachineOperand> Op;
5598   // Some arguments' frame index is recorded during argument lowering.
5599   int FI = FuncInfo.getArgumentFrameIndex(Arg);
5600   if (FI != std::numeric_limits<int>::max())
5601     Op = MachineOperand::CreateFI(FI);
5602 
5603   SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
5604   if (!Op && N.getNode()) {
5605     getUnderlyingArgRegs(ArgRegsAndSizes, N);
5606     Register Reg;
5607     if (ArgRegsAndSizes.size() == 1)
5608       Reg = ArgRegsAndSizes.front().first;
5609 
5610     if (Reg && Reg.isVirtual()) {
5611       MachineRegisterInfo &RegInfo = MF.getRegInfo();
5612       Register PR = RegInfo.getLiveInPhysReg(Reg);
5613       if (PR)
5614         Reg = PR;
5615     }
5616     if (Reg) {
5617       Op = MachineOperand::CreateReg(Reg, false);
5618       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5619     }
5620   }
5621 
5622   if (!Op && N.getNode()) {
5623     // Check if frame index is available.
5624     SDValue LCandidate = peekThroughBitcasts(N);
5625     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
5626       if (FrameIndexSDNode *FINode =
5627           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
5628         Op = MachineOperand::CreateFI(FINode->getIndex());
5629   }
5630 
5631   if (!Op) {
5632     // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
5633     auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
5634                                          SplitRegs) {
5635       unsigned Offset = 0;
5636       for (const auto &RegAndSize : SplitRegs) {
5637         // If the expression is already a fragment, the current register
5638         // offset+size might extend beyond the fragment. In this case, only
5639         // the register bits that are inside the fragment are relevant.
5640         int RegFragmentSizeInBits = RegAndSize.second;
5641         if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
5642           uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
5643           // The register is entirely outside the expression fragment,
5644           // so is irrelevant for debug info.
5645           if (Offset >= ExprFragmentSizeInBits)
5646             break;
5647           // The register is partially outside the expression fragment, only
5648           // the low bits within the fragment are relevant for debug info.
5649           if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
5650             RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
5651           }
5652         }
5653 
5654         auto FragmentExpr = DIExpression::createFragmentExpression(
5655             Expr, Offset, RegFragmentSizeInBits);
5656         Offset += RegAndSize.second;
5657         // If a valid fragment expression cannot be created, the variable's
5658         // correct value cannot be determined and so it is set as Undef.
5659         if (!FragmentExpr) {
5660           SDDbgValue *SDV = DAG.getConstantDbgValue(
5661               Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
5662           DAG.AddDbgValue(SDV, false);
5663           continue;
5664         }
5665         MachineInstr *NewMI =
5666             MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
5667                              Kind != FuncArgumentDbgValueKind::Value);
5668         FuncInfo.ArgDbgValues.push_back(NewMI);
5669       }
5670     };
5671 
5672     // Check if ValueMap has reg number.
5673     DenseMap<const Value *, Register>::const_iterator
5674       VMI = FuncInfo.ValueMap.find(V);
5675     if (VMI != FuncInfo.ValueMap.end()) {
5676       const auto &TLI = DAG.getTargetLoweringInfo();
5677       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
5678                        V->getType(), None);
5679       if (RFV.occupiesMultipleRegs()) {
5680         splitMultiRegDbgValue(RFV.getRegsAndSizes());
5681         return true;
5682       }
5683 
5684       Op = MachineOperand::CreateReg(VMI->second, false);
5685       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5686     } else if (ArgRegsAndSizes.size() > 1) {
5687       // This was split due to the calling convention, and no virtual register
5688       // mapping exists for the value.
5689       splitMultiRegDbgValue(ArgRegsAndSizes);
5690       return true;
5691     }
5692   }
5693 
5694   if (!Op)
5695     return false;
5696 
5697   assert(Variable->isValidLocationForIntrinsic(DL) &&
5698          "Expected inlined-at fields to agree");
5699   MachineInstr *NewMI = nullptr;
5700 
5701   if (Op->isReg())
5702     NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
5703   else
5704     NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
5705                     Variable, Expr);
5706 
5707   // Otherwise, use ArgDbgValues.
5708   FuncInfo.ArgDbgValues.push_back(NewMI);
5709   return true;
5710 }
5711 
5712 /// Return the appropriate SDDbgValue based on N.
5713 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5714                                              DILocalVariable *Variable,
5715                                              DIExpression *Expr,
5716                                              const DebugLoc &dl,
5717                                              unsigned DbgSDNodeOrder) {
5718   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5719     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5720     // stack slot locations.
5721     //
5722     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5723     // debug values here after optimization:
5724     //
5725     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5726     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5727     //
5728     // Both describe the direct values of their associated variables.
5729     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5730                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5731   }
5732   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5733                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5734 }
5735 
5736 static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
5737   switch (Intrinsic) {
5738   case Intrinsic::smul_fix:
5739     return ISD::SMULFIX;
5740   case Intrinsic::umul_fix:
5741     return ISD::UMULFIX;
5742   case Intrinsic::smul_fix_sat:
5743     return ISD::SMULFIXSAT;
5744   case Intrinsic::umul_fix_sat:
5745     return ISD::UMULFIXSAT;
5746   case Intrinsic::sdiv_fix:
5747     return ISD::SDIVFIX;
5748   case Intrinsic::udiv_fix:
5749     return ISD::UDIVFIX;
5750   case Intrinsic::sdiv_fix_sat:
5751     return ISD::SDIVFIXSAT;
5752   case Intrinsic::udiv_fix_sat:
5753     return ISD::UDIVFIXSAT;
5754   default:
5755     llvm_unreachable("Unhandled fixed point intrinsic");
5756   }
5757 }
5758 
5759 void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
5760                                            const char *FunctionName) {
5761   assert(FunctionName && "FunctionName must not be nullptr");
5762   SDValue Callee = DAG.getExternalSymbol(
5763       FunctionName,
5764       DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
5765   LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5766 }
5767 
5768 /// Given a @llvm.call.preallocated.setup, return the corresponding
5769 /// preallocated call.
5770 static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
5771   assert(cast<CallBase>(PreallocatedSetup)
5772                  ->getCalledFunction()
5773                  ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
5774          "expected call_preallocated_setup Value");
5775   for (auto *U : PreallocatedSetup->users()) {
5776     auto *UseCall = cast<CallBase>(U);
5777     const Function *Fn = UseCall->getCalledFunction();
5778     if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
5779       return UseCall;
5780     }
5781   }
5782   llvm_unreachable("expected corresponding call to preallocated setup/arg");
5783 }
5784 
5785 /// Lower the call to the specified intrinsic function.
5786 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
5787                                              unsigned Intrinsic) {
5788   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5789   SDLoc sdl = getCurSDLoc();
5790   DebugLoc dl = getCurDebugLoc();
5791   SDValue Res;
5792 
5793   SDNodeFlags Flags;
5794   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
5795     Flags.copyFMF(*FPOp);
5796 
5797   switch (Intrinsic) {
5798   default:
5799     // By default, turn this into a target intrinsic node.
5800     visitTargetIntrinsic(I, Intrinsic);
5801     return;
5802   case Intrinsic::vscale: {
5803     match(&I, m_VScale(DAG.getDataLayout()));
5804     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5805     setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
5806     return;
5807   }
5808   case Intrinsic::vastart:  visitVAStart(I); return;
5809   case Intrinsic::vaend:    visitVAEnd(I); return;
5810   case Intrinsic::vacopy:   visitVACopy(I); return;
5811   case Intrinsic::returnaddress:
5812     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5813                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5814                              getValue(I.getArgOperand(0))));
5815     return;
5816   case Intrinsic::addressofreturnaddress:
5817     setValue(&I,
5818              DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5819                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5820     return;
5821   case Intrinsic::sponentry:
5822     setValue(&I,
5823              DAG.getNode(ISD::SPONENTRY, sdl,
5824                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5825     return;
5826   case Intrinsic::frameaddress:
5827     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5828                              TLI.getFrameIndexTy(DAG.getDataLayout()),
5829                              getValue(I.getArgOperand(0))));
5830     return;
5831   case Intrinsic::read_volatile_register:
5832   case Intrinsic::read_register: {
5833     Value *Reg = I.getArgOperand(0);
5834     SDValue Chain = getRoot();
5835     SDValue RegName =
5836         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5837     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5838     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5839       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5840     setValue(&I, Res);
5841     DAG.setRoot(Res.getValue(1));
5842     return;
5843   }
5844   case Intrinsic::write_register: {
5845     Value *Reg = I.getArgOperand(0);
5846     Value *RegValue = I.getArgOperand(1);
5847     SDValue Chain = getRoot();
5848     SDValue RegName =
5849         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5850     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5851                             RegName, getValue(RegValue)));
5852     return;
5853   }
5854   case Intrinsic::memcpy: {
5855     const auto &MCI = cast<MemCpyInst>(I);
5856     SDValue Op1 = getValue(I.getArgOperand(0));
5857     SDValue Op2 = getValue(I.getArgOperand(1));
5858     SDValue Op3 = getValue(I.getArgOperand(2));
5859     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5860     Align DstAlign = MCI.getDestAlign().valueOrOne();
5861     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5862     Align Alignment = std::min(DstAlign, SrcAlign);
5863     bool isVol = MCI.isVolatile();
5864     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5865     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5866     // node.
5867     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5868     SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5869                                /* AlwaysInline */ false, isTC,
5870                                MachinePointerInfo(I.getArgOperand(0)),
5871                                MachinePointerInfo(I.getArgOperand(1)),
5872                                I.getAAMetadata());
5873     updateDAGForMaybeTailCall(MC);
5874     return;
5875   }
5876   case Intrinsic::memcpy_inline: {
5877     const auto &MCI = cast<MemCpyInlineInst>(I);
5878     SDValue Dst = getValue(I.getArgOperand(0));
5879     SDValue Src = getValue(I.getArgOperand(1));
5880     SDValue Size = getValue(I.getArgOperand(2));
5881     assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
5882     // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
5883     Align DstAlign = MCI.getDestAlign().valueOrOne();
5884     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5885     Align Alignment = std::min(DstAlign, SrcAlign);
5886     bool isVol = MCI.isVolatile();
5887     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5888     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5889     // node.
5890     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
5891                                /* AlwaysInline */ true, isTC,
5892                                MachinePointerInfo(I.getArgOperand(0)),
5893                                MachinePointerInfo(I.getArgOperand(1)),
5894                                I.getAAMetadata());
5895     updateDAGForMaybeTailCall(MC);
5896     return;
5897   }
5898   case Intrinsic::memset: {
5899     const auto &MSI = cast<MemSetInst>(I);
5900     SDValue Op1 = getValue(I.getArgOperand(0));
5901     SDValue Op2 = getValue(I.getArgOperand(1));
5902     SDValue Op3 = getValue(I.getArgOperand(2));
5903     // @llvm.memset defines 0 and 1 to both mean no alignment.
5904     Align Alignment = MSI.getDestAlign().valueOrOne();
5905     bool isVol = MSI.isVolatile();
5906     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5907     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5908     SDValue MS = DAG.getMemset(
5909         Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
5910         isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
5911     updateDAGForMaybeTailCall(MS);
5912     return;
5913   }
5914   case Intrinsic::memset_inline: {
5915     const auto &MSII = cast<MemSetInlineInst>(I);
5916     SDValue Dst = getValue(I.getArgOperand(0));
5917     SDValue Value = getValue(I.getArgOperand(1));
5918     SDValue Size = getValue(I.getArgOperand(2));
5919     assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
5920     // @llvm.memset defines 0 and 1 to both mean no alignment.
5921     Align DstAlign = MSII.getDestAlign().valueOrOne();
5922     bool isVol = MSII.isVolatile();
5923     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5924     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5925     SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
5926                                /* AlwaysInline */ true, isTC,
5927                                MachinePointerInfo(I.getArgOperand(0)),
5928                                I.getAAMetadata());
5929     updateDAGForMaybeTailCall(MC);
5930     return;
5931   }
5932   case Intrinsic::memmove: {
5933     const auto &MMI = cast<MemMoveInst>(I);
5934     SDValue Op1 = getValue(I.getArgOperand(0));
5935     SDValue Op2 = getValue(I.getArgOperand(1));
5936     SDValue Op3 = getValue(I.getArgOperand(2));
5937     // @llvm.memmove defines 0 and 1 to both mean no alignment.
5938     Align DstAlign = MMI.getDestAlign().valueOrOne();
5939     Align SrcAlign = MMI.getSourceAlign().valueOrOne();
5940     Align Alignment = std::min(DstAlign, SrcAlign);
5941     bool isVol = MMI.isVolatile();
5942     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5943     // FIXME: Support passing different dest/src alignments to the memmove DAG
5944     // node.
5945     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5946     SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5947                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
5948                                 MachinePointerInfo(I.getArgOperand(1)),
5949                                 I.getAAMetadata());
5950     updateDAGForMaybeTailCall(MM);
5951     return;
5952   }
5953   case Intrinsic::memcpy_element_unordered_atomic: {
5954     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5955     SDValue Dst = getValue(MI.getRawDest());
5956     SDValue Src = getValue(MI.getRawSource());
5957     SDValue Length = getValue(MI.getLength());
5958 
5959     Type *LengthTy = MI.getLength()->getType();
5960     unsigned ElemSz = MI.getElementSizeInBytes();
5961     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5962     SDValue MC =
5963         DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
5964                             isTC, MachinePointerInfo(MI.getRawDest()),
5965                             MachinePointerInfo(MI.getRawSource()));
5966     updateDAGForMaybeTailCall(MC);
5967     return;
5968   }
5969   case Intrinsic::memmove_element_unordered_atomic: {
5970     auto &MI = cast<AtomicMemMoveInst>(I);
5971     SDValue Dst = getValue(MI.getRawDest());
5972     SDValue Src = getValue(MI.getRawSource());
5973     SDValue Length = getValue(MI.getLength());
5974 
5975     Type *LengthTy = MI.getLength()->getType();
5976     unsigned ElemSz = MI.getElementSizeInBytes();
5977     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5978     SDValue MC =
5979         DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
5980                              isTC, MachinePointerInfo(MI.getRawDest()),
5981                              MachinePointerInfo(MI.getRawSource()));
5982     updateDAGForMaybeTailCall(MC);
5983     return;
5984   }
5985   case Intrinsic::memset_element_unordered_atomic: {
5986     auto &MI = cast<AtomicMemSetInst>(I);
5987     SDValue Dst = getValue(MI.getRawDest());
5988     SDValue Val = getValue(MI.getValue());
5989     SDValue Length = getValue(MI.getLength());
5990 
5991     Type *LengthTy = MI.getLength()->getType();
5992     unsigned ElemSz = MI.getElementSizeInBytes();
5993     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5994     SDValue MC =
5995         DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
5996                             isTC, MachinePointerInfo(MI.getRawDest()));
5997     updateDAGForMaybeTailCall(MC);
5998     return;
5999   }
6000   case Intrinsic::call_preallocated_setup: {
6001     const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6002     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6003     SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6004                               getRoot(), SrcValue);
6005     setValue(&I, Res);
6006     DAG.setRoot(Res);
6007     return;
6008   }
6009   case Intrinsic::call_preallocated_arg: {
6010     const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6011     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6012     SDValue Ops[3];
6013     Ops[0] = getRoot();
6014     Ops[1] = SrcValue;
6015     Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6016                                    MVT::i32); // arg index
6017     SDValue Res = DAG.getNode(
6018         ISD::PREALLOCATED_ARG, sdl,
6019         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6020     setValue(&I, Res);
6021     DAG.setRoot(Res.getValue(1));
6022     return;
6023   }
6024   case Intrinsic::dbg_addr:
6025   case Intrinsic::dbg_declare: {
6026     // Assume dbg.addr and dbg.declare can not currently use DIArgList, i.e.
6027     // they are non-variadic.
6028     const auto &DI = cast<DbgVariableIntrinsic>(I);
6029     assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6030     DILocalVariable *Variable = DI.getVariable();
6031     DIExpression *Expression = DI.getExpression();
6032     dropDanglingDebugInfo(Variable, Expression);
6033     assert(Variable && "Missing variable");
6034     LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DI
6035                       << "\n");
6036     // Check if address has undef value.
6037     const Value *Address = DI.getVariableLocationOp(0);
6038     if (!Address || isa<UndefValue>(Address) ||
6039         (Address->use_empty() && !isa<Argument>(Address))) {
6040       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6041                         << " (bad/undef/unused-arg address)\n");
6042       return;
6043     }
6044 
6045     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
6046 
6047     // Check if this variable can be described by a frame index, typically
6048     // either as a static alloca or a byval parameter.
6049     int FI = std::numeric_limits<int>::max();
6050     if (const auto *AI =
6051             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
6052       if (AI->isStaticAlloca()) {
6053         auto I = FuncInfo.StaticAllocaMap.find(AI);
6054         if (I != FuncInfo.StaticAllocaMap.end())
6055           FI = I->second;
6056       }
6057     } else if (const auto *Arg = dyn_cast<Argument>(
6058                    Address->stripInBoundsConstantOffsets())) {
6059       FI = FuncInfo.getArgumentFrameIndex(Arg);
6060     }
6061 
6062     // llvm.dbg.addr is control dependent and always generates indirect
6063     // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
6064     // the MachineFunction variable table.
6065     if (FI != std::numeric_limits<int>::max()) {
6066       if (Intrinsic == Intrinsic::dbg_addr) {
6067         SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
6068             Variable, Expression, FI, getRoot().getNode(), /*IsIndirect*/ true,
6069             dl, SDNodeOrder);
6070         DAG.AddDbgValue(SDV, isParameter);
6071       } else {
6072         LLVM_DEBUG(dbgs() << "Skipping " << DI
6073                           << " (variable info stashed in MF side table)\n");
6074       }
6075       return;
6076     }
6077 
6078     SDValue &N = NodeMap[Address];
6079     if (!N.getNode() && isa<Argument>(Address))
6080       // Check unused arguments map.
6081       N = UnusedArgNodeMap[Address];
6082     SDDbgValue *SDV;
6083     if (N.getNode()) {
6084       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
6085         Address = BCI->getOperand(0);
6086       // Parameters are handled specially.
6087       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
6088       if (isParameter && FINode) {
6089         // Byval parameter. We have a frame index at this point.
6090         SDV =
6091             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
6092                                       /*IsIndirect*/ true, dl, SDNodeOrder);
6093       } else if (isa<Argument>(Address)) {
6094         // Address is an argument, so try to emit its dbg value using
6095         // virtual register info from the FuncInfo.ValueMap.
6096         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6097                                  FuncArgumentDbgValueKind::Declare, N);
6098         return;
6099       } else {
6100         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
6101                               true, dl, SDNodeOrder);
6102       }
6103       DAG.AddDbgValue(SDV, isParameter);
6104     } else {
6105       // If Address is an argument then try to emit its dbg value using
6106       // virtual register info from the FuncInfo.ValueMap.
6107       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6108                                     FuncArgumentDbgValueKind::Declare, N)) {
6109         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6110                           << " (could not emit func-arg dbg_value)\n");
6111       }
6112     }
6113     return;
6114   }
6115   case Intrinsic::dbg_label: {
6116     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6117     DILabel *Label = DI.getLabel();
6118     assert(Label && "Missing label");
6119 
6120     SDDbgLabel *SDV;
6121     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6122     DAG.AddDbgLabel(SDV);
6123     return;
6124   }
6125   case Intrinsic::dbg_value: {
6126     const DbgValueInst &DI = cast<DbgValueInst>(I);
6127     assert(DI.getVariable() && "Missing variable");
6128 
6129     DILocalVariable *Variable = DI.getVariable();
6130     DIExpression *Expression = DI.getExpression();
6131     dropDanglingDebugInfo(Variable, Expression);
6132     SmallVector<Value *, 4> Values(DI.getValues());
6133     if (Values.empty())
6134       return;
6135 
6136     if (llvm::is_contained(Values, nullptr))
6137       return;
6138 
6139     bool IsVariadic = DI.hasArgList();
6140     if (!handleDebugValue(Values, Variable, Expression, dl, DI.getDebugLoc(),
6141                           SDNodeOrder, IsVariadic))
6142       addDanglingDebugInfo(&DI, dl, SDNodeOrder);
6143     return;
6144   }
6145 
6146   case Intrinsic::eh_typeid_for: {
6147     // Find the type id for the given typeinfo.
6148     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6149     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6150     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6151     setValue(&I, Res);
6152     return;
6153   }
6154 
6155   case Intrinsic::eh_return_i32:
6156   case Intrinsic::eh_return_i64:
6157     DAG.getMachineFunction().setCallsEHReturn(true);
6158     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6159                             MVT::Other,
6160                             getControlRoot(),
6161                             getValue(I.getArgOperand(0)),
6162                             getValue(I.getArgOperand(1))));
6163     return;
6164   case Intrinsic::eh_unwind_init:
6165     DAG.getMachineFunction().setCallsUnwindInit(true);
6166     return;
6167   case Intrinsic::eh_dwarf_cfa:
6168     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
6169                              TLI.getPointerTy(DAG.getDataLayout()),
6170                              getValue(I.getArgOperand(0))));
6171     return;
6172   case Intrinsic::eh_sjlj_callsite: {
6173     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6174     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6175     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6176 
6177     MMI.setCurrentCallSite(CI->getZExtValue());
6178     return;
6179   }
6180   case Intrinsic::eh_sjlj_functioncontext: {
6181     // Get and store the index of the function context.
6182     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6183     AllocaInst *FnCtx =
6184       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6185     int FI = FuncInfo.StaticAllocaMap[FnCtx];
6186     MFI.setFunctionContextIndex(FI);
6187     return;
6188   }
6189   case Intrinsic::eh_sjlj_setjmp: {
6190     SDValue Ops[2];
6191     Ops[0] = getRoot();
6192     Ops[1] = getValue(I.getArgOperand(0));
6193     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6194                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
6195     setValue(&I, Op.getValue(0));
6196     DAG.setRoot(Op.getValue(1));
6197     return;
6198   }
6199   case Intrinsic::eh_sjlj_longjmp:
6200     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6201                             getRoot(), getValue(I.getArgOperand(0))));
6202     return;
6203   case Intrinsic::eh_sjlj_setup_dispatch:
6204     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6205                             getRoot()));
6206     return;
6207   case Intrinsic::masked_gather:
6208     visitMaskedGather(I);
6209     return;
6210   case Intrinsic::masked_load:
6211     visitMaskedLoad(I);
6212     return;
6213   case Intrinsic::masked_scatter:
6214     visitMaskedScatter(I);
6215     return;
6216   case Intrinsic::masked_store:
6217     visitMaskedStore(I);
6218     return;
6219   case Intrinsic::masked_expandload:
6220     visitMaskedLoad(I, true /* IsExpanding */);
6221     return;
6222   case Intrinsic::masked_compressstore:
6223     visitMaskedStore(I, true /* IsCompressing */);
6224     return;
6225   case Intrinsic::powi:
6226     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6227                             getValue(I.getArgOperand(1)), DAG));
6228     return;
6229   case Intrinsic::log:
6230     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6231     return;
6232   case Intrinsic::log2:
6233     setValue(&I,
6234              expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6235     return;
6236   case Intrinsic::log10:
6237     setValue(&I,
6238              expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6239     return;
6240   case Intrinsic::exp:
6241     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6242     return;
6243   case Intrinsic::exp2:
6244     setValue(&I,
6245              expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6246     return;
6247   case Intrinsic::pow:
6248     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6249                            getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6250     return;
6251   case Intrinsic::sqrt:
6252   case Intrinsic::fabs:
6253   case Intrinsic::sin:
6254   case Intrinsic::cos:
6255   case Intrinsic::floor:
6256   case Intrinsic::ceil:
6257   case Intrinsic::trunc:
6258   case Intrinsic::rint:
6259   case Intrinsic::nearbyint:
6260   case Intrinsic::round:
6261   case Intrinsic::roundeven:
6262   case Intrinsic::canonicalize: {
6263     unsigned Opcode;
6264     switch (Intrinsic) {
6265     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6266     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
6267     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
6268     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
6269     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
6270     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
6271     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
6272     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
6273     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
6274     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6275     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
6276     case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6277     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6278     }
6279 
6280     setValue(&I, DAG.getNode(Opcode, sdl,
6281                              getValue(I.getArgOperand(0)).getValueType(),
6282                              getValue(I.getArgOperand(0)), Flags));
6283     return;
6284   }
6285   case Intrinsic::lround:
6286   case Intrinsic::llround:
6287   case Intrinsic::lrint:
6288   case Intrinsic::llrint: {
6289     unsigned Opcode;
6290     switch (Intrinsic) {
6291     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6292     case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
6293     case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6294     case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
6295     case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
6296     }
6297 
6298     EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6299     setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6300                              getValue(I.getArgOperand(0))));
6301     return;
6302   }
6303   case Intrinsic::minnum:
6304     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
6305                              getValue(I.getArgOperand(0)).getValueType(),
6306                              getValue(I.getArgOperand(0)),
6307                              getValue(I.getArgOperand(1)), Flags));
6308     return;
6309   case Intrinsic::maxnum:
6310     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
6311                              getValue(I.getArgOperand(0)).getValueType(),
6312                              getValue(I.getArgOperand(0)),
6313                              getValue(I.getArgOperand(1)), Flags));
6314     return;
6315   case Intrinsic::minimum:
6316     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
6317                              getValue(I.getArgOperand(0)).getValueType(),
6318                              getValue(I.getArgOperand(0)),
6319                              getValue(I.getArgOperand(1)), Flags));
6320     return;
6321   case Intrinsic::maximum:
6322     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
6323                              getValue(I.getArgOperand(0)).getValueType(),
6324                              getValue(I.getArgOperand(0)),
6325                              getValue(I.getArgOperand(1)), Flags));
6326     return;
6327   case Intrinsic::copysign:
6328     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
6329                              getValue(I.getArgOperand(0)).getValueType(),
6330                              getValue(I.getArgOperand(0)),
6331                              getValue(I.getArgOperand(1)), Flags));
6332     return;
6333   case Intrinsic::arithmetic_fence: {
6334     setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
6335                              getValue(I.getArgOperand(0)).getValueType(),
6336                              getValue(I.getArgOperand(0)), Flags));
6337     return;
6338   }
6339   case Intrinsic::fma:
6340     setValue(&I, DAG.getNode(
6341                      ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6342                      getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6343                      getValue(I.getArgOperand(2)), Flags));
6344     return;
6345 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
6346   case Intrinsic::INTRINSIC:
6347 #include "llvm/IR/ConstrainedOps.def"
6348     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6349     return;
6350 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6351 #include "llvm/IR/VPIntrinsics.def"
6352     visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6353     return;
6354   case Intrinsic::fptrunc_round: {
6355     // Get the last argument, the metadata and convert it to an integer in the
6356     // call
6357     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6358     Optional<RoundingMode> RoundMode =
6359         convertStrToRoundingMode(cast<MDString>(MD)->getString());
6360 
6361     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6362 
6363     // Propagate fast-math-flags from IR to node(s).
6364     SDNodeFlags Flags;
6365     Flags.copyFMF(*cast<FPMathOperator>(&I));
6366     SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6367 
6368     SDValue Result;
6369     Result = DAG.getNode(
6370         ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6371         DAG.getTargetConstant((int)*RoundMode, sdl,
6372                               TLI.getPointerTy(DAG.getDataLayout())));
6373     setValue(&I, Result);
6374 
6375     return;
6376   }
6377   case Intrinsic::fmuladd: {
6378     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6379     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6380         TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
6381       setValue(&I, DAG.getNode(ISD::FMA, sdl,
6382                                getValue(I.getArgOperand(0)).getValueType(),
6383                                getValue(I.getArgOperand(0)),
6384                                getValue(I.getArgOperand(1)),
6385                                getValue(I.getArgOperand(2)), Flags));
6386     } else {
6387       // TODO: Intrinsic calls should have fast-math-flags.
6388       SDValue Mul = DAG.getNode(
6389           ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6390           getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6391       SDValue Add = DAG.getNode(ISD::FADD, sdl,
6392                                 getValue(I.getArgOperand(0)).getValueType(),
6393                                 Mul, getValue(I.getArgOperand(2)), Flags);
6394       setValue(&I, Add);
6395     }
6396     return;
6397   }
6398   case Intrinsic::convert_to_fp16:
6399     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6400                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6401                                          getValue(I.getArgOperand(0)),
6402                                          DAG.getTargetConstant(0, sdl,
6403                                                                MVT::i32))));
6404     return;
6405   case Intrinsic::convert_from_fp16:
6406     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6407                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
6408                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6409                                          getValue(I.getArgOperand(0)))));
6410     return;
6411   case Intrinsic::fptosi_sat: {
6412     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6413     setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
6414                              getValue(I.getArgOperand(0)),
6415                              DAG.getValueType(VT.getScalarType())));
6416     return;
6417   }
6418   case Intrinsic::fptoui_sat: {
6419     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6420     setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
6421                              getValue(I.getArgOperand(0)),
6422                              DAG.getValueType(VT.getScalarType())));
6423     return;
6424   }
6425   case Intrinsic::set_rounding:
6426     Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6427                       {getRoot(), getValue(I.getArgOperand(0))});
6428     setValue(&I, Res);
6429     DAG.setRoot(Res.getValue(0));
6430     return;
6431   case Intrinsic::is_fpclass: {
6432     const DataLayout DLayout = DAG.getDataLayout();
6433     EVT DestVT = TLI.getValueType(DLayout, I.getType());
6434     EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6435     unsigned Test = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6436     MachineFunction &MF = DAG.getMachineFunction();
6437     const Function &F = MF.getFunction();
6438     SDValue Op = getValue(I.getArgOperand(0));
6439     SDNodeFlags Flags;
6440     Flags.setNoFPExcept(
6441         !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6442     // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6443     // expansion can use illegal types. Making expansion early allows
6444     // legalizing these types prior to selection.
6445     if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6446       SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6447       setValue(&I, Result);
6448       return;
6449     }
6450 
6451     SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6452     SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6453     setValue(&I, V);
6454     return;
6455   }
6456   case Intrinsic::pcmarker: {
6457     SDValue Tmp = getValue(I.getArgOperand(0));
6458     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6459     return;
6460   }
6461   case Intrinsic::readcyclecounter: {
6462     SDValue Op = getRoot();
6463     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
6464                       DAG.getVTList(MVT::i64, MVT::Other), Op);
6465     setValue(&I, Res);
6466     DAG.setRoot(Res.getValue(1));
6467     return;
6468   }
6469   case Intrinsic::bitreverse:
6470     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
6471                              getValue(I.getArgOperand(0)).getValueType(),
6472                              getValue(I.getArgOperand(0))));
6473     return;
6474   case Intrinsic::bswap:
6475     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
6476                              getValue(I.getArgOperand(0)).getValueType(),
6477                              getValue(I.getArgOperand(0))));
6478     return;
6479   case Intrinsic::cttz: {
6480     SDValue Arg = getValue(I.getArgOperand(0));
6481     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6482     EVT Ty = Arg.getValueType();
6483     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
6484                              sdl, Ty, Arg));
6485     return;
6486   }
6487   case Intrinsic::ctlz: {
6488     SDValue Arg = getValue(I.getArgOperand(0));
6489     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6490     EVT Ty = Arg.getValueType();
6491     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
6492                              sdl, Ty, Arg));
6493     return;
6494   }
6495   case Intrinsic::ctpop: {
6496     SDValue Arg = getValue(I.getArgOperand(0));
6497     EVT Ty = Arg.getValueType();
6498     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
6499     return;
6500   }
6501   case Intrinsic::fshl:
6502   case Intrinsic::fshr: {
6503     bool IsFSHL = Intrinsic == Intrinsic::fshl;
6504     SDValue X = getValue(I.getArgOperand(0));
6505     SDValue Y = getValue(I.getArgOperand(1));
6506     SDValue Z = getValue(I.getArgOperand(2));
6507     EVT VT = X.getValueType();
6508 
6509     if (X == Y) {
6510       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
6511       setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
6512     } else {
6513       auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
6514       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
6515     }
6516     return;
6517   }
6518   case Intrinsic::sadd_sat: {
6519     SDValue Op1 = getValue(I.getArgOperand(0));
6520     SDValue Op2 = getValue(I.getArgOperand(1));
6521     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6522     return;
6523   }
6524   case Intrinsic::uadd_sat: {
6525     SDValue Op1 = getValue(I.getArgOperand(0));
6526     SDValue Op2 = getValue(I.getArgOperand(1));
6527     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6528     return;
6529   }
6530   case Intrinsic::ssub_sat: {
6531     SDValue Op1 = getValue(I.getArgOperand(0));
6532     SDValue Op2 = getValue(I.getArgOperand(1));
6533     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6534     return;
6535   }
6536   case Intrinsic::usub_sat: {
6537     SDValue Op1 = getValue(I.getArgOperand(0));
6538     SDValue Op2 = getValue(I.getArgOperand(1));
6539     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6540     return;
6541   }
6542   case Intrinsic::sshl_sat: {
6543     SDValue Op1 = getValue(I.getArgOperand(0));
6544     SDValue Op2 = getValue(I.getArgOperand(1));
6545     setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6546     return;
6547   }
6548   case Intrinsic::ushl_sat: {
6549     SDValue Op1 = getValue(I.getArgOperand(0));
6550     SDValue Op2 = getValue(I.getArgOperand(1));
6551     setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6552     return;
6553   }
6554   case Intrinsic::smul_fix:
6555   case Intrinsic::umul_fix:
6556   case Intrinsic::smul_fix_sat:
6557   case Intrinsic::umul_fix_sat: {
6558     SDValue Op1 = getValue(I.getArgOperand(0));
6559     SDValue Op2 = getValue(I.getArgOperand(1));
6560     SDValue Op3 = getValue(I.getArgOperand(2));
6561     setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6562                              Op1.getValueType(), Op1, Op2, Op3));
6563     return;
6564   }
6565   case Intrinsic::sdiv_fix:
6566   case Intrinsic::udiv_fix:
6567   case Intrinsic::sdiv_fix_sat:
6568   case Intrinsic::udiv_fix_sat: {
6569     SDValue Op1 = getValue(I.getArgOperand(0));
6570     SDValue Op2 = getValue(I.getArgOperand(1));
6571     SDValue Op3 = getValue(I.getArgOperand(2));
6572     setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6573                               Op1, Op2, Op3, DAG, TLI));
6574     return;
6575   }
6576   case Intrinsic::smax: {
6577     SDValue Op1 = getValue(I.getArgOperand(0));
6578     SDValue Op2 = getValue(I.getArgOperand(1));
6579     setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
6580     return;
6581   }
6582   case Intrinsic::smin: {
6583     SDValue Op1 = getValue(I.getArgOperand(0));
6584     SDValue Op2 = getValue(I.getArgOperand(1));
6585     setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
6586     return;
6587   }
6588   case Intrinsic::umax: {
6589     SDValue Op1 = getValue(I.getArgOperand(0));
6590     SDValue Op2 = getValue(I.getArgOperand(1));
6591     setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
6592     return;
6593   }
6594   case Intrinsic::umin: {
6595     SDValue Op1 = getValue(I.getArgOperand(0));
6596     SDValue Op2 = getValue(I.getArgOperand(1));
6597     setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
6598     return;
6599   }
6600   case Intrinsic::abs: {
6601     // TODO: Preserve "int min is poison" arg in SDAG?
6602     SDValue Op1 = getValue(I.getArgOperand(0));
6603     setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
6604     return;
6605   }
6606   case Intrinsic::stacksave: {
6607     SDValue Op = getRoot();
6608     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6609     Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
6610     setValue(&I, Res);
6611     DAG.setRoot(Res.getValue(1));
6612     return;
6613   }
6614   case Intrinsic::stackrestore:
6615     Res = getValue(I.getArgOperand(0));
6616     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
6617     return;
6618   case Intrinsic::get_dynamic_area_offset: {
6619     SDValue Op = getRoot();
6620     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6621     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6622     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
6623     // target.
6624     if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
6625       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
6626                          " intrinsic!");
6627     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
6628                       Op);
6629     DAG.setRoot(Op);
6630     setValue(&I, Res);
6631     return;
6632   }
6633   case Intrinsic::stackguard: {
6634     MachineFunction &MF = DAG.getMachineFunction();
6635     const Module &M = *MF.getFunction().getParent();
6636     SDValue Chain = getRoot();
6637     if (TLI.useLoadStackGuardNode()) {
6638       Res = getLoadStackGuard(DAG, sdl, Chain);
6639     } else {
6640       EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6641       const Value *Global = TLI.getSDagStackGuard(M);
6642       Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
6643       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
6644                         MachinePointerInfo(Global, 0), Align,
6645                         MachineMemOperand::MOVolatile);
6646     }
6647     if (TLI.useStackGuardXorFP())
6648       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
6649     DAG.setRoot(Chain);
6650     setValue(&I, Res);
6651     return;
6652   }
6653   case Intrinsic::stackprotector: {
6654     // Emit code into the DAG to store the stack guard onto the stack.
6655     MachineFunction &MF = DAG.getMachineFunction();
6656     MachineFrameInfo &MFI = MF.getFrameInfo();
6657     SDValue Src, Chain = getRoot();
6658 
6659     if (TLI.useLoadStackGuardNode())
6660       Src = getLoadStackGuard(DAG, sdl, Chain);
6661     else
6662       Src = getValue(I.getArgOperand(0));   // The guard's value.
6663 
6664     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
6665 
6666     int FI = FuncInfo.StaticAllocaMap[Slot];
6667     MFI.setStackProtectorIndex(FI);
6668     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6669 
6670     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
6671 
6672     // Store the stack protector onto the stack.
6673     Res = DAG.getStore(
6674         Chain, sdl, Src, FIN,
6675         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
6676         MaybeAlign(), MachineMemOperand::MOVolatile);
6677     setValue(&I, Res);
6678     DAG.setRoot(Res);
6679     return;
6680   }
6681   case Intrinsic::objectsize:
6682     llvm_unreachable("llvm.objectsize.* should have been lowered already");
6683 
6684   case Intrinsic::is_constant:
6685     llvm_unreachable("llvm.is.constant.* should have been lowered already");
6686 
6687   case Intrinsic::annotation:
6688   case Intrinsic::ptr_annotation:
6689   case Intrinsic::launder_invariant_group:
6690   case Intrinsic::strip_invariant_group:
6691     // Drop the intrinsic, but forward the value
6692     setValue(&I, getValue(I.getOperand(0)));
6693     return;
6694 
6695   case Intrinsic::assume:
6696   case Intrinsic::experimental_noalias_scope_decl:
6697   case Intrinsic::var_annotation:
6698   case Intrinsic::sideeffect:
6699     // Discard annotate attributes, noalias scope declarations, assumptions, and
6700     // artificial side-effects.
6701     return;
6702 
6703   case Intrinsic::codeview_annotation: {
6704     // Emit a label associated with this metadata.
6705     MachineFunction &MF = DAG.getMachineFunction();
6706     MCSymbol *Label =
6707         MF.getMMI().getContext().createTempSymbol("annotation", true);
6708     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
6709     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
6710     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
6711     DAG.setRoot(Res);
6712     return;
6713   }
6714 
6715   case Intrinsic::init_trampoline: {
6716     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
6717 
6718     SDValue Ops[6];
6719     Ops[0] = getRoot();
6720     Ops[1] = getValue(I.getArgOperand(0));
6721     Ops[2] = getValue(I.getArgOperand(1));
6722     Ops[3] = getValue(I.getArgOperand(2));
6723     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
6724     Ops[5] = DAG.getSrcValue(F);
6725 
6726     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
6727 
6728     DAG.setRoot(Res);
6729     return;
6730   }
6731   case Intrinsic::adjust_trampoline:
6732     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
6733                              TLI.getPointerTy(DAG.getDataLayout()),
6734                              getValue(I.getArgOperand(0))));
6735     return;
6736   case Intrinsic::gcroot: {
6737     assert(DAG.getMachineFunction().getFunction().hasGC() &&
6738            "only valid in functions with gc specified, enforced by Verifier");
6739     assert(GFI && "implied by previous");
6740     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
6741     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
6742 
6743     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
6744     GFI->addStackRoot(FI->getIndex(), TypeMap);
6745     return;
6746   }
6747   case Intrinsic::gcread:
6748   case Intrinsic::gcwrite:
6749     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
6750   case Intrinsic::flt_rounds:
6751     Res = DAG.getNode(ISD::FLT_ROUNDS_, sdl, {MVT::i32, MVT::Other}, getRoot());
6752     setValue(&I, Res);
6753     DAG.setRoot(Res.getValue(1));
6754     return;
6755 
6756   case Intrinsic::expect:
6757     // Just replace __builtin_expect(exp, c) with EXP.
6758     setValue(&I, getValue(I.getArgOperand(0)));
6759     return;
6760 
6761   case Intrinsic::ubsantrap:
6762   case Intrinsic::debugtrap:
6763   case Intrinsic::trap: {
6764     StringRef TrapFuncName =
6765         I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
6766     if (TrapFuncName.empty()) {
6767       switch (Intrinsic) {
6768       case Intrinsic::trap:
6769         DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
6770         break;
6771       case Intrinsic::debugtrap:
6772         DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
6773         break;
6774       case Intrinsic::ubsantrap:
6775         DAG.setRoot(DAG.getNode(
6776             ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
6777             DAG.getTargetConstant(
6778                 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
6779                 MVT::i32)));
6780         break;
6781       default: llvm_unreachable("unknown trap intrinsic");
6782       }
6783       return;
6784     }
6785     TargetLowering::ArgListTy Args;
6786     if (Intrinsic == Intrinsic::ubsantrap) {
6787       Args.push_back(TargetLoweringBase::ArgListEntry());
6788       Args[0].Val = I.getArgOperand(0);
6789       Args[0].Node = getValue(Args[0].Val);
6790       Args[0].Ty = Args[0].Val->getType();
6791     }
6792 
6793     TargetLowering::CallLoweringInfo CLI(DAG);
6794     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6795         CallingConv::C, I.getType(),
6796         DAG.getExternalSymbol(TrapFuncName.data(),
6797                               TLI.getPointerTy(DAG.getDataLayout())),
6798         std::move(Args));
6799 
6800     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6801     DAG.setRoot(Result.second);
6802     return;
6803   }
6804 
6805   case Intrinsic::uadd_with_overflow:
6806   case Intrinsic::sadd_with_overflow:
6807   case Intrinsic::usub_with_overflow:
6808   case Intrinsic::ssub_with_overflow:
6809   case Intrinsic::umul_with_overflow:
6810   case Intrinsic::smul_with_overflow: {
6811     ISD::NodeType Op;
6812     switch (Intrinsic) {
6813     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6814     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6815     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6816     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6817     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6818     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6819     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6820     }
6821     SDValue Op1 = getValue(I.getArgOperand(0));
6822     SDValue Op2 = getValue(I.getArgOperand(1));
6823 
6824     EVT ResultVT = Op1.getValueType();
6825     EVT OverflowVT = MVT::i1;
6826     if (ResultVT.isVector())
6827       OverflowVT = EVT::getVectorVT(
6828           *Context, OverflowVT, ResultVT.getVectorElementCount());
6829 
6830     SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
6831     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6832     return;
6833   }
6834   case Intrinsic::prefetch: {
6835     SDValue Ops[5];
6836     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6837     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6838     Ops[0] = DAG.getRoot();
6839     Ops[1] = getValue(I.getArgOperand(0));
6840     Ops[2] = getValue(I.getArgOperand(1));
6841     Ops[3] = getValue(I.getArgOperand(2));
6842     Ops[4] = getValue(I.getArgOperand(3));
6843     SDValue Result = DAG.getMemIntrinsicNode(
6844         ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
6845         EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
6846         /* align */ None, Flags);
6847 
6848     // Chain the prefetch in parallell with any pending loads, to stay out of
6849     // the way of later optimizations.
6850     PendingLoads.push_back(Result);
6851     Result = getRoot();
6852     DAG.setRoot(Result);
6853     return;
6854   }
6855   case Intrinsic::lifetime_start:
6856   case Intrinsic::lifetime_end: {
6857     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6858     // Stack coloring is not enabled in O0, discard region information.
6859     if (TM.getOptLevel() == CodeGenOpt::None)
6860       return;
6861 
6862     const int64_t ObjectSize =
6863         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
6864     Value *const ObjectPtr = I.getArgOperand(1);
6865     SmallVector<const Value *, 4> Allocas;
6866     getUnderlyingObjects(ObjectPtr, Allocas);
6867 
6868     for (const Value *Alloca : Allocas) {
6869       const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
6870 
6871       // Could not find an Alloca.
6872       if (!LifetimeObject)
6873         continue;
6874 
6875       // First check that the Alloca is static, otherwise it won't have a
6876       // valid frame index.
6877       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6878       if (SI == FuncInfo.StaticAllocaMap.end())
6879         return;
6880 
6881       const int FrameIndex = SI->second;
6882       int64_t Offset;
6883       if (GetPointerBaseWithConstantOffset(
6884               ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
6885         Offset = -1; // Cannot determine offset from alloca to lifetime object.
6886       Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
6887                                 Offset);
6888       DAG.setRoot(Res);
6889     }
6890     return;
6891   }
6892   case Intrinsic::pseudoprobe: {
6893     auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
6894     auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6895     auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
6896     Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
6897     DAG.setRoot(Res);
6898     return;
6899   }
6900   case Intrinsic::invariant_start:
6901     // Discard region information.
6902     setValue(&I,
6903              DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
6904     return;
6905   case Intrinsic::invariant_end:
6906     // Discard region information.
6907     return;
6908   case Intrinsic::clear_cache:
6909     /// FunctionName may be null.
6910     if (const char *FunctionName = TLI.getClearCacheBuiltinName())
6911       lowerCallToExternalSymbol(I, FunctionName);
6912     return;
6913   case Intrinsic::donothing:
6914   case Intrinsic::seh_try_begin:
6915   case Intrinsic::seh_scope_begin:
6916   case Intrinsic::seh_try_end:
6917   case Intrinsic::seh_scope_end:
6918     // ignore
6919     return;
6920   case Intrinsic::experimental_stackmap:
6921     visitStackmap(I);
6922     return;
6923   case Intrinsic::experimental_patchpoint_void:
6924   case Intrinsic::experimental_patchpoint_i64:
6925     visitPatchpoint(I);
6926     return;
6927   case Intrinsic::experimental_gc_statepoint:
6928     LowerStatepoint(cast<GCStatepointInst>(I));
6929     return;
6930   case Intrinsic::experimental_gc_result:
6931     visitGCResult(cast<GCResultInst>(I));
6932     return;
6933   case Intrinsic::experimental_gc_relocate:
6934     visitGCRelocate(cast<GCRelocateInst>(I));
6935     return;
6936   case Intrinsic::instrprof_cover:
6937     llvm_unreachable("instrprof failed to lower a cover");
6938   case Intrinsic::instrprof_increment:
6939     llvm_unreachable("instrprof failed to lower an increment");
6940   case Intrinsic::instrprof_value_profile:
6941     llvm_unreachable("instrprof failed to lower a value profiling call");
6942   case Intrinsic::localescape: {
6943     MachineFunction &MF = DAG.getMachineFunction();
6944     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6945 
6946     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6947     // is the same on all targets.
6948     for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
6949       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6950       if (isa<ConstantPointerNull>(Arg))
6951         continue; // Skip null pointers. They represent a hole in index space.
6952       AllocaInst *Slot = cast<AllocaInst>(Arg);
6953       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
6954              "can only escape static allocas");
6955       int FI = FuncInfo.StaticAllocaMap[Slot];
6956       MCSymbol *FrameAllocSym =
6957           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6958               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6959       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6960               TII->get(TargetOpcode::LOCAL_ESCAPE))
6961           .addSym(FrameAllocSym)
6962           .addFrameIndex(FI);
6963     }
6964 
6965     return;
6966   }
6967 
6968   case Intrinsic::localrecover: {
6969     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6970     MachineFunction &MF = DAG.getMachineFunction();
6971 
6972     // Get the symbol that defines the frame offset.
6973     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6974     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6975     unsigned IdxVal =
6976         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6977     MCSymbol *FrameAllocSym =
6978         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6979             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6980 
6981     Value *FP = I.getArgOperand(1);
6982     SDValue FPVal = getValue(FP);
6983     EVT PtrVT = FPVal.getValueType();
6984 
6985     // Create a MCSymbol for the label to avoid any target lowering
6986     // that would make this PC relative.
6987     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6988     SDValue OffsetVal =
6989         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6990 
6991     // Add the offset to the FP.
6992     SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
6993     setValue(&I, Add);
6994 
6995     return;
6996   }
6997 
6998   case Intrinsic::eh_exceptionpointer:
6999   case Intrinsic::eh_exceptioncode: {
7000     // Get the exception pointer vreg, copy from it, and resize it to fit.
7001     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7002     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7003     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7004     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7005     SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7006     if (Intrinsic == Intrinsic::eh_exceptioncode)
7007       N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7008     setValue(&I, N);
7009     return;
7010   }
7011   case Intrinsic::xray_customevent: {
7012     // Here we want to make sure that the intrinsic behaves as if it has a
7013     // specific calling convention, and only for x86_64.
7014     // FIXME: Support other platforms later.
7015     const auto &Triple = DAG.getTarget().getTargetTriple();
7016     if (Triple.getArch() != Triple::x86_64)
7017       return;
7018 
7019     SmallVector<SDValue, 8> Ops;
7020 
7021     // We want to say that we always want the arguments in registers.
7022     SDValue LogEntryVal = getValue(I.getArgOperand(0));
7023     SDValue StrSizeVal = getValue(I.getArgOperand(1));
7024     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7025     SDValue Chain = getRoot();
7026     Ops.push_back(LogEntryVal);
7027     Ops.push_back(StrSizeVal);
7028     Ops.push_back(Chain);
7029 
7030     // We need to enforce the calling convention for the callsite, so that
7031     // argument ordering is enforced correctly, and that register allocation can
7032     // see that some registers may be assumed clobbered and have to preserve
7033     // them across calls to the intrinsic.
7034     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7035                                            sdl, NodeTys, Ops);
7036     SDValue patchableNode = SDValue(MN, 0);
7037     DAG.setRoot(patchableNode);
7038     setValue(&I, patchableNode);
7039     return;
7040   }
7041   case Intrinsic::xray_typedevent: {
7042     // Here we want to make sure that the intrinsic behaves as if it has a
7043     // specific calling convention, and only for x86_64.
7044     // FIXME: Support other platforms later.
7045     const auto &Triple = DAG.getTarget().getTargetTriple();
7046     if (Triple.getArch() != Triple::x86_64)
7047       return;
7048 
7049     SmallVector<SDValue, 8> Ops;
7050 
7051     // We want to say that we always want the arguments in registers.
7052     // It's unclear to me how manipulating the selection DAG here forces callers
7053     // to provide arguments in registers instead of on the stack.
7054     SDValue LogTypeId = getValue(I.getArgOperand(0));
7055     SDValue LogEntryVal = getValue(I.getArgOperand(1));
7056     SDValue StrSizeVal = getValue(I.getArgOperand(2));
7057     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7058     SDValue Chain = getRoot();
7059     Ops.push_back(LogTypeId);
7060     Ops.push_back(LogEntryVal);
7061     Ops.push_back(StrSizeVal);
7062     Ops.push_back(Chain);
7063 
7064     // We need to enforce the calling convention for the callsite, so that
7065     // argument ordering is enforced correctly, and that register allocation can
7066     // see that some registers may be assumed clobbered and have to preserve
7067     // them across calls to the intrinsic.
7068     MachineSDNode *MN = DAG.getMachineNode(
7069         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7070     SDValue patchableNode = SDValue(MN, 0);
7071     DAG.setRoot(patchableNode);
7072     setValue(&I, patchableNode);
7073     return;
7074   }
7075   case Intrinsic::experimental_deoptimize:
7076     LowerDeoptimizeCall(&I);
7077     return;
7078   case Intrinsic::experimental_stepvector:
7079     visitStepVector(I);
7080     return;
7081   case Intrinsic::vector_reduce_fadd:
7082   case Intrinsic::vector_reduce_fmul:
7083   case Intrinsic::vector_reduce_add:
7084   case Intrinsic::vector_reduce_mul:
7085   case Intrinsic::vector_reduce_and:
7086   case Intrinsic::vector_reduce_or:
7087   case Intrinsic::vector_reduce_xor:
7088   case Intrinsic::vector_reduce_smax:
7089   case Intrinsic::vector_reduce_smin:
7090   case Intrinsic::vector_reduce_umax:
7091   case Intrinsic::vector_reduce_umin:
7092   case Intrinsic::vector_reduce_fmax:
7093   case Intrinsic::vector_reduce_fmin:
7094     visitVectorReduce(I, Intrinsic);
7095     return;
7096 
7097   case Intrinsic::icall_branch_funnel: {
7098     SmallVector<SDValue, 16> Ops;
7099     Ops.push_back(getValue(I.getArgOperand(0)));
7100 
7101     int64_t Offset;
7102     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7103         I.getArgOperand(1), Offset, DAG.getDataLayout()));
7104     if (!Base)
7105       report_fatal_error(
7106           "llvm.icall.branch.funnel operand must be a GlobalValue");
7107     Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7108 
7109     struct BranchFunnelTarget {
7110       int64_t Offset;
7111       SDValue Target;
7112     };
7113     SmallVector<BranchFunnelTarget, 8> Targets;
7114 
7115     for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7116       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7117           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7118       if (ElemBase != Base)
7119         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7120                            "to the same GlobalValue");
7121 
7122       SDValue Val = getValue(I.getArgOperand(Op + 1));
7123       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7124       if (!GA)
7125         report_fatal_error(
7126             "llvm.icall.branch.funnel operand must be a GlobalValue");
7127       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
7128                                      GA->getGlobal(), sdl, Val.getValueType(),
7129                                      GA->getOffset())});
7130     }
7131     llvm::sort(Targets,
7132                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7133                  return T1.Offset < T2.Offset;
7134                });
7135 
7136     for (auto &T : Targets) {
7137       Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7138       Ops.push_back(T.Target);
7139     }
7140 
7141     Ops.push_back(DAG.getRoot()); // Chain
7142     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7143                                  MVT::Other, Ops),
7144               0);
7145     DAG.setRoot(N);
7146     setValue(&I, N);
7147     HasTailCall = true;
7148     return;
7149   }
7150 
7151   case Intrinsic::wasm_landingpad_index:
7152     // Information this intrinsic contained has been transferred to
7153     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7154     // delete it now.
7155     return;
7156 
7157   case Intrinsic::aarch64_settag:
7158   case Intrinsic::aarch64_settag_zero: {
7159     const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7160     bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7161     SDValue Val = TSI.EmitTargetCodeForSetTag(
7162         DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7163         getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7164         ZeroMemory);
7165     DAG.setRoot(Val);
7166     setValue(&I, Val);
7167     return;
7168   }
7169   case Intrinsic::ptrmask: {
7170     SDValue Ptr = getValue(I.getOperand(0));
7171     SDValue Const = getValue(I.getOperand(1));
7172 
7173     EVT PtrVT = Ptr.getValueType();
7174     setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr,
7175                              DAG.getZExtOrTrunc(Const, sdl, PtrVT)));
7176     return;
7177   }
7178   case Intrinsic::get_active_lane_mask: {
7179     EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7180     SDValue Index = getValue(I.getOperand(0));
7181     EVT ElementVT = Index.getValueType();
7182 
7183     if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7184       visitTargetIntrinsic(I, Intrinsic);
7185       return;
7186     }
7187 
7188     SDValue TripCount = getValue(I.getOperand(1));
7189     auto VecTy = CCVT.changeVectorElementType(ElementVT);
7190 
7191     SDValue VectorIndex, VectorTripCount;
7192     if (VecTy.isScalableVector()) {
7193       VectorIndex = DAG.getSplatVector(VecTy, sdl, Index);
7194       VectorTripCount = DAG.getSplatVector(VecTy, sdl, TripCount);
7195     } else {
7196       VectorIndex = DAG.getSplatBuildVector(VecTy, sdl, Index);
7197       VectorTripCount = DAG.getSplatBuildVector(VecTy, sdl, TripCount);
7198     }
7199     SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7200     SDValue VectorInduction = DAG.getNode(
7201         ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7202     SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7203                                  VectorTripCount, ISD::CondCode::SETULT);
7204     setValue(&I, SetCC);
7205     return;
7206   }
7207   case Intrinsic::vector_insert: {
7208     SDValue Vec = getValue(I.getOperand(0));
7209     SDValue SubVec = getValue(I.getOperand(1));
7210     SDValue Index = getValue(I.getOperand(2));
7211 
7212     // The intrinsic's index type is i64, but the SDNode requires an index type
7213     // suitable for the target. Convert the index as required.
7214     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7215     if (Index.getValueType() != VectorIdxTy)
7216       Index = DAG.getVectorIdxConstant(
7217           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7218 
7219     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7220     setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7221                              Index));
7222     return;
7223   }
7224   case Intrinsic::vector_extract: {
7225     SDValue Vec = getValue(I.getOperand(0));
7226     SDValue Index = getValue(I.getOperand(1));
7227     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7228 
7229     // The intrinsic's index type is i64, but the SDNode requires an index type
7230     // suitable for the target. Convert the index as required.
7231     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7232     if (Index.getValueType() != VectorIdxTy)
7233       Index = DAG.getVectorIdxConstant(
7234           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7235 
7236     setValue(&I,
7237              DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7238     return;
7239   }
7240   case Intrinsic::experimental_vector_reverse:
7241     visitVectorReverse(I);
7242     return;
7243   case Intrinsic::experimental_vector_splice:
7244     visitVectorSplice(I);
7245     return;
7246   }
7247 }
7248 
7249 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7250     const ConstrainedFPIntrinsic &FPI) {
7251   SDLoc sdl = getCurSDLoc();
7252 
7253   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7254   SmallVector<EVT, 4> ValueVTs;
7255   ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
7256   ValueVTs.push_back(MVT::Other); // Out chain
7257 
7258   // We do not need to serialize constrained FP intrinsics against
7259   // each other or against (nonvolatile) loads, so they can be
7260   // chained like loads.
7261   SDValue Chain = DAG.getRoot();
7262   SmallVector<SDValue, 4> Opers;
7263   Opers.push_back(Chain);
7264   if (FPI.isUnaryOp()) {
7265     Opers.push_back(getValue(FPI.getArgOperand(0)));
7266   } else if (FPI.isTernaryOp()) {
7267     Opers.push_back(getValue(FPI.getArgOperand(0)));
7268     Opers.push_back(getValue(FPI.getArgOperand(1)));
7269     Opers.push_back(getValue(FPI.getArgOperand(2)));
7270   } else {
7271     Opers.push_back(getValue(FPI.getArgOperand(0)));
7272     Opers.push_back(getValue(FPI.getArgOperand(1)));
7273   }
7274 
7275   auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7276     assert(Result.getNode()->getNumValues() == 2);
7277 
7278     // Push node to the appropriate list so that future instructions can be
7279     // chained up correctly.
7280     SDValue OutChain = Result.getValue(1);
7281     switch (EB) {
7282     case fp::ExceptionBehavior::ebIgnore:
7283       // The only reason why ebIgnore nodes still need to be chained is that
7284       // they might depend on the current rounding mode, and therefore must
7285       // not be moved across instruction that may change that mode.
7286       LLVM_FALLTHROUGH;
7287     case fp::ExceptionBehavior::ebMayTrap:
7288       // These must not be moved across calls or instructions that may change
7289       // floating-point exception masks.
7290       PendingConstrainedFP.push_back(OutChain);
7291       break;
7292     case fp::ExceptionBehavior::ebStrict:
7293       // These must not be moved across calls or instructions that may change
7294       // floating-point exception masks or read floating-point exception flags.
7295       // In addition, they cannot be optimized out even if unused.
7296       PendingConstrainedFPStrict.push_back(OutChain);
7297       break;
7298     }
7299   };
7300 
7301   SDVTList VTs = DAG.getVTList(ValueVTs);
7302   fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
7303 
7304   SDNodeFlags Flags;
7305   if (EB == fp::ExceptionBehavior::ebIgnore)
7306     Flags.setNoFPExcept(true);
7307 
7308   if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7309     Flags.copyFMF(*FPOp);
7310 
7311   unsigned Opcode;
7312   switch (FPI.getIntrinsicID()) {
7313   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
7314 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
7315   case Intrinsic::INTRINSIC:                                                   \
7316     Opcode = ISD::STRICT_##DAGN;                                               \
7317     break;
7318 #include "llvm/IR/ConstrainedOps.def"
7319   case Intrinsic::experimental_constrained_fmuladd: {
7320     Opcode = ISD::STRICT_FMA;
7321     // Break fmuladd into fmul and fadd.
7322     if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
7323         !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(),
7324                                         ValueVTs[0])) {
7325       Opers.pop_back();
7326       SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
7327       pushOutChain(Mul, EB);
7328       Opcode = ISD::STRICT_FADD;
7329       Opers.clear();
7330       Opers.push_back(Mul.getValue(1));
7331       Opers.push_back(Mul.getValue(0));
7332       Opers.push_back(getValue(FPI.getArgOperand(2)));
7333     }
7334     break;
7335   }
7336   }
7337 
7338   // A few strict DAG nodes carry additional operands that are not
7339   // set up by the default code above.
7340   switch (Opcode) {
7341   default: break;
7342   case ISD::STRICT_FP_ROUND:
7343     Opers.push_back(
7344         DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
7345     break;
7346   case ISD::STRICT_FSETCC:
7347   case ISD::STRICT_FSETCCS: {
7348     auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
7349     ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
7350     if (TM.Options.NoNaNsFPMath)
7351       Condition = getFCmpCodeWithoutNaN(Condition);
7352     Opers.push_back(DAG.getCondCode(Condition));
7353     break;
7354   }
7355   }
7356 
7357   SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
7358   pushOutChain(Result, EB);
7359 
7360   SDValue FPResult = Result.getValue(0);
7361   setValue(&FPI, FPResult);
7362 }
7363 
7364 static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
7365   Optional<unsigned> ResOPC;
7366   switch (VPIntrin.getIntrinsicID()) {
7367 #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)                                    \
7368   case Intrinsic::VPID:                                                        \
7369     ResOPC = ISD::VPSD;                                                        \
7370     break;
7371 #include "llvm/IR/VPIntrinsics.def"
7372   }
7373 
7374   if (!ResOPC)
7375     llvm_unreachable(
7376         "Inconsistency: no SDNode available for this VPIntrinsic!");
7377 
7378   if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
7379       *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
7380     if (VPIntrin.getFastMathFlags().allowReassoc())
7381       return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
7382                                                 : ISD::VP_REDUCE_FMUL;
7383   }
7384 
7385   return *ResOPC;
7386 }
7387 
7388 void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
7389                                             SmallVector<SDValue, 7> &OpValues,
7390                                             bool IsGather) {
7391   SDLoc DL = getCurSDLoc();
7392   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7393   Value *PtrOperand = VPIntrin.getArgOperand(0);
7394   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7395   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7396   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7397   SDValue LD;
7398   bool AddToChain = true;
7399   if (!IsGather) {
7400     // Do not serialize variable-length loads of constant memory with
7401     // anything.
7402     if (!Alignment)
7403       Alignment = DAG.getEVTAlign(VT);
7404     MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7405     AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7406     SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7407     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7408         MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7409         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7410     LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
7411                        MMO, false /*IsExpanding */);
7412   } else {
7413     if (!Alignment)
7414       Alignment = DAG.getEVTAlign(VT.getScalarType());
7415     unsigned AS =
7416         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7417     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7418         MachinePointerInfo(AS), MachineMemOperand::MOLoad,
7419         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7420     SDValue Base, Index, Scale;
7421     ISD::MemIndexType IndexType;
7422     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7423                                       this, VPIntrin.getParent(),
7424                                       VT.getScalarStoreSize());
7425     if (!UniformBase) {
7426       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7427       Index = getValue(PtrOperand);
7428       IndexType = ISD::SIGNED_SCALED;
7429       Scale =
7430           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7431     }
7432     EVT IdxVT = Index.getValueType();
7433     EVT EltTy = IdxVT.getVectorElementType();
7434     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7435       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7436       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7437     }
7438     LD = DAG.getGatherVP(
7439         DAG.getVTList(VT, MVT::Other), VT, DL,
7440         {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
7441         IndexType);
7442   }
7443   if (AddToChain)
7444     PendingLoads.push_back(LD.getValue(1));
7445   setValue(&VPIntrin, LD);
7446 }
7447 
7448 void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
7449                                               SmallVector<SDValue, 7> &OpValues,
7450                                               bool IsScatter) {
7451   SDLoc DL = getCurSDLoc();
7452   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7453   Value *PtrOperand = VPIntrin.getArgOperand(1);
7454   EVT VT = OpValues[0].getValueType();
7455   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7456   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7457   SDValue ST;
7458   if (!IsScatter) {
7459     if (!Alignment)
7460       Alignment = DAG.getEVTAlign(VT);
7461     SDValue Ptr = OpValues[1];
7462     SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
7463     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7464         MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7465         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7466     ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
7467                         OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
7468                         /* IsTruncating */ false, /*IsCompressing*/ false);
7469   } else {
7470     if (!Alignment)
7471       Alignment = DAG.getEVTAlign(VT.getScalarType());
7472     unsigned AS =
7473         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7474     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7475         MachinePointerInfo(AS), MachineMemOperand::MOStore,
7476         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7477     SDValue Base, Index, Scale;
7478     ISD::MemIndexType IndexType;
7479     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7480                                       this, VPIntrin.getParent(),
7481                                       VT.getScalarStoreSize());
7482     if (!UniformBase) {
7483       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7484       Index = getValue(PtrOperand);
7485       IndexType = ISD::SIGNED_SCALED;
7486       Scale =
7487           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7488     }
7489     EVT IdxVT = Index.getValueType();
7490     EVT EltTy = IdxVT.getVectorElementType();
7491     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7492       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7493       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7494     }
7495     ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
7496                           {getMemoryRoot(), OpValues[0], Base, Index, Scale,
7497                            OpValues[2], OpValues[3]},
7498                           MMO, IndexType);
7499   }
7500   DAG.setRoot(ST);
7501   setValue(&VPIntrin, ST);
7502 }
7503 
7504 void SelectionDAGBuilder::visitVPStridedLoad(
7505     const VPIntrinsic &VPIntrin, EVT VT, SmallVectorImpl<SDValue> &OpValues) {
7506   SDLoc DL = getCurSDLoc();
7507   Value *PtrOperand = VPIntrin.getArgOperand(0);
7508   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7509   if (!Alignment)
7510     Alignment = DAG.getEVTAlign(VT.getScalarType());
7511   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7512   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7513   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7514   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7515   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7516   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7517       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7518       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7519 
7520   SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
7521                                     OpValues[2], OpValues[3], MMO,
7522                                     false /*IsExpanding*/);
7523 
7524   if (AddToChain)
7525     PendingLoads.push_back(LD.getValue(1));
7526   setValue(&VPIntrin, LD);
7527 }
7528 
7529 void SelectionDAGBuilder::visitVPStridedStore(
7530     const VPIntrinsic &VPIntrin, SmallVectorImpl<SDValue> &OpValues) {
7531   SDLoc DL = getCurSDLoc();
7532   Value *PtrOperand = VPIntrin.getArgOperand(1);
7533   EVT VT = OpValues[0].getValueType();
7534   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7535   if (!Alignment)
7536     Alignment = DAG.getEVTAlign(VT.getScalarType());
7537   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7538   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7539       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7540       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7541 
7542   SDValue ST = DAG.getStridedStoreVP(
7543       getMemoryRoot(), DL, OpValues[0], OpValues[1],
7544       DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
7545       OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
7546       /*IsCompressing*/ false);
7547 
7548   DAG.setRoot(ST);
7549   setValue(&VPIntrin, ST);
7550 }
7551 
7552 void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
7553   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7554   SDLoc DL = getCurSDLoc();
7555 
7556   ISD::CondCode Condition;
7557   CmpInst::Predicate CondCode = VPIntrin.getPredicate();
7558   bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
7559   if (IsFP) {
7560     // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
7561     // flags, but calls that don't return floating-point types can't be
7562     // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
7563     Condition = getFCmpCondCode(CondCode);
7564     if (TM.Options.NoNaNsFPMath)
7565       Condition = getFCmpCodeWithoutNaN(Condition);
7566   } else {
7567     Condition = getICmpCondCode(CondCode);
7568   }
7569 
7570   SDValue Op1 = getValue(VPIntrin.getOperand(0));
7571   SDValue Op2 = getValue(VPIntrin.getOperand(1));
7572   // #2 is the condition code
7573   SDValue MaskOp = getValue(VPIntrin.getOperand(3));
7574   SDValue EVL = getValue(VPIntrin.getOperand(4));
7575   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7576   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7577          "Unexpected target EVL type");
7578   EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
7579 
7580   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7581                                                         VPIntrin.getType());
7582   setValue(&VPIntrin,
7583            DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
7584 }
7585 
7586 void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
7587     const VPIntrinsic &VPIntrin) {
7588   SDLoc DL = getCurSDLoc();
7589   unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
7590 
7591   auto IID = VPIntrin.getIntrinsicID();
7592 
7593   if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
7594     return visitVPCmp(*CmpI);
7595 
7596   SmallVector<EVT, 4> ValueVTs;
7597   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7598   ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
7599   SDVTList VTs = DAG.getVTList(ValueVTs);
7600 
7601   auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
7602 
7603   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7604   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7605          "Unexpected target EVL type");
7606 
7607   // Request operands.
7608   SmallVector<SDValue, 7> OpValues;
7609   for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
7610     auto Op = getValue(VPIntrin.getArgOperand(I));
7611     if (I == EVLParamPos)
7612       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
7613     OpValues.push_back(Op);
7614   }
7615 
7616   switch (Opcode) {
7617   default: {
7618     SDNodeFlags SDFlags;
7619     if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
7620       SDFlags.copyFMF(*FPMO);
7621     SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
7622     setValue(&VPIntrin, Result);
7623     break;
7624   }
7625   case ISD::VP_LOAD:
7626   case ISD::VP_GATHER:
7627     visitVPLoadGather(VPIntrin, ValueVTs[0], OpValues,
7628                       Opcode == ISD::VP_GATHER);
7629     break;
7630   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7631     visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
7632     break;
7633   case ISD::VP_STORE:
7634   case ISD::VP_SCATTER:
7635     visitVPStoreScatter(VPIntrin, OpValues, Opcode == ISD::VP_SCATTER);
7636     break;
7637   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7638     visitVPStridedStore(VPIntrin, OpValues);
7639     break;
7640   }
7641 }
7642 
7643 SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
7644                                           const BasicBlock *EHPadBB,
7645                                           MCSymbol *&BeginLabel) {
7646   MachineFunction &MF = DAG.getMachineFunction();
7647   MachineModuleInfo &MMI = MF.getMMI();
7648 
7649   // Insert a label before the invoke call to mark the try range.  This can be
7650   // used to detect deletion of the invoke via the MachineModuleInfo.
7651   BeginLabel = MMI.getContext().createTempSymbol();
7652 
7653   // For SjLj, keep track of which landing pads go with which invokes
7654   // so as to maintain the ordering of pads in the LSDA.
7655   unsigned CallSiteIndex = MMI.getCurrentCallSite();
7656   if (CallSiteIndex) {
7657     MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
7658     LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
7659 
7660     // Now that the call site is handled, stop tracking it.
7661     MMI.setCurrentCallSite(0);
7662   }
7663 
7664   return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
7665 }
7666 
7667 SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
7668                                         const BasicBlock *EHPadBB,
7669                                         MCSymbol *BeginLabel) {
7670   assert(BeginLabel && "BeginLabel should've been set");
7671 
7672   MachineFunction &MF = DAG.getMachineFunction();
7673   MachineModuleInfo &MMI = MF.getMMI();
7674 
7675   // Insert a label at the end of the invoke call to mark the try range.  This
7676   // can be used to detect deletion of the invoke via the MachineModuleInfo.
7677   MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
7678   Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
7679 
7680   // Inform MachineModuleInfo of range.
7681   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
7682   // There is a platform (e.g. wasm) that uses funclet style IR but does not
7683   // actually use outlined funclets and their LSDA info style.
7684   if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
7685     assert(II && "II should've been set");
7686     WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
7687     EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
7688   } else if (!isScopedEHPersonality(Pers)) {
7689     assert(EHPadBB);
7690     MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
7691   }
7692 
7693   return Chain;
7694 }
7695 
7696 std::pair<SDValue, SDValue>
7697 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
7698                                     const BasicBlock *EHPadBB) {
7699   MCSymbol *BeginLabel = nullptr;
7700 
7701   if (EHPadBB) {
7702     // Both PendingLoads and PendingExports must be flushed here;
7703     // this call might not return.
7704     (void)getRoot();
7705     DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
7706     CLI.setChain(getRoot());
7707   }
7708 
7709   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7710   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7711 
7712   assert((CLI.IsTailCall || Result.second.getNode()) &&
7713          "Non-null chain expected with non-tail call!");
7714   assert((Result.second.getNode() || !Result.first.getNode()) &&
7715          "Null value expected with tail call!");
7716 
7717   if (!Result.second.getNode()) {
7718     // As a special case, a null chain means that a tail call has been emitted
7719     // and the DAG root is already updated.
7720     HasTailCall = true;
7721 
7722     // Since there's no actual continuation from this block, nothing can be
7723     // relying on us setting vregs for them.
7724     PendingExports.clear();
7725   } else {
7726     DAG.setRoot(Result.second);
7727   }
7728 
7729   if (EHPadBB) {
7730     DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
7731                            BeginLabel));
7732   }
7733 
7734   return Result;
7735 }
7736 
7737 void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
7738                                       bool isTailCall,
7739                                       bool isMustTailCall,
7740                                       const BasicBlock *EHPadBB) {
7741   auto &DL = DAG.getDataLayout();
7742   FunctionType *FTy = CB.getFunctionType();
7743   Type *RetTy = CB.getType();
7744 
7745   TargetLowering::ArgListTy Args;
7746   Args.reserve(CB.arg_size());
7747 
7748   const Value *SwiftErrorVal = nullptr;
7749   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7750 
7751   if (isTailCall) {
7752     // Avoid emitting tail calls in functions with the disable-tail-calls
7753     // attribute.
7754     auto *Caller = CB.getParent()->getParent();
7755     if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
7756         "true" && !isMustTailCall)
7757       isTailCall = false;
7758 
7759     // We can't tail call inside a function with a swifterror argument. Lowering
7760     // does not support this yet. It would have to move into the swifterror
7761     // register before the call.
7762     if (TLI.supportSwiftError() &&
7763         Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
7764       isTailCall = false;
7765   }
7766 
7767   for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
7768     TargetLowering::ArgListEntry Entry;
7769     const Value *V = *I;
7770 
7771     // Skip empty types
7772     if (V->getType()->isEmptyTy())
7773       continue;
7774 
7775     SDValue ArgNode = getValue(V);
7776     Entry.Node = ArgNode; Entry.Ty = V->getType();
7777 
7778     Entry.setAttributes(&CB, I - CB.arg_begin());
7779 
7780     // Use swifterror virtual register as input to the call.
7781     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
7782       SwiftErrorVal = V;
7783       // We find the virtual register for the actual swifterror argument.
7784       // Instead of using the Value, we use the virtual register instead.
7785       Entry.Node =
7786           DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
7787                           EVT(TLI.getPointerTy(DL)));
7788     }
7789 
7790     Args.push_back(Entry);
7791 
7792     // If we have an explicit sret argument that is an Instruction, (i.e., it
7793     // might point to function-local memory), we can't meaningfully tail-call.
7794     if (Entry.IsSRet && isa<Instruction>(V))
7795       isTailCall = false;
7796   }
7797 
7798   // If call site has a cfguardtarget operand bundle, create and add an
7799   // additional ArgListEntry.
7800   if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
7801     TargetLowering::ArgListEntry Entry;
7802     Value *V = Bundle->Inputs[0];
7803     SDValue ArgNode = getValue(V);
7804     Entry.Node = ArgNode;
7805     Entry.Ty = V->getType();
7806     Entry.IsCFGuardTarget = true;
7807     Args.push_back(Entry);
7808   }
7809 
7810   // Check if target-independent constraints permit a tail call here.
7811   // Target-dependent constraints are checked within TLI->LowerCallTo.
7812   if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
7813     isTailCall = false;
7814 
7815   // Disable tail calls if there is an swifterror argument. Targets have not
7816   // been updated to support tail calls.
7817   if (TLI.supportSwiftError() && SwiftErrorVal)
7818     isTailCall = false;
7819 
7820   TargetLowering::CallLoweringInfo CLI(DAG);
7821   CLI.setDebugLoc(getCurSDLoc())
7822       .setChain(getRoot())
7823       .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
7824       .setTailCall(isTailCall)
7825       .setConvergent(CB.isConvergent())
7826       .setIsPreallocated(
7827           CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
7828   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7829 
7830   if (Result.first.getNode()) {
7831     Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
7832     setValue(&CB, Result.first);
7833   }
7834 
7835   // The last element of CLI.InVals has the SDValue for swifterror return.
7836   // Here we copy it to a virtual register and update SwiftErrorMap for
7837   // book-keeping.
7838   if (SwiftErrorVal && TLI.supportSwiftError()) {
7839     // Get the last element of InVals.
7840     SDValue Src = CLI.InVals.back();
7841     Register VReg =
7842         SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
7843     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
7844     DAG.setRoot(CopyNode);
7845   }
7846 }
7847 
7848 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
7849                              SelectionDAGBuilder &Builder) {
7850   // Check to see if this load can be trivially constant folded, e.g. if the
7851   // input is from a string literal.
7852   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
7853     // Cast pointer to the type we really want to load.
7854     Type *LoadTy =
7855         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
7856     if (LoadVT.isVector())
7857       LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
7858 
7859     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
7860                                          PointerType::getUnqual(LoadTy));
7861 
7862     if (const Constant *LoadCst =
7863             ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
7864                                          LoadTy, Builder.DAG.getDataLayout()))
7865       return Builder.getValue(LoadCst);
7866   }
7867 
7868   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
7869   // still constant memory, the input chain can be the entry node.
7870   SDValue Root;
7871   bool ConstantMemory = false;
7872 
7873   // Do not serialize (non-volatile) loads of constant memory with anything.
7874   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
7875     Root = Builder.DAG.getEntryNode();
7876     ConstantMemory = true;
7877   } else {
7878     // Do not serialize non-volatile loads against each other.
7879     Root = Builder.DAG.getRoot();
7880   }
7881 
7882   SDValue Ptr = Builder.getValue(PtrVal);
7883   SDValue LoadVal =
7884       Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
7885                           MachinePointerInfo(PtrVal), Align(1));
7886 
7887   if (!ConstantMemory)
7888     Builder.PendingLoads.push_back(LoadVal.getValue(1));
7889   return LoadVal;
7890 }
7891 
7892 /// Record the value for an instruction that produces an integer result,
7893 /// converting the type where necessary.
7894 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
7895                                                   SDValue Value,
7896                                                   bool IsSigned) {
7897   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7898                                                     I.getType(), true);
7899   if (IsSigned)
7900     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
7901   else
7902     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
7903   setValue(&I, Value);
7904 }
7905 
7906 /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
7907 /// true and lower it. Otherwise return false, and it will be lowered like a
7908 /// normal call.
7909 /// The caller already checked that \p I calls the appropriate LibFunc with a
7910 /// correct prototype.
7911 bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
7912   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
7913   const Value *Size = I.getArgOperand(2);
7914   const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
7915   if (CSize && CSize->getZExtValue() == 0) {
7916     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7917                                                           I.getType(), true);
7918     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
7919     return true;
7920   }
7921 
7922   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7923   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
7924       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
7925       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
7926   if (Res.first.getNode()) {
7927     processIntegerCallValue(I, Res.first, true);
7928     PendingLoads.push_back(Res.second);
7929     return true;
7930   }
7931 
7932   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
7933   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
7934   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
7935     return false;
7936 
7937   // If the target has a fast compare for the given size, it will return a
7938   // preferred load type for that size. Require that the load VT is legal and
7939   // that the target supports unaligned loads of that type. Otherwise, return
7940   // INVALID.
7941   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
7942     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7943     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
7944     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
7945       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
7946       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
7947       // TODO: Check alignment of src and dest ptrs.
7948       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
7949       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
7950       if (!TLI.isTypeLegal(LVT) ||
7951           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
7952           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
7953         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
7954     }
7955 
7956     return LVT;
7957   };
7958 
7959   // This turns into unaligned loads. We only do this if the target natively
7960   // supports the MVT we'll be loading or if it is small enough (<= 4) that
7961   // we'll only produce a small number of byte loads.
7962   MVT LoadVT;
7963   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
7964   switch (NumBitsToCompare) {
7965   default:
7966     return false;
7967   case 16:
7968     LoadVT = MVT::i16;
7969     break;
7970   case 32:
7971     LoadVT = MVT::i32;
7972     break;
7973   case 64:
7974   case 128:
7975   case 256:
7976     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
7977     break;
7978   }
7979 
7980   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
7981     return false;
7982 
7983   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
7984   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
7985 
7986   // Bitcast to a wide integer type if the loads are vectors.
7987   if (LoadVT.isVector()) {
7988     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
7989     LoadL = DAG.getBitcast(CmpVT, LoadL);
7990     LoadR = DAG.getBitcast(CmpVT, LoadR);
7991   }
7992 
7993   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
7994   processIntegerCallValue(I, Cmp, false);
7995   return true;
7996 }
7997 
7998 /// See if we can lower a memchr call into an optimized form. If so, return
7999 /// true and lower it. Otherwise return false, and it will be lowered like a
8000 /// normal call.
8001 /// The caller already checked that \p I calls the appropriate LibFunc with a
8002 /// correct prototype.
8003 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8004   const Value *Src = I.getArgOperand(0);
8005   const Value *Char = I.getArgOperand(1);
8006   const Value *Length = I.getArgOperand(2);
8007 
8008   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8009   std::pair<SDValue, SDValue> Res =
8010     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
8011                                 getValue(Src), getValue(Char), getValue(Length),
8012                                 MachinePointerInfo(Src));
8013   if (Res.first.getNode()) {
8014     setValue(&I, Res.first);
8015     PendingLoads.push_back(Res.second);
8016     return true;
8017   }
8018 
8019   return false;
8020 }
8021 
8022 /// See if we can lower a mempcpy call into an optimized form. If so, return
8023 /// true and lower it. Otherwise return false, and it will be lowered like a
8024 /// normal call.
8025 /// The caller already checked that \p I calls the appropriate LibFunc with a
8026 /// correct prototype.
8027 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8028   SDValue Dst = getValue(I.getArgOperand(0));
8029   SDValue Src = getValue(I.getArgOperand(1));
8030   SDValue Size = getValue(I.getArgOperand(2));
8031 
8032   Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8033   Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8034   // DAG::getMemcpy needs Alignment to be defined.
8035   Align Alignment = std::min(DstAlign, SrcAlign);
8036 
8037   bool isVol = false;
8038   SDLoc sdl = getCurSDLoc();
8039 
8040   // In the mempcpy context we need to pass in a false value for isTailCall
8041   // because the return pointer needs to be adjusted by the size of
8042   // the copied memory.
8043   SDValue Root = isVol ? getRoot() : getMemoryRoot();
8044   SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false,
8045                              /*isTailCall=*/false,
8046                              MachinePointerInfo(I.getArgOperand(0)),
8047                              MachinePointerInfo(I.getArgOperand(1)),
8048                              I.getAAMetadata());
8049   assert(MC.getNode() != nullptr &&
8050          "** memcpy should not be lowered as TailCall in mempcpy context **");
8051   DAG.setRoot(MC);
8052 
8053   // Check if Size needs to be truncated or extended.
8054   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8055 
8056   // Adjust return pointer to point just past the last dst byte.
8057   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8058                                     Dst, Size);
8059   setValue(&I, DstPlusSize);
8060   return true;
8061 }
8062 
8063 /// See if we can lower a strcpy call into an optimized form.  If so, return
8064 /// true and lower it, otherwise return false and it will be lowered like a
8065 /// normal call.
8066 /// The caller already checked that \p I calls the appropriate LibFunc with a
8067 /// correct prototype.
8068 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8069   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8070 
8071   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8072   std::pair<SDValue, SDValue> Res =
8073     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
8074                                 getValue(Arg0), getValue(Arg1),
8075                                 MachinePointerInfo(Arg0),
8076                                 MachinePointerInfo(Arg1), isStpcpy);
8077   if (Res.first.getNode()) {
8078     setValue(&I, Res.first);
8079     DAG.setRoot(Res.second);
8080     return true;
8081   }
8082 
8083   return false;
8084 }
8085 
8086 /// See if we can lower a strcmp call into an optimized form.  If so, return
8087 /// true and lower it, otherwise return false and it will be lowered like a
8088 /// normal call.
8089 /// The caller already checked that \p I calls the appropriate LibFunc with a
8090 /// correct prototype.
8091 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8092   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8093 
8094   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8095   std::pair<SDValue, SDValue> Res =
8096     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
8097                                 getValue(Arg0), getValue(Arg1),
8098                                 MachinePointerInfo(Arg0),
8099                                 MachinePointerInfo(Arg1));
8100   if (Res.first.getNode()) {
8101     processIntegerCallValue(I, Res.first, true);
8102     PendingLoads.push_back(Res.second);
8103     return true;
8104   }
8105 
8106   return false;
8107 }
8108 
8109 /// See if we can lower a strlen call into an optimized form.  If so, return
8110 /// true and lower it, otherwise return false and it will be lowered like a
8111 /// normal call.
8112 /// The caller already checked that \p I calls the appropriate LibFunc with a
8113 /// correct prototype.
8114 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8115   const Value *Arg0 = I.getArgOperand(0);
8116 
8117   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8118   std::pair<SDValue, SDValue> Res =
8119     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
8120                                 getValue(Arg0), MachinePointerInfo(Arg0));
8121   if (Res.first.getNode()) {
8122     processIntegerCallValue(I, Res.first, false);
8123     PendingLoads.push_back(Res.second);
8124     return true;
8125   }
8126 
8127   return false;
8128 }
8129 
8130 /// See if we can lower a strnlen call into an optimized form.  If so, return
8131 /// true and lower it, otherwise return false and it will be lowered like a
8132 /// normal call.
8133 /// The caller already checked that \p I calls the appropriate LibFunc with a
8134 /// correct prototype.
8135 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8136   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8137 
8138   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8139   std::pair<SDValue, SDValue> Res =
8140     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
8141                                  getValue(Arg0), getValue(Arg1),
8142                                  MachinePointerInfo(Arg0));
8143   if (Res.first.getNode()) {
8144     processIntegerCallValue(I, Res.first, false);
8145     PendingLoads.push_back(Res.second);
8146     return true;
8147   }
8148 
8149   return false;
8150 }
8151 
8152 /// See if we can lower a unary floating-point operation into an SDNode with
8153 /// the specified Opcode.  If so, return true and lower it, otherwise return
8154 /// false and it will be lowered like a normal call.
8155 /// The caller already checked that \p I calls the appropriate LibFunc with a
8156 /// correct prototype.
8157 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8158                                               unsigned Opcode) {
8159   // We already checked this call's prototype; verify it doesn't modify errno.
8160   if (!I.onlyReadsMemory())
8161     return false;
8162 
8163   SDNodeFlags Flags;
8164   Flags.copyFMF(cast<FPMathOperator>(I));
8165 
8166   SDValue Tmp = getValue(I.getArgOperand(0));
8167   setValue(&I,
8168            DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8169   return true;
8170 }
8171 
8172 /// See if we can lower a binary floating-point operation into an SDNode with
8173 /// the specified Opcode. If so, return true and lower it. Otherwise return
8174 /// false, and it will be lowered like a normal call.
8175 /// The caller already checked that \p I calls the appropriate LibFunc with a
8176 /// correct prototype.
8177 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8178                                                unsigned Opcode) {
8179   // We already checked this call's prototype; verify it doesn't modify errno.
8180   if (!I.onlyReadsMemory())
8181     return false;
8182 
8183   SDNodeFlags Flags;
8184   Flags.copyFMF(cast<FPMathOperator>(I));
8185 
8186   SDValue Tmp0 = getValue(I.getArgOperand(0));
8187   SDValue Tmp1 = getValue(I.getArgOperand(1));
8188   EVT VT = Tmp0.getValueType();
8189   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8190   return true;
8191 }
8192 
8193 void SelectionDAGBuilder::visitCall(const CallInst &I) {
8194   // Handle inline assembly differently.
8195   if (I.isInlineAsm()) {
8196     visitInlineAsm(I);
8197     return;
8198   }
8199 
8200   if (Function *F = I.getCalledFunction()) {
8201     diagnoseDontCall(I);
8202 
8203     if (F->isDeclaration()) {
8204       // Is this an LLVM intrinsic or a target-specific intrinsic?
8205       unsigned IID = F->getIntrinsicID();
8206       if (!IID)
8207         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8208           IID = II->getIntrinsicID(F);
8209 
8210       if (IID) {
8211         visitIntrinsicCall(I, IID);
8212         return;
8213       }
8214     }
8215 
8216     // Check for well-known libc/libm calls.  If the function is internal, it
8217     // can't be a library call.  Don't do the check if marked as nobuiltin for
8218     // some reason or the call site requires strict floating point semantics.
8219     LibFunc Func;
8220     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
8221         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
8222         LibInfo->hasOptimizedCodeGen(Func)) {
8223       switch (Func) {
8224       default: break;
8225       case LibFunc_bcmp:
8226         if (visitMemCmpBCmpCall(I))
8227           return;
8228         break;
8229       case LibFunc_copysign:
8230       case LibFunc_copysignf:
8231       case LibFunc_copysignl:
8232         // We already checked this call's prototype; verify it doesn't modify
8233         // errno.
8234         if (I.onlyReadsMemory()) {
8235           SDValue LHS = getValue(I.getArgOperand(0));
8236           SDValue RHS = getValue(I.getArgOperand(1));
8237           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
8238                                    LHS.getValueType(), LHS, RHS));
8239           return;
8240         }
8241         break;
8242       case LibFunc_fabs:
8243       case LibFunc_fabsf:
8244       case LibFunc_fabsl:
8245         if (visitUnaryFloatCall(I, ISD::FABS))
8246           return;
8247         break;
8248       case LibFunc_fmin:
8249       case LibFunc_fminf:
8250       case LibFunc_fminl:
8251         if (visitBinaryFloatCall(I, ISD::FMINNUM))
8252           return;
8253         break;
8254       case LibFunc_fmax:
8255       case LibFunc_fmaxf:
8256       case LibFunc_fmaxl:
8257         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
8258           return;
8259         break;
8260       case LibFunc_sin:
8261       case LibFunc_sinf:
8262       case LibFunc_sinl:
8263         if (visitUnaryFloatCall(I, ISD::FSIN))
8264           return;
8265         break;
8266       case LibFunc_cos:
8267       case LibFunc_cosf:
8268       case LibFunc_cosl:
8269         if (visitUnaryFloatCall(I, ISD::FCOS))
8270           return;
8271         break;
8272       case LibFunc_sqrt:
8273       case LibFunc_sqrtf:
8274       case LibFunc_sqrtl:
8275       case LibFunc_sqrt_finite:
8276       case LibFunc_sqrtf_finite:
8277       case LibFunc_sqrtl_finite:
8278         if (visitUnaryFloatCall(I, ISD::FSQRT))
8279           return;
8280         break;
8281       case LibFunc_floor:
8282       case LibFunc_floorf:
8283       case LibFunc_floorl:
8284         if (visitUnaryFloatCall(I, ISD::FFLOOR))
8285           return;
8286         break;
8287       case LibFunc_nearbyint:
8288       case LibFunc_nearbyintf:
8289       case LibFunc_nearbyintl:
8290         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
8291           return;
8292         break;
8293       case LibFunc_ceil:
8294       case LibFunc_ceilf:
8295       case LibFunc_ceill:
8296         if (visitUnaryFloatCall(I, ISD::FCEIL))
8297           return;
8298         break;
8299       case LibFunc_rint:
8300       case LibFunc_rintf:
8301       case LibFunc_rintl:
8302         if (visitUnaryFloatCall(I, ISD::FRINT))
8303           return;
8304         break;
8305       case LibFunc_round:
8306       case LibFunc_roundf:
8307       case LibFunc_roundl:
8308         if (visitUnaryFloatCall(I, ISD::FROUND))
8309           return;
8310         break;
8311       case LibFunc_trunc:
8312       case LibFunc_truncf:
8313       case LibFunc_truncl:
8314         if (visitUnaryFloatCall(I, ISD::FTRUNC))
8315           return;
8316         break;
8317       case LibFunc_log2:
8318       case LibFunc_log2f:
8319       case LibFunc_log2l:
8320         if (visitUnaryFloatCall(I, ISD::FLOG2))
8321           return;
8322         break;
8323       case LibFunc_exp2:
8324       case LibFunc_exp2f:
8325       case LibFunc_exp2l:
8326         if (visitUnaryFloatCall(I, ISD::FEXP2))
8327           return;
8328         break;
8329       case LibFunc_memcmp:
8330         if (visitMemCmpBCmpCall(I))
8331           return;
8332         break;
8333       case LibFunc_mempcpy:
8334         if (visitMemPCpyCall(I))
8335           return;
8336         break;
8337       case LibFunc_memchr:
8338         if (visitMemChrCall(I))
8339           return;
8340         break;
8341       case LibFunc_strcpy:
8342         if (visitStrCpyCall(I, false))
8343           return;
8344         break;
8345       case LibFunc_stpcpy:
8346         if (visitStrCpyCall(I, true))
8347           return;
8348         break;
8349       case LibFunc_strcmp:
8350         if (visitStrCmpCall(I))
8351           return;
8352         break;
8353       case LibFunc_strlen:
8354         if (visitStrLenCall(I))
8355           return;
8356         break;
8357       case LibFunc_strnlen:
8358         if (visitStrNLenCall(I))
8359           return;
8360         break;
8361       }
8362     }
8363   }
8364 
8365   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
8366   // have to do anything here to lower funclet bundles.
8367   // CFGuardTarget bundles are lowered in LowerCallTo.
8368   assert(!I.hasOperandBundlesOtherThan(
8369              {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
8370               LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
8371               LLVMContext::OB_clang_arc_attachedcall}) &&
8372          "Cannot lower calls with arbitrary operand bundles!");
8373 
8374   SDValue Callee = getValue(I.getCalledOperand());
8375 
8376   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
8377     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
8378   else
8379     // Check if we can potentially perform a tail call. More detailed checking
8380     // is be done within LowerCallTo, after more information about the call is
8381     // known.
8382     LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
8383 }
8384 
8385 namespace {
8386 
8387 /// AsmOperandInfo - This contains information for each constraint that we are
8388 /// lowering.
8389 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
8390 public:
8391   /// CallOperand - If this is the result output operand or a clobber
8392   /// this is null, otherwise it is the incoming operand to the CallInst.
8393   /// This gets modified as the asm is processed.
8394   SDValue CallOperand;
8395 
8396   /// AssignedRegs - If this is a register or register class operand, this
8397   /// contains the set of register corresponding to the operand.
8398   RegsForValue AssignedRegs;
8399 
8400   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
8401     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
8402   }
8403 
8404   /// Whether or not this operand accesses memory
8405   bool hasMemory(const TargetLowering &TLI) const {
8406     // Indirect operand accesses access memory.
8407     if (isIndirect)
8408       return true;
8409 
8410     for (const auto &Code : Codes)
8411       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
8412         return true;
8413 
8414     return false;
8415   }
8416 };
8417 
8418 
8419 } // end anonymous namespace
8420 
8421 /// Make sure that the output operand \p OpInfo and its corresponding input
8422 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
8423 /// out).
8424 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
8425                                SDISelAsmOperandInfo &MatchingOpInfo,
8426                                SelectionDAG &DAG) {
8427   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
8428     return;
8429 
8430   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
8431   const auto &TLI = DAG.getTargetLoweringInfo();
8432 
8433   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
8434       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
8435                                        OpInfo.ConstraintVT);
8436   std::pair<unsigned, const TargetRegisterClass *> InputRC =
8437       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
8438                                        MatchingOpInfo.ConstraintVT);
8439   if ((OpInfo.ConstraintVT.isInteger() !=
8440        MatchingOpInfo.ConstraintVT.isInteger()) ||
8441       (MatchRC.second != InputRC.second)) {
8442     // FIXME: error out in a more elegant fashion
8443     report_fatal_error("Unsupported asm: input constraint"
8444                        " with a matching output constraint of"
8445                        " incompatible type!");
8446   }
8447   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
8448 }
8449 
8450 /// Get a direct memory input to behave well as an indirect operand.
8451 /// This may introduce stores, hence the need for a \p Chain.
8452 /// \return The (possibly updated) chain.
8453 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
8454                                         SDISelAsmOperandInfo &OpInfo,
8455                                         SelectionDAG &DAG) {
8456   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8457 
8458   // If we don't have an indirect input, put it in the constpool if we can,
8459   // otherwise spill it to a stack slot.
8460   // TODO: This isn't quite right. We need to handle these according to
8461   // the addressing mode that the constraint wants. Also, this may take
8462   // an additional register for the computation and we don't want that
8463   // either.
8464 
8465   // If the operand is a float, integer, or vector constant, spill to a
8466   // constant pool entry to get its address.
8467   const Value *OpVal = OpInfo.CallOperandVal;
8468   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
8469       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
8470     OpInfo.CallOperand = DAG.getConstantPool(
8471         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
8472     return Chain;
8473   }
8474 
8475   // Otherwise, create a stack slot and emit a store to it before the asm.
8476   Type *Ty = OpVal->getType();
8477   auto &DL = DAG.getDataLayout();
8478   uint64_t TySize = DL.getTypeAllocSize(Ty);
8479   MachineFunction &MF = DAG.getMachineFunction();
8480   int SSFI = MF.getFrameInfo().CreateStackObject(
8481       TySize, DL.getPrefTypeAlign(Ty), false);
8482   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
8483   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
8484                             MachinePointerInfo::getFixedStack(MF, SSFI),
8485                             TLI.getMemValueType(DL, Ty));
8486   OpInfo.CallOperand = StackSlot;
8487 
8488   return Chain;
8489 }
8490 
8491 /// GetRegistersForValue - Assign registers (virtual or physical) for the
8492 /// specified operand.  We prefer to assign virtual registers, to allow the
8493 /// register allocator to handle the assignment process.  However, if the asm
8494 /// uses features that we can't model on machineinstrs, we have SDISel do the
8495 /// allocation.  This produces generally horrible, but correct, code.
8496 ///
8497 ///   OpInfo describes the operand
8498 ///   RefOpInfo describes the matching operand if any, the operand otherwise
8499 static llvm::Optional<unsigned>
8500 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
8501                      SDISelAsmOperandInfo &OpInfo,
8502                      SDISelAsmOperandInfo &RefOpInfo) {
8503   LLVMContext &Context = *DAG.getContext();
8504   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8505 
8506   MachineFunction &MF = DAG.getMachineFunction();
8507   SmallVector<unsigned, 4> Regs;
8508   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8509 
8510   // No work to do for memory/address operands.
8511   if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8512       OpInfo.ConstraintType == TargetLowering::C_Address)
8513     return None;
8514 
8515   // If this is a constraint for a single physreg, or a constraint for a
8516   // register class, find it.
8517   unsigned AssignedReg;
8518   const TargetRegisterClass *RC;
8519   std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
8520       &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
8521   // RC is unset only on failure. Return immediately.
8522   if (!RC)
8523     return None;
8524 
8525   // Get the actual register value type.  This is important, because the user
8526   // may have asked for (e.g.) the AX register in i32 type.  We need to
8527   // remember that AX is actually i16 to get the right extension.
8528   const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
8529 
8530   if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
8531     // If this is an FP operand in an integer register (or visa versa), or more
8532     // generally if the operand value disagrees with the register class we plan
8533     // to stick it in, fix the operand type.
8534     //
8535     // If this is an input value, the bitcast to the new type is done now.
8536     // Bitcast for output value is done at the end of visitInlineAsm().
8537     if ((OpInfo.Type == InlineAsm::isOutput ||
8538          OpInfo.Type == InlineAsm::isInput) &&
8539         !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
8540       // Try to convert to the first EVT that the reg class contains.  If the
8541       // types are identical size, use a bitcast to convert (e.g. two differing
8542       // vector types).  Note: output bitcast is done at the end of
8543       // visitInlineAsm().
8544       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
8545         // Exclude indirect inputs while they are unsupported because the code
8546         // to perform the load is missing and thus OpInfo.CallOperand still
8547         // refers to the input address rather than the pointed-to value.
8548         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
8549           OpInfo.CallOperand =
8550               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
8551         OpInfo.ConstraintVT = RegVT;
8552         // If the operand is an FP value and we want it in integer registers,
8553         // use the corresponding integer type. This turns an f64 value into
8554         // i64, which can be passed with two i32 values on a 32-bit machine.
8555       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
8556         MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
8557         if (OpInfo.Type == InlineAsm::isInput)
8558           OpInfo.CallOperand =
8559               DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
8560         OpInfo.ConstraintVT = VT;
8561       }
8562     }
8563   }
8564 
8565   // No need to allocate a matching input constraint since the constraint it's
8566   // matching to has already been allocated.
8567   if (OpInfo.isMatchingInputConstraint())
8568     return None;
8569 
8570   EVT ValueVT = OpInfo.ConstraintVT;
8571   if (OpInfo.ConstraintVT == MVT::Other)
8572     ValueVT = RegVT;
8573 
8574   // Initialize NumRegs.
8575   unsigned NumRegs = 1;
8576   if (OpInfo.ConstraintVT != MVT::Other)
8577     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
8578 
8579   // If this is a constraint for a specific physical register, like {r17},
8580   // assign it now.
8581 
8582   // If this associated to a specific register, initialize iterator to correct
8583   // place. If virtual, make sure we have enough registers
8584 
8585   // Initialize iterator if necessary
8586   TargetRegisterClass::iterator I = RC->begin();
8587   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8588 
8589   // Do not check for single registers.
8590   if (AssignedReg) {
8591     I = std::find(I, RC->end(), AssignedReg);
8592     if (I == RC->end()) {
8593       // RC does not contain the selected register, which indicates a
8594       // mismatch between the register and the required type/bitwidth.
8595       return {AssignedReg};
8596     }
8597   }
8598 
8599   for (; NumRegs; --NumRegs, ++I) {
8600     assert(I != RC->end() && "Ran out of registers to allocate!");
8601     Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
8602     Regs.push_back(R);
8603   }
8604 
8605   OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
8606   return None;
8607 }
8608 
8609 static unsigned
8610 findMatchingInlineAsmOperand(unsigned OperandNo,
8611                              const std::vector<SDValue> &AsmNodeOperands) {
8612   // Scan until we find the definition we already emitted of this operand.
8613   unsigned CurOp = InlineAsm::Op_FirstOperand;
8614   for (; OperandNo; --OperandNo) {
8615     // Advance to the next operand.
8616     unsigned OpFlag =
8617         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8618     assert((InlineAsm::isRegDefKind(OpFlag) ||
8619             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
8620             InlineAsm::isMemKind(OpFlag)) &&
8621            "Skipped past definitions?");
8622     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
8623   }
8624   return CurOp;
8625 }
8626 
8627 namespace {
8628 
8629 class ExtraFlags {
8630   unsigned Flags = 0;
8631 
8632 public:
8633   explicit ExtraFlags(const CallBase &Call) {
8634     const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8635     if (IA->hasSideEffects())
8636       Flags |= InlineAsm::Extra_HasSideEffects;
8637     if (IA->isAlignStack())
8638       Flags |= InlineAsm::Extra_IsAlignStack;
8639     if (Call.isConvergent())
8640       Flags |= InlineAsm::Extra_IsConvergent;
8641     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
8642   }
8643 
8644   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
8645     // Ideally, we would only check against memory constraints.  However, the
8646     // meaning of an Other constraint can be target-specific and we can't easily
8647     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
8648     // for Other constraints as well.
8649     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8650         OpInfo.ConstraintType == TargetLowering::C_Other) {
8651       if (OpInfo.Type == InlineAsm::isInput)
8652         Flags |= InlineAsm::Extra_MayLoad;
8653       else if (OpInfo.Type == InlineAsm::isOutput)
8654         Flags |= InlineAsm::Extra_MayStore;
8655       else if (OpInfo.Type == InlineAsm::isClobber)
8656         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
8657     }
8658   }
8659 
8660   unsigned get() const { return Flags; }
8661 };
8662 
8663 } // end anonymous namespace
8664 
8665 /// visitInlineAsm - Handle a call to an InlineAsm object.
8666 void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
8667                                          const BasicBlock *EHPadBB) {
8668   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8669 
8670   /// ConstraintOperands - Information about all of the constraints.
8671   SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
8672 
8673   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8674   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
8675       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
8676 
8677   // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
8678   // AsmDialect, MayLoad, MayStore).
8679   bool HasSideEffect = IA->hasSideEffects();
8680   ExtraFlags ExtraInfo(Call);
8681 
8682   for (auto &T : TargetConstraints) {
8683     ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
8684     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
8685 
8686     if (OpInfo.CallOperandVal)
8687       OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
8688 
8689     if (!HasSideEffect)
8690       HasSideEffect = OpInfo.hasMemory(TLI);
8691 
8692     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
8693     // FIXME: Could we compute this on OpInfo rather than T?
8694 
8695     // Compute the constraint code and ConstraintType to use.
8696     TLI.ComputeConstraintToUse(T, SDValue());
8697 
8698     if (T.ConstraintType == TargetLowering::C_Immediate &&
8699         OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
8700       // We've delayed emitting a diagnostic like the "n" constraint because
8701       // inlining could cause an integer showing up.
8702       return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
8703                                           "' expects an integer constant "
8704                                           "expression");
8705 
8706     ExtraInfo.update(T);
8707   }
8708 
8709   // We won't need to flush pending loads if this asm doesn't touch
8710   // memory and is nonvolatile.
8711   SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
8712 
8713   bool EmitEHLabels = isa<InvokeInst>(Call) && IA->canThrow();
8714   if (EmitEHLabels) {
8715     assert(EHPadBB && "InvokeInst must have an EHPadBB");
8716   }
8717   bool IsCallBr = isa<CallBrInst>(Call);
8718 
8719   if (IsCallBr || EmitEHLabels) {
8720     // If this is a callbr or invoke we need to flush pending exports since
8721     // inlineasm_br and invoke are terminators.
8722     // We need to do this before nodes are glued to the inlineasm_br node.
8723     Chain = getControlRoot();
8724   }
8725 
8726   MCSymbol *BeginLabel = nullptr;
8727   if (EmitEHLabels) {
8728     Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
8729   }
8730 
8731   // Second pass over the constraints: compute which constraint option to use.
8732   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8733     // If this is an output operand with a matching input operand, look up the
8734     // matching input. If their types mismatch, e.g. one is an integer, the
8735     // other is floating point, or their sizes are different, flag it as an
8736     // error.
8737     if (OpInfo.hasMatchingInput()) {
8738       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
8739       patchMatchingInput(OpInfo, Input, DAG);
8740     }
8741 
8742     // Compute the constraint code and ConstraintType to use.
8743     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
8744 
8745     if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
8746          OpInfo.Type == InlineAsm::isClobber) ||
8747         OpInfo.ConstraintType == TargetLowering::C_Address)
8748       continue;
8749 
8750     // If this is a memory input, and if the operand is not indirect, do what we
8751     // need to provide an address for the memory input.
8752     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
8753         !OpInfo.isIndirect) {
8754       assert((OpInfo.isMultipleAlternative ||
8755               (OpInfo.Type == InlineAsm::isInput)) &&
8756              "Can only indirectify direct input operands!");
8757 
8758       // Memory operands really want the address of the value.
8759       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
8760 
8761       // There is no longer a Value* corresponding to this operand.
8762       OpInfo.CallOperandVal = nullptr;
8763 
8764       // It is now an indirect operand.
8765       OpInfo.isIndirect = true;
8766     }
8767 
8768   }
8769 
8770   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
8771   std::vector<SDValue> AsmNodeOperands;
8772   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
8773   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
8774       IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
8775 
8776   // If we have a !srcloc metadata node associated with it, we want to attach
8777   // this to the ultimately generated inline asm machineinstr.  To do this, we
8778   // pass in the third operand as this (potentially null) inline asm MDNode.
8779   const MDNode *SrcLoc = Call.getMetadata("srcloc");
8780   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
8781 
8782   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
8783   // bits as operand 3.
8784   AsmNodeOperands.push_back(DAG.getTargetConstant(
8785       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8786 
8787   // Third pass: Loop over operands to prepare DAG-level operands.. As part of
8788   // this, assign virtual and physical registers for inputs and otput.
8789   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8790     // Assign Registers.
8791     SDISelAsmOperandInfo &RefOpInfo =
8792         OpInfo.isMatchingInputConstraint()
8793             ? ConstraintOperands[OpInfo.getMatchedOperand()]
8794             : OpInfo;
8795     const auto RegError =
8796         getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
8797     if (RegError) {
8798       const MachineFunction &MF = DAG.getMachineFunction();
8799       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8800       const char *RegName = TRI.getName(RegError.value());
8801       emitInlineAsmError(Call, "register '" + Twine(RegName) +
8802                                    "' allocated for constraint '" +
8803                                    Twine(OpInfo.ConstraintCode) +
8804                                    "' does not match required type");
8805       return;
8806     }
8807 
8808     auto DetectWriteToReservedRegister = [&]() {
8809       const MachineFunction &MF = DAG.getMachineFunction();
8810       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8811       for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
8812         if (Register::isPhysicalRegister(Reg) &&
8813             TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
8814           const char *RegName = TRI.getName(Reg);
8815           emitInlineAsmError(Call, "write to reserved register '" +
8816                                        Twine(RegName) + "'");
8817           return true;
8818         }
8819       }
8820       return false;
8821     };
8822     assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
8823             (OpInfo.Type == InlineAsm::isInput &&
8824              !OpInfo.isMatchingInputConstraint())) &&
8825            "Only address as input operand is allowed.");
8826 
8827     switch (OpInfo.Type) {
8828     case InlineAsm::isOutput:
8829       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
8830         unsigned ConstraintID =
8831             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8832         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8833                "Failed to convert memory constraint code to constraint id.");
8834 
8835         // Add information to the INLINEASM node to know about this output.
8836         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
8837         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
8838         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
8839                                                         MVT::i32));
8840         AsmNodeOperands.push_back(OpInfo.CallOperand);
8841       } else {
8842         // Otherwise, this outputs to a register (directly for C_Register /
8843         // C_RegisterClass, and a target-defined fashion for
8844         // C_Immediate/C_Other). Find a register that we can use.
8845         if (OpInfo.AssignedRegs.Regs.empty()) {
8846           emitInlineAsmError(
8847               Call, "couldn't allocate output register for constraint '" +
8848                         Twine(OpInfo.ConstraintCode) + "'");
8849           return;
8850         }
8851 
8852         if (DetectWriteToReservedRegister())
8853           return;
8854 
8855         // Add information to the INLINEASM node to know that this register is
8856         // set.
8857         OpInfo.AssignedRegs.AddInlineAsmOperands(
8858             OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
8859                                   : InlineAsm::Kind_RegDef,
8860             false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
8861       }
8862       break;
8863 
8864     case InlineAsm::isInput:
8865     case InlineAsm::isLabel: {
8866       SDValue InOperandVal = OpInfo.CallOperand;
8867 
8868       if (OpInfo.isMatchingInputConstraint()) {
8869         // If this is required to match an output register we have already set,
8870         // just use its register.
8871         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
8872                                                   AsmNodeOperands);
8873         unsigned OpFlag =
8874           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8875         if (InlineAsm::isRegDefKind(OpFlag) ||
8876             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
8877           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
8878           if (OpInfo.isIndirect) {
8879             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
8880             emitInlineAsmError(Call, "inline asm not supported yet: "
8881                                      "don't know how to handle tied "
8882                                      "indirect register inputs");
8883             return;
8884           }
8885 
8886           SmallVector<unsigned, 4> Regs;
8887           MachineFunction &MF = DAG.getMachineFunction();
8888           MachineRegisterInfo &MRI = MF.getRegInfo();
8889           const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8890           auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
8891           Register TiedReg = R->getReg();
8892           MVT RegVT = R->getSimpleValueType(0);
8893           const TargetRegisterClass *RC =
8894               TiedReg.isVirtual()     ? MRI.getRegClass(TiedReg)
8895               : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
8896                                       : TRI.getMinimalPhysRegClass(TiedReg);
8897           unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
8898           for (unsigned i = 0; i != NumRegs; ++i)
8899             Regs.push_back(MRI.createVirtualRegister(RC));
8900 
8901           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
8902 
8903           SDLoc dl = getCurSDLoc();
8904           // Use the produced MatchedRegs object to
8905           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call);
8906           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
8907                                            true, OpInfo.getMatchedOperand(), dl,
8908                                            DAG, AsmNodeOperands);
8909           break;
8910         }
8911 
8912         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
8913         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
8914                "Unexpected number of operands");
8915         // Add information to the INLINEASM node to know about this input.
8916         // See InlineAsm.h isUseOperandTiedToDef.
8917         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
8918         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
8919                                                     OpInfo.getMatchedOperand());
8920         AsmNodeOperands.push_back(DAG.getTargetConstant(
8921             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8922         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
8923         break;
8924       }
8925 
8926       // Treat indirect 'X' constraint as memory.
8927       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
8928           OpInfo.isIndirect)
8929         OpInfo.ConstraintType = TargetLowering::C_Memory;
8930 
8931       if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
8932           OpInfo.ConstraintType == TargetLowering::C_Other) {
8933         std::vector<SDValue> Ops;
8934         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
8935                                           Ops, DAG);
8936         if (Ops.empty()) {
8937           if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
8938             if (isa<ConstantSDNode>(InOperandVal)) {
8939               emitInlineAsmError(Call, "value out of range for constraint '" +
8940                                            Twine(OpInfo.ConstraintCode) + "'");
8941               return;
8942             }
8943 
8944           emitInlineAsmError(Call,
8945                              "invalid operand for inline asm constraint '" +
8946                                  Twine(OpInfo.ConstraintCode) + "'");
8947           return;
8948         }
8949 
8950         // Add information to the INLINEASM node to know about this input.
8951         unsigned ResOpType =
8952           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
8953         AsmNodeOperands.push_back(DAG.getTargetConstant(
8954             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8955         llvm::append_range(AsmNodeOperands, Ops);
8956         break;
8957       }
8958 
8959       if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8960           OpInfo.ConstraintType == TargetLowering::C_Address) {
8961         assert((OpInfo.isIndirect ||
8962                 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
8963                "Operand must be indirect to be a mem!");
8964         assert(InOperandVal.getValueType() ==
8965                    TLI.getPointerTy(DAG.getDataLayout()) &&
8966                "Memory operands expect pointer values");
8967 
8968         unsigned ConstraintID =
8969             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8970         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8971                "Failed to convert memory constraint code to constraint id.");
8972 
8973         // Add information to the INLINEASM node to know about this input.
8974         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
8975         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
8976         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
8977                                                         getCurSDLoc(),
8978                                                         MVT::i32));
8979         AsmNodeOperands.push_back(InOperandVal);
8980         break;
8981       }
8982 
8983       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
8984               OpInfo.ConstraintType == TargetLowering::C_Register) &&
8985              "Unknown constraint type!");
8986 
8987       // TODO: Support this.
8988       if (OpInfo.isIndirect) {
8989         emitInlineAsmError(
8990             Call, "Don't know how to handle indirect register inputs yet "
8991                   "for constraint '" +
8992                       Twine(OpInfo.ConstraintCode) + "'");
8993         return;
8994       }
8995 
8996       // Copy the input into the appropriate registers.
8997       if (OpInfo.AssignedRegs.Regs.empty()) {
8998         emitInlineAsmError(Call,
8999                            "couldn't allocate input reg for constraint '" +
9000                                Twine(OpInfo.ConstraintCode) + "'");
9001         return;
9002       }
9003 
9004       if (DetectWriteToReservedRegister())
9005         return;
9006 
9007       SDLoc dl = getCurSDLoc();
9008 
9009       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
9010                                         &Call);
9011 
9012       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
9013                                                dl, DAG, AsmNodeOperands);
9014       break;
9015     }
9016     case InlineAsm::isClobber:
9017       // Add the clobbered value to the operand list, so that the register
9018       // allocator is aware that the physreg got clobbered.
9019       if (!OpInfo.AssignedRegs.Regs.empty())
9020         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
9021                                                  false, 0, getCurSDLoc(), DAG,
9022                                                  AsmNodeOperands);
9023       break;
9024     }
9025   }
9026 
9027   // Finish up input operands.  Set the input chain and add the flag last.
9028   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9029   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
9030 
9031   unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9032   Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9033                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9034   Flag = Chain.getValue(1);
9035 
9036   // Do additional work to generate outputs.
9037 
9038   SmallVector<EVT, 1> ResultVTs;
9039   SmallVector<SDValue, 1> ResultValues;
9040   SmallVector<SDValue, 8> OutChains;
9041 
9042   llvm::Type *CallResultType = Call.getType();
9043   ArrayRef<Type *> ResultTypes;
9044   if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9045     ResultTypes = StructResult->elements();
9046   else if (!CallResultType->isVoidTy())
9047     ResultTypes = makeArrayRef(CallResultType);
9048 
9049   auto CurResultType = ResultTypes.begin();
9050   auto handleRegAssign = [&](SDValue V) {
9051     assert(CurResultType != ResultTypes.end() && "Unexpected value");
9052     assert((*CurResultType)->isSized() && "Unexpected unsized type");
9053     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9054     ++CurResultType;
9055     // If the type of the inline asm call site return value is different but has
9056     // same size as the type of the asm output bitcast it.  One example of this
9057     // is for vectors with different width / number of elements.  This can
9058     // happen for register classes that can contain multiple different value
9059     // types.  The preg or vreg allocated may not have the same VT as was
9060     // expected.
9061     //
9062     // This can also happen for a return value that disagrees with the register
9063     // class it is put in, eg. a double in a general-purpose register on a
9064     // 32-bit machine.
9065     if (ResultVT != V.getValueType() &&
9066         ResultVT.getSizeInBits() == V.getValueSizeInBits())
9067       V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9068     else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9069              V.getValueType().isInteger()) {
9070       // If a result value was tied to an input value, the computed result
9071       // may have a wider width than the expected result.  Extract the
9072       // relevant portion.
9073       V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9074     }
9075     assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9076     ResultVTs.push_back(ResultVT);
9077     ResultValues.push_back(V);
9078   };
9079 
9080   // Deal with output operands.
9081   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9082     if (OpInfo.Type == InlineAsm::isOutput) {
9083       SDValue Val;
9084       // Skip trivial output operands.
9085       if (OpInfo.AssignedRegs.Regs.empty())
9086         continue;
9087 
9088       switch (OpInfo.ConstraintType) {
9089       case TargetLowering::C_Register:
9090       case TargetLowering::C_RegisterClass:
9091         Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9092                                                   Chain, &Flag, &Call);
9093         break;
9094       case TargetLowering::C_Immediate:
9095       case TargetLowering::C_Other:
9096         Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
9097                                               OpInfo, DAG);
9098         break;
9099       case TargetLowering::C_Memory:
9100         break; // Already handled.
9101       case TargetLowering::C_Address:
9102         break; // Silence warning.
9103       case TargetLowering::C_Unknown:
9104         assert(false && "Unexpected unknown constraint");
9105       }
9106 
9107       // Indirect output manifest as stores. Record output chains.
9108       if (OpInfo.isIndirect) {
9109         const Value *Ptr = OpInfo.CallOperandVal;
9110         assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9111         SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9112                                      MachinePointerInfo(Ptr));
9113         OutChains.push_back(Store);
9114       } else {
9115         // generate CopyFromRegs to associated registers.
9116         assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9117         if (Val.getOpcode() == ISD::MERGE_VALUES) {
9118           for (const SDValue &V : Val->op_values())
9119             handleRegAssign(V);
9120         } else
9121           handleRegAssign(Val);
9122       }
9123     }
9124   }
9125 
9126   // Set results.
9127   if (!ResultValues.empty()) {
9128     assert(CurResultType == ResultTypes.end() &&
9129            "Mismatch in number of ResultTypes");
9130     assert(ResultValues.size() == ResultTypes.size() &&
9131            "Mismatch in number of output operands in asm result");
9132 
9133     SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
9134                             DAG.getVTList(ResultVTs), ResultValues);
9135     setValue(&Call, V);
9136   }
9137 
9138   // Collect store chains.
9139   if (!OutChains.empty())
9140     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
9141 
9142   if (EmitEHLabels) {
9143     Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
9144   }
9145 
9146   // Only Update Root if inline assembly has a memory effect.
9147   if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
9148       EmitEHLabels)
9149     DAG.setRoot(Chain);
9150 }
9151 
9152 void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
9153                                              const Twine &Message) {
9154   LLVMContext &Ctx = *DAG.getContext();
9155   Ctx.emitError(&Call, Message);
9156 
9157   // Make sure we leave the DAG in a valid state
9158   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9159   SmallVector<EVT, 1> ValueVTs;
9160   ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
9161 
9162   if (ValueVTs.empty())
9163     return;
9164 
9165   SmallVector<SDValue, 1> Ops;
9166   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
9167     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
9168 
9169   setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
9170 }
9171 
9172 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
9173   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
9174                           MVT::Other, getRoot(),
9175                           getValue(I.getArgOperand(0)),
9176                           DAG.getSrcValue(I.getArgOperand(0))));
9177 }
9178 
9179 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
9180   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9181   const DataLayout &DL = DAG.getDataLayout();
9182   SDValue V = DAG.getVAArg(
9183       TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
9184       getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
9185       DL.getABITypeAlign(I.getType()).value());
9186   DAG.setRoot(V.getValue(1));
9187 
9188   if (I.getType()->isPointerTy())
9189     V = DAG.getPtrExtOrTrunc(
9190         V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
9191   setValue(&I, V);
9192 }
9193 
9194 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
9195   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
9196                           MVT::Other, getRoot(),
9197                           getValue(I.getArgOperand(0)),
9198                           DAG.getSrcValue(I.getArgOperand(0))));
9199 }
9200 
9201 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
9202   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
9203                           MVT::Other, getRoot(),
9204                           getValue(I.getArgOperand(0)),
9205                           getValue(I.getArgOperand(1)),
9206                           DAG.getSrcValue(I.getArgOperand(0)),
9207                           DAG.getSrcValue(I.getArgOperand(1))));
9208 }
9209 
9210 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
9211                                                     const Instruction &I,
9212                                                     SDValue Op) {
9213   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
9214   if (!Range)
9215     return Op;
9216 
9217   ConstantRange CR = getConstantRangeFromMetadata(*Range);
9218   if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
9219     return Op;
9220 
9221   APInt Lo = CR.getUnsignedMin();
9222   if (!Lo.isMinValue())
9223     return Op;
9224 
9225   APInt Hi = CR.getUnsignedMax();
9226   unsigned Bits = std::max(Hi.getActiveBits(),
9227                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
9228 
9229   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
9230 
9231   SDLoc SL = getCurSDLoc();
9232 
9233   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
9234                              DAG.getValueType(SmallVT));
9235   unsigned NumVals = Op.getNode()->getNumValues();
9236   if (NumVals == 1)
9237     return ZExt;
9238 
9239   SmallVector<SDValue, 4> Ops;
9240 
9241   Ops.push_back(ZExt);
9242   for (unsigned I = 1; I != NumVals; ++I)
9243     Ops.push_back(Op.getValue(I));
9244 
9245   return DAG.getMergeValues(Ops, SL);
9246 }
9247 
9248 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
9249 /// the call being lowered.
9250 ///
9251 /// This is a helper for lowering intrinsics that follow a target calling
9252 /// convention or require stack pointer adjustment. Only a subset of the
9253 /// intrinsic's operands need to participate in the calling convention.
9254 void SelectionDAGBuilder::populateCallLoweringInfo(
9255     TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
9256     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
9257     bool IsPatchPoint) {
9258   TargetLowering::ArgListTy Args;
9259   Args.reserve(NumArgs);
9260 
9261   // Populate the argument list.
9262   // Attributes for args start at offset 1, after the return attribute.
9263   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
9264        ArgI != ArgE; ++ArgI) {
9265     const Value *V = Call->getOperand(ArgI);
9266 
9267     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
9268 
9269     TargetLowering::ArgListEntry Entry;
9270     Entry.Node = getValue(V);
9271     Entry.Ty = V->getType();
9272     Entry.setAttributes(Call, ArgI);
9273     Args.push_back(Entry);
9274   }
9275 
9276   CLI.setDebugLoc(getCurSDLoc())
9277       .setChain(getRoot())
9278       .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
9279       .setDiscardResult(Call->use_empty())
9280       .setIsPatchPoint(IsPatchPoint)
9281       .setIsPreallocated(
9282           Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
9283 }
9284 
9285 /// Add a stack map intrinsic call's live variable operands to a stackmap
9286 /// or patchpoint target node's operand list.
9287 ///
9288 /// Constants are converted to TargetConstants purely as an optimization to
9289 /// avoid constant materialization and register allocation.
9290 ///
9291 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
9292 /// generate addess computation nodes, and so FinalizeISel can convert the
9293 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
9294 /// address materialization and register allocation, but may also be required
9295 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
9296 /// alloca in the entry block, then the runtime may assume that the alloca's
9297 /// StackMap location can be read immediately after compilation and that the
9298 /// location is valid at any point during execution (this is similar to the
9299 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
9300 /// only available in a register, then the runtime would need to trap when
9301 /// execution reaches the StackMap in order to read the alloca's location.
9302 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
9303                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
9304                                 SelectionDAGBuilder &Builder) {
9305   for (unsigned i = StartIdx, e = Call.arg_size(); i != e; ++i) {
9306     SDValue OpVal = Builder.getValue(Call.getArgOperand(i));
9307     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
9308       Ops.push_back(
9309         Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
9310       Ops.push_back(
9311         Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
9312     } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
9313       const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
9314       Ops.push_back(Builder.DAG.getTargetFrameIndex(
9315           FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
9316     } else
9317       Ops.push_back(OpVal);
9318   }
9319 }
9320 
9321 /// Lower llvm.experimental.stackmap.
9322 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
9323   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
9324   //                                  [live variables...])
9325 
9326   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
9327 
9328   SDValue Chain, InFlag, Callee, NullPtr;
9329   SmallVector<SDValue, 32> Ops;
9330 
9331   SDLoc DL = getCurSDLoc();
9332   Callee = getValue(CI.getCalledOperand());
9333   NullPtr = DAG.getIntPtrConstant(0, DL, true);
9334 
9335   // The stackmap intrinsic only records the live variables (the arguments
9336   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
9337   // intrinsic, this won't be lowered to a function call. This means we don't
9338   // have to worry about calling conventions and target specific lowering code.
9339   // Instead we perform the call lowering right here.
9340   //
9341   // chain, flag = CALLSEQ_START(chain, 0, 0)
9342   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
9343   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
9344   //
9345   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
9346   InFlag = Chain.getValue(1);
9347 
9348   // Add the STACKMAP operands, starting with DAG house-keeping.
9349   Ops.push_back(Chain);
9350   Ops.push_back(InFlag);
9351 
9352   // Add the <id>, <numShadowBytes> operands.
9353   //
9354   // These do not require legalisation, and can be emitted directly to target
9355   // constant nodes.
9356   SDValue ID = getValue(CI.getArgOperand(0));
9357   assert(ID.getValueType() == MVT::i64);
9358   SDValue IDConst = DAG.getTargetConstant(
9359       cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType());
9360   Ops.push_back(IDConst);
9361 
9362   SDValue Shad = getValue(CI.getArgOperand(1));
9363   assert(Shad.getValueType() == MVT::i32);
9364   SDValue ShadConst = DAG.getTargetConstant(
9365       cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
9366   Ops.push_back(ShadConst);
9367 
9368   // Add the live variables.
9369   for (unsigned I = 2; I < CI.arg_size(); I++) {
9370     SDValue Op = getValue(CI.getArgOperand(I));
9371 
9372     // Things on the stack are pointer-typed, meaning that they are already
9373     // legal and can be emitted directly to target nodes.
9374     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
9375       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9376       Ops.push_back(DAG.getTargetFrameIndex(
9377           FI->getIndex(), TLI.getFrameIndexTy(DAG.getDataLayout())));
9378     } else {
9379       // Otherwise emit a target independent node to be legalised.
9380       Ops.push_back(getValue(CI.getArgOperand(I)));
9381     }
9382   }
9383 
9384   // Create the STACKMAP node.
9385   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9386   Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
9387   InFlag = Chain.getValue(1);
9388 
9389   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
9390 
9391   // Stackmaps don't generate values, so nothing goes into the NodeMap.
9392 
9393   // Set the root to the target-lowered call chain.
9394   DAG.setRoot(Chain);
9395 
9396   // Inform the Frame Information that we have a stackmap in this function.
9397   FuncInfo.MF->getFrameInfo().setHasStackMap();
9398 }
9399 
9400 /// Lower llvm.experimental.patchpoint directly to its target opcode.
9401 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
9402                                           const BasicBlock *EHPadBB) {
9403   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
9404   //                                                 i32 <numBytes>,
9405   //                                                 i8* <target>,
9406   //                                                 i32 <numArgs>,
9407   //                                                 [Args...],
9408   //                                                 [live variables...])
9409 
9410   CallingConv::ID CC = CB.getCallingConv();
9411   bool IsAnyRegCC = CC == CallingConv::AnyReg;
9412   bool HasDef = !CB.getType()->isVoidTy();
9413   SDLoc dl = getCurSDLoc();
9414   SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));
9415 
9416   // Handle immediate and symbolic callees.
9417   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
9418     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
9419                                    /*isTarget=*/true);
9420   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
9421     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
9422                                          SDLoc(SymbolicCallee),
9423                                          SymbolicCallee->getValueType(0));
9424 
9425   // Get the real number of arguments participating in the call <numArgs>
9426   SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
9427   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
9428 
9429   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
9430   // Intrinsics include all meta-operands up to but not including CC.
9431   unsigned NumMetaOpers = PatchPointOpers::CCPos;
9432   assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
9433          "Not enough arguments provided to the patchpoint intrinsic");
9434 
9435   // For AnyRegCC the arguments are lowered later on manually.
9436   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
9437   Type *ReturnTy =
9438       IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
9439 
9440   TargetLowering::CallLoweringInfo CLI(DAG);
9441   populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
9442                            ReturnTy, true);
9443   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9444 
9445   SDNode *CallEnd = Result.second.getNode();
9446   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
9447     CallEnd = CallEnd->getOperand(0).getNode();
9448 
9449   /// Get a call instruction from the call sequence chain.
9450   /// Tail calls are not allowed.
9451   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
9452          "Expected a callseq node.");
9453   SDNode *Call = CallEnd->getOperand(0).getNode();
9454   bool HasGlue = Call->getGluedNode();
9455 
9456   // Replace the target specific call node with the patchable intrinsic.
9457   SmallVector<SDValue, 8> Ops;
9458 
9459   // Add the <id> and <numBytes> constants.
9460   SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
9461   Ops.push_back(DAG.getTargetConstant(
9462                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
9463   SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
9464   Ops.push_back(DAG.getTargetConstant(
9465                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
9466                   MVT::i32));
9467 
9468   // Add the callee.
9469   Ops.push_back(Callee);
9470 
9471   // Adjust <numArgs> to account for any arguments that have been passed on the
9472   // stack instead.
9473   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
9474   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
9475   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
9476   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
9477 
9478   // Add the calling convention
9479   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
9480 
9481   // Add the arguments we omitted previously. The register allocator should
9482   // place these in any free register.
9483   if (IsAnyRegCC)
9484     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
9485       Ops.push_back(getValue(CB.getArgOperand(i)));
9486 
9487   // Push the arguments from the call instruction up to the register mask.
9488   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
9489   Ops.append(Call->op_begin() + 2, e);
9490 
9491   // Push live variables for the stack map.
9492   addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
9493 
9494   // Push the register mask info.
9495   if (HasGlue)
9496     Ops.push_back(*(Call->op_end()-2));
9497   else
9498     Ops.push_back(*(Call->op_end()-1));
9499 
9500   // Push the chain (this is originally the first operand of the call, but
9501   // becomes now the last or second to last operand).
9502   Ops.push_back(*(Call->op_begin()));
9503 
9504   // Push the glue flag (last operand).
9505   if (HasGlue)
9506     Ops.push_back(*(Call->op_end()-1));
9507 
9508   SDVTList NodeTys;
9509   if (IsAnyRegCC && HasDef) {
9510     // Create the return types based on the intrinsic definition
9511     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9512     SmallVector<EVT, 3> ValueVTs;
9513     ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
9514     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
9515 
9516     // There is always a chain and a glue type at the end
9517     ValueVTs.push_back(MVT::Other);
9518     ValueVTs.push_back(MVT::Glue);
9519     NodeTys = DAG.getVTList(ValueVTs);
9520   } else
9521     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9522 
9523   // Replace the target specific call node with a PATCHPOINT node.
9524   MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
9525                                          dl, NodeTys, Ops);
9526 
9527   // Update the NodeMap.
9528   if (HasDef) {
9529     if (IsAnyRegCC)
9530       setValue(&CB, SDValue(MN, 0));
9531     else
9532       setValue(&CB, Result.first);
9533   }
9534 
9535   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
9536   // call sequence. Furthermore the location of the chain and glue can change
9537   // when the AnyReg calling convention is used and the intrinsic returns a
9538   // value.
9539   if (IsAnyRegCC && HasDef) {
9540     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
9541     SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
9542     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
9543   } else
9544     DAG.ReplaceAllUsesWith(Call, MN);
9545   DAG.DeleteNode(Call);
9546 
9547   // Inform the Frame Information that we have a patchpoint in this function.
9548   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
9549 }
9550 
9551 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
9552                                             unsigned Intrinsic) {
9553   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9554   SDValue Op1 = getValue(I.getArgOperand(0));
9555   SDValue Op2;
9556   if (I.arg_size() > 1)
9557     Op2 = getValue(I.getArgOperand(1));
9558   SDLoc dl = getCurSDLoc();
9559   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
9560   SDValue Res;
9561   SDNodeFlags SDFlags;
9562   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
9563     SDFlags.copyFMF(*FPMO);
9564 
9565   switch (Intrinsic) {
9566   case Intrinsic::vector_reduce_fadd:
9567     if (SDFlags.hasAllowReassociation())
9568       Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
9569                         DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
9570                         SDFlags);
9571     else
9572       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
9573     break;
9574   case Intrinsic::vector_reduce_fmul:
9575     if (SDFlags.hasAllowReassociation())
9576       Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
9577                         DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
9578                         SDFlags);
9579     else
9580       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
9581     break;
9582   case Intrinsic::vector_reduce_add:
9583     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
9584     break;
9585   case Intrinsic::vector_reduce_mul:
9586     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
9587     break;
9588   case Intrinsic::vector_reduce_and:
9589     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
9590     break;
9591   case Intrinsic::vector_reduce_or:
9592     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
9593     break;
9594   case Intrinsic::vector_reduce_xor:
9595     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
9596     break;
9597   case Intrinsic::vector_reduce_smax:
9598     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
9599     break;
9600   case Intrinsic::vector_reduce_smin:
9601     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
9602     break;
9603   case Intrinsic::vector_reduce_umax:
9604     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
9605     break;
9606   case Intrinsic::vector_reduce_umin:
9607     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
9608     break;
9609   case Intrinsic::vector_reduce_fmax:
9610     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
9611     break;
9612   case Intrinsic::vector_reduce_fmin:
9613     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
9614     break;
9615   default:
9616     llvm_unreachable("Unhandled vector reduce intrinsic");
9617   }
9618   setValue(&I, Res);
9619 }
9620 
9621 /// Returns an AttributeList representing the attributes applied to the return
9622 /// value of the given call.
9623 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
9624   SmallVector<Attribute::AttrKind, 2> Attrs;
9625   if (CLI.RetSExt)
9626     Attrs.push_back(Attribute::SExt);
9627   if (CLI.RetZExt)
9628     Attrs.push_back(Attribute::ZExt);
9629   if (CLI.IsInReg)
9630     Attrs.push_back(Attribute::InReg);
9631 
9632   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
9633                             Attrs);
9634 }
9635 
9636 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
9637 /// implementation, which just calls LowerCall.
9638 /// FIXME: When all targets are
9639 /// migrated to using LowerCall, this hook should be integrated into SDISel.
9640 std::pair<SDValue, SDValue>
9641 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
9642   // Handle the incoming return values from the call.
9643   CLI.Ins.clear();
9644   Type *OrigRetTy = CLI.RetTy;
9645   SmallVector<EVT, 4> RetTys;
9646   SmallVector<uint64_t, 4> Offsets;
9647   auto &DL = CLI.DAG.getDataLayout();
9648   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
9649 
9650   if (CLI.IsPostTypeLegalization) {
9651     // If we are lowering a libcall after legalization, split the return type.
9652     SmallVector<EVT, 4> OldRetTys;
9653     SmallVector<uint64_t, 4> OldOffsets;
9654     RetTys.swap(OldRetTys);
9655     Offsets.swap(OldOffsets);
9656 
9657     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
9658       EVT RetVT = OldRetTys[i];
9659       uint64_t Offset = OldOffsets[i];
9660       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
9661       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
9662       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
9663       RetTys.append(NumRegs, RegisterVT);
9664       for (unsigned j = 0; j != NumRegs; ++j)
9665         Offsets.push_back(Offset + j * RegisterVTByteSZ);
9666     }
9667   }
9668 
9669   SmallVector<ISD::OutputArg, 4> Outs;
9670   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
9671 
9672   bool CanLowerReturn =
9673       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
9674                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
9675 
9676   SDValue DemoteStackSlot;
9677   int DemoteStackIdx = -100;
9678   if (!CanLowerReturn) {
9679     // FIXME: equivalent assert?
9680     // assert(!CS.hasInAllocaArgument() &&
9681     //        "sret demotion is incompatible with inalloca");
9682     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
9683     Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
9684     MachineFunction &MF = CLI.DAG.getMachineFunction();
9685     DemoteStackIdx =
9686         MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
9687     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
9688                                               DL.getAllocaAddrSpace());
9689 
9690     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
9691     ArgListEntry Entry;
9692     Entry.Node = DemoteStackSlot;
9693     Entry.Ty = StackSlotPtrType;
9694     Entry.IsSExt = false;
9695     Entry.IsZExt = false;
9696     Entry.IsInReg = false;
9697     Entry.IsSRet = true;
9698     Entry.IsNest = false;
9699     Entry.IsByVal = false;
9700     Entry.IsByRef = false;
9701     Entry.IsReturned = false;
9702     Entry.IsSwiftSelf = false;
9703     Entry.IsSwiftAsync = false;
9704     Entry.IsSwiftError = false;
9705     Entry.IsCFGuardTarget = false;
9706     Entry.Alignment = Alignment;
9707     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
9708     CLI.NumFixedArgs += 1;
9709     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
9710 
9711     // sret demotion isn't compatible with tail-calls, since the sret argument
9712     // points into the callers stack frame.
9713     CLI.IsTailCall = false;
9714   } else {
9715     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9716         CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
9717     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
9718       ISD::ArgFlagsTy Flags;
9719       if (NeedsRegBlock) {
9720         Flags.setInConsecutiveRegs();
9721         if (I == RetTys.size() - 1)
9722           Flags.setInConsecutiveRegsLast();
9723       }
9724       EVT VT = RetTys[I];
9725       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9726                                                      CLI.CallConv, VT);
9727       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9728                                                        CLI.CallConv, VT);
9729       for (unsigned i = 0; i != NumRegs; ++i) {
9730         ISD::InputArg MyFlags;
9731         MyFlags.Flags = Flags;
9732         MyFlags.VT = RegisterVT;
9733         MyFlags.ArgVT = VT;
9734         MyFlags.Used = CLI.IsReturnValueUsed;
9735         if (CLI.RetTy->isPointerTy()) {
9736           MyFlags.Flags.setPointer();
9737           MyFlags.Flags.setPointerAddrSpace(
9738               cast<PointerType>(CLI.RetTy)->getAddressSpace());
9739         }
9740         if (CLI.RetSExt)
9741           MyFlags.Flags.setSExt();
9742         if (CLI.RetZExt)
9743           MyFlags.Flags.setZExt();
9744         if (CLI.IsInReg)
9745           MyFlags.Flags.setInReg();
9746         CLI.Ins.push_back(MyFlags);
9747       }
9748     }
9749   }
9750 
9751   // We push in swifterror return as the last element of CLI.Ins.
9752   ArgListTy &Args = CLI.getArgs();
9753   if (supportSwiftError()) {
9754     for (const ArgListEntry &Arg : Args) {
9755       if (Arg.IsSwiftError) {
9756         ISD::InputArg MyFlags;
9757         MyFlags.VT = getPointerTy(DL);
9758         MyFlags.ArgVT = EVT(getPointerTy(DL));
9759         MyFlags.Flags.setSwiftError();
9760         CLI.Ins.push_back(MyFlags);
9761       }
9762     }
9763   }
9764 
9765   // Handle all of the outgoing arguments.
9766   CLI.Outs.clear();
9767   CLI.OutVals.clear();
9768   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
9769     SmallVector<EVT, 4> ValueVTs;
9770     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
9771     // FIXME: Split arguments if CLI.IsPostTypeLegalization
9772     Type *FinalType = Args[i].Ty;
9773     if (Args[i].IsByVal)
9774       FinalType = Args[i].IndirectType;
9775     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9776         FinalType, CLI.CallConv, CLI.IsVarArg, DL);
9777     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
9778          ++Value) {
9779       EVT VT = ValueVTs[Value];
9780       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
9781       SDValue Op = SDValue(Args[i].Node.getNode(),
9782                            Args[i].Node.getResNo() + Value);
9783       ISD::ArgFlagsTy Flags;
9784 
9785       // Certain targets (such as MIPS), may have a different ABI alignment
9786       // for a type depending on the context. Give the target a chance to
9787       // specify the alignment it wants.
9788       const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
9789       Flags.setOrigAlign(OriginalAlignment);
9790 
9791       if (Args[i].Ty->isPointerTy()) {
9792         Flags.setPointer();
9793         Flags.setPointerAddrSpace(
9794             cast<PointerType>(Args[i].Ty)->getAddressSpace());
9795       }
9796       if (Args[i].IsZExt)
9797         Flags.setZExt();
9798       if (Args[i].IsSExt)
9799         Flags.setSExt();
9800       if (Args[i].IsInReg) {
9801         // If we are using vectorcall calling convention, a structure that is
9802         // passed InReg - is surely an HVA
9803         if (CLI.CallConv == CallingConv::X86_VectorCall &&
9804             isa<StructType>(FinalType)) {
9805           // The first value of a structure is marked
9806           if (0 == Value)
9807             Flags.setHvaStart();
9808           Flags.setHva();
9809         }
9810         // Set InReg Flag
9811         Flags.setInReg();
9812       }
9813       if (Args[i].IsSRet)
9814         Flags.setSRet();
9815       if (Args[i].IsSwiftSelf)
9816         Flags.setSwiftSelf();
9817       if (Args[i].IsSwiftAsync)
9818         Flags.setSwiftAsync();
9819       if (Args[i].IsSwiftError)
9820         Flags.setSwiftError();
9821       if (Args[i].IsCFGuardTarget)
9822         Flags.setCFGuardTarget();
9823       if (Args[i].IsByVal)
9824         Flags.setByVal();
9825       if (Args[i].IsByRef)
9826         Flags.setByRef();
9827       if (Args[i].IsPreallocated) {
9828         Flags.setPreallocated();
9829         // Set the byval flag for CCAssignFn callbacks that don't know about
9830         // preallocated.  This way we can know how many bytes we should've
9831         // allocated and how many bytes a callee cleanup function will pop.  If
9832         // we port preallocated to more targets, we'll have to add custom
9833         // preallocated handling in the various CC lowering callbacks.
9834         Flags.setByVal();
9835       }
9836       if (Args[i].IsInAlloca) {
9837         Flags.setInAlloca();
9838         // Set the byval flag for CCAssignFn callbacks that don't know about
9839         // inalloca.  This way we can know how many bytes we should've allocated
9840         // and how many bytes a callee cleanup function will pop.  If we port
9841         // inalloca to more targets, we'll have to add custom inalloca handling
9842         // in the various CC lowering callbacks.
9843         Flags.setByVal();
9844       }
9845       Align MemAlign;
9846       if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
9847         unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
9848         Flags.setByValSize(FrameSize);
9849 
9850         // info is not there but there are cases it cannot get right.
9851         if (auto MA = Args[i].Alignment)
9852           MemAlign = *MA;
9853         else
9854           MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
9855       } else if (auto MA = Args[i].Alignment) {
9856         MemAlign = *MA;
9857       } else {
9858         MemAlign = OriginalAlignment;
9859       }
9860       Flags.setMemAlign(MemAlign);
9861       if (Args[i].IsNest)
9862         Flags.setNest();
9863       if (NeedsRegBlock)
9864         Flags.setInConsecutiveRegs();
9865 
9866       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9867                                                  CLI.CallConv, VT);
9868       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9869                                                         CLI.CallConv, VT);
9870       SmallVector<SDValue, 4> Parts(NumParts);
9871       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
9872 
9873       if (Args[i].IsSExt)
9874         ExtendKind = ISD::SIGN_EXTEND;
9875       else if (Args[i].IsZExt)
9876         ExtendKind = ISD::ZERO_EXTEND;
9877 
9878       // Conservatively only handle 'returned' on non-vectors that can be lowered,
9879       // for now.
9880       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
9881           CanLowerReturn) {
9882         assert((CLI.RetTy == Args[i].Ty ||
9883                 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
9884                  CLI.RetTy->getPointerAddressSpace() ==
9885                      Args[i].Ty->getPointerAddressSpace())) &&
9886                RetTys.size() == NumValues && "unexpected use of 'returned'");
9887         // Before passing 'returned' to the target lowering code, ensure that
9888         // either the register MVT and the actual EVT are the same size or that
9889         // the return value and argument are extended in the same way; in these
9890         // cases it's safe to pass the argument register value unchanged as the
9891         // return register value (although it's at the target's option whether
9892         // to do so)
9893         // TODO: allow code generation to take advantage of partially preserved
9894         // registers rather than clobbering the entire register when the
9895         // parameter extension method is not compatible with the return
9896         // extension method
9897         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
9898             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
9899              CLI.RetZExt == Args[i].IsZExt))
9900           Flags.setReturned();
9901       }
9902 
9903       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
9904                      CLI.CallConv, ExtendKind);
9905 
9906       for (unsigned j = 0; j != NumParts; ++j) {
9907         // if it isn't first piece, alignment must be 1
9908         // For scalable vectors the scalable part is currently handled
9909         // by individual targets, so we just use the known minimum size here.
9910         ISD::OutputArg MyFlags(
9911             Flags, Parts[j].getValueType().getSimpleVT(), VT,
9912             i < CLI.NumFixedArgs, i,
9913             j * Parts[j].getValueType().getStoreSize().getKnownMinSize());
9914         if (NumParts > 1 && j == 0)
9915           MyFlags.Flags.setSplit();
9916         else if (j != 0) {
9917           MyFlags.Flags.setOrigAlign(Align(1));
9918           if (j == NumParts - 1)
9919             MyFlags.Flags.setSplitEnd();
9920         }
9921 
9922         CLI.Outs.push_back(MyFlags);
9923         CLI.OutVals.push_back(Parts[j]);
9924       }
9925 
9926       if (NeedsRegBlock && Value == NumValues - 1)
9927         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
9928     }
9929   }
9930 
9931   SmallVector<SDValue, 4> InVals;
9932   CLI.Chain = LowerCall(CLI, InVals);
9933 
9934   // Update CLI.InVals to use outside of this function.
9935   CLI.InVals = InVals;
9936 
9937   // Verify that the target's LowerCall behaved as expected.
9938   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
9939          "LowerCall didn't return a valid chain!");
9940   assert((!CLI.IsTailCall || InVals.empty()) &&
9941          "LowerCall emitted a return value for a tail call!");
9942   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
9943          "LowerCall didn't emit the correct number of values!");
9944 
9945   // For a tail call, the return value is merely live-out and there aren't
9946   // any nodes in the DAG representing it. Return a special value to
9947   // indicate that a tail call has been emitted and no more Instructions
9948   // should be processed in the current block.
9949   if (CLI.IsTailCall) {
9950     CLI.DAG.setRoot(CLI.Chain);
9951     return std::make_pair(SDValue(), SDValue());
9952   }
9953 
9954 #ifndef NDEBUG
9955   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
9956     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
9957     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
9958            "LowerCall emitted a value with the wrong type!");
9959   }
9960 #endif
9961 
9962   SmallVector<SDValue, 4> ReturnValues;
9963   if (!CanLowerReturn) {
9964     // The instruction result is the result of loading from the
9965     // hidden sret parameter.
9966     SmallVector<EVT, 1> PVTs;
9967     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
9968 
9969     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
9970     assert(PVTs.size() == 1 && "Pointers should fit in one register");
9971     EVT PtrVT = PVTs[0];
9972 
9973     unsigned NumValues = RetTys.size();
9974     ReturnValues.resize(NumValues);
9975     SmallVector<SDValue, 4> Chains(NumValues);
9976 
9977     // An aggregate return value cannot wrap around the address space, so
9978     // offsets to its parts don't wrap either.
9979     SDNodeFlags Flags;
9980     Flags.setNoUnsignedWrap(true);
9981 
9982     MachineFunction &MF = CLI.DAG.getMachineFunction();
9983     Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
9984     for (unsigned i = 0; i < NumValues; ++i) {
9985       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
9986                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
9987                                                         PtrVT), Flags);
9988       SDValue L = CLI.DAG.getLoad(
9989           RetTys[i], CLI.DL, CLI.Chain, Add,
9990           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
9991                                             DemoteStackIdx, Offsets[i]),
9992           HiddenSRetAlign);
9993       ReturnValues[i] = L;
9994       Chains[i] = L.getValue(1);
9995     }
9996 
9997     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
9998   } else {
9999     // Collect the legal value parts into potentially illegal values
10000     // that correspond to the original function's return values.
10001     Optional<ISD::NodeType> AssertOp;
10002     if (CLI.RetSExt)
10003       AssertOp = ISD::AssertSext;
10004     else if (CLI.RetZExt)
10005       AssertOp = ISD::AssertZext;
10006     unsigned CurReg = 0;
10007     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10008       EVT VT = RetTys[I];
10009       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
10010                                                      CLI.CallConv, VT);
10011       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10012                                                        CLI.CallConv, VT);
10013 
10014       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
10015                                               NumRegs, RegisterVT, VT, nullptr,
10016                                               CLI.CallConv, AssertOp));
10017       CurReg += NumRegs;
10018     }
10019 
10020     // For a function returning void, there is no return value. We can't create
10021     // such a node, so we just return a null return value in that case. In
10022     // that case, nothing will actually look at the value.
10023     if (ReturnValues.empty())
10024       return std::make_pair(SDValue(), CLI.Chain);
10025   }
10026 
10027   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10028                                 CLI.DAG.getVTList(RetTys), ReturnValues);
10029   return std::make_pair(Res, CLI.Chain);
10030 }
10031 
10032 /// Places new result values for the node in Results (their number
10033 /// and types must exactly match those of the original return values of
10034 /// the node), or leaves Results empty, which indicates that the node is not
10035 /// to be custom lowered after all.
10036 void TargetLowering::LowerOperationWrapper(SDNode *N,
10037                                            SmallVectorImpl<SDValue> &Results,
10038                                            SelectionDAG &DAG) const {
10039   SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10040 
10041   if (!Res.getNode())
10042     return;
10043 
10044   // If the original node has one result, take the return value from
10045   // LowerOperation as is. It might not be result number 0.
10046   if (N->getNumValues() == 1) {
10047     Results.push_back(Res);
10048     return;
10049   }
10050 
10051   // If the original node has multiple results, then the return node should
10052   // have the same number of results.
10053   assert((N->getNumValues() == Res->getNumValues()) &&
10054       "Lowering returned the wrong number of results!");
10055 
10056   // Places new result values base on N result number.
10057   for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10058     Results.push_back(Res.getValue(I));
10059 }
10060 
10061 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10062   llvm_unreachable("LowerOperation not implemented for this target!");
10063 }
10064 
10065 void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
10066                                                      unsigned Reg,
10067                                                      ISD::NodeType ExtendType) {
10068   SDValue Op = getNonRegisterValue(V);
10069   assert((Op.getOpcode() != ISD::CopyFromReg ||
10070           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10071          "Copy from a reg to the same reg!");
10072   assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10073 
10074   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10075   // If this is an InlineAsm we have to match the registers required, not the
10076   // notional registers required by the type.
10077 
10078   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10079                    None); // This is not an ABI copy.
10080   SDValue Chain = DAG.getEntryNode();
10081 
10082   if (ExtendType == ISD::ANY_EXTEND) {
10083     auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10084     if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10085       ExtendType = PreferredExtendIt->second;
10086   }
10087   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10088   PendingExports.push_back(Chain);
10089 }
10090 
10091 #include "llvm/CodeGen/SelectionDAGISel.h"
10092 
10093 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10094 /// entry block, return true.  This includes arguments used by switches, since
10095 /// the switch may expand into multiple basic blocks.
10096 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10097   // With FastISel active, we may be splitting blocks, so force creation
10098   // of virtual registers for all non-dead arguments.
10099   if (FastISel)
10100     return A->use_empty();
10101 
10102   const BasicBlock &Entry = A->getParent()->front();
10103   for (const User *U : A->users())
10104     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10105       return false;  // Use not in entry block.
10106 
10107   return true;
10108 }
10109 
10110 using ArgCopyElisionMapTy =
10111     DenseMap<const Argument *,
10112              std::pair<const AllocaInst *, const StoreInst *>>;
10113 
10114 /// Scan the entry block of the function in FuncInfo for arguments that look
10115 /// like copies into a local alloca. Record any copied arguments in
10116 /// ArgCopyElisionCandidates.
10117 static void
10118 findArgumentCopyElisionCandidates(const DataLayout &DL,
10119                                   FunctionLoweringInfo *FuncInfo,
10120                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10121   // Record the state of every static alloca used in the entry block. Argument
10122   // allocas are all used in the entry block, so we need approximately as many
10123   // entries as we have arguments.
10124   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10125   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
10126   unsigned NumArgs = FuncInfo->Fn->arg_size();
10127   StaticAllocas.reserve(NumArgs * 2);
10128 
10129   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10130     if (!V)
10131       return nullptr;
10132     V = V->stripPointerCasts();
10133     const auto *AI = dyn_cast<AllocaInst>(V);
10134     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10135       return nullptr;
10136     auto Iter = StaticAllocas.insert({AI, Unknown});
10137     return &Iter.first->second;
10138   };
10139 
10140   // Look for stores of arguments to static allocas. Look through bitcasts and
10141   // GEPs to handle type coercions, as long as the alloca is fully initialized
10142   // by the store. Any non-store use of an alloca escapes it and any subsequent
10143   // unanalyzed store might write it.
10144   // FIXME: Handle structs initialized with multiple stores.
10145   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
10146     // Look for stores, and handle non-store uses conservatively.
10147     const auto *SI = dyn_cast<StoreInst>(&I);
10148     if (!SI) {
10149       // We will look through cast uses, so ignore them completely.
10150       if (I.isCast())
10151         continue;
10152       // Ignore debug info and pseudo op intrinsics, they don't escape or store
10153       // to allocas.
10154       if (I.isDebugOrPseudoInst())
10155         continue;
10156       // This is an unknown instruction. Assume it escapes or writes to all
10157       // static alloca operands.
10158       for (const Use &U : I.operands()) {
10159         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
10160           *Info = StaticAllocaInfo::Clobbered;
10161       }
10162       continue;
10163     }
10164 
10165     // If the stored value is a static alloca, mark it as escaped.
10166     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
10167       *Info = StaticAllocaInfo::Clobbered;
10168 
10169     // Check if the destination is a static alloca.
10170     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
10171     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
10172     if (!Info)
10173       continue;
10174     const AllocaInst *AI = cast<AllocaInst>(Dst);
10175 
10176     // Skip allocas that have been initialized or clobbered.
10177     if (*Info != StaticAllocaInfo::Unknown)
10178       continue;
10179 
10180     // Check if the stored value is an argument, and that this store fully
10181     // initializes the alloca.
10182     // If the argument type has padding bits we can't directly forward a pointer
10183     // as the upper bits may contain garbage.
10184     // Don't elide copies from the same argument twice.
10185     const Value *Val = SI->getValueOperand()->stripPointerCasts();
10186     const auto *Arg = dyn_cast<Argument>(Val);
10187     if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
10188         Arg->getType()->isEmptyTy() ||
10189         DL.getTypeStoreSize(Arg->getType()) !=
10190             DL.getTypeAllocSize(AI->getAllocatedType()) ||
10191         !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
10192         ArgCopyElisionCandidates.count(Arg)) {
10193       *Info = StaticAllocaInfo::Clobbered;
10194       continue;
10195     }
10196 
10197     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
10198                       << '\n');
10199 
10200     // Mark this alloca and store for argument copy elision.
10201     *Info = StaticAllocaInfo::Elidable;
10202     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
10203 
10204     // Stop scanning if we've seen all arguments. This will happen early in -O0
10205     // builds, which is useful, because -O0 builds have large entry blocks and
10206     // many allocas.
10207     if (ArgCopyElisionCandidates.size() == NumArgs)
10208       break;
10209   }
10210 }
10211 
10212 /// Try to elide argument copies from memory into a local alloca. Succeeds if
10213 /// ArgVal is a load from a suitable fixed stack object.
10214 static void tryToElideArgumentCopy(
10215     FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
10216     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
10217     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
10218     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
10219     SDValue ArgVal, bool &ArgHasUses) {
10220   // Check if this is a load from a fixed stack object.
10221   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
10222   if (!LNode)
10223     return;
10224   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
10225   if (!FINode)
10226     return;
10227 
10228   // Check that the fixed stack object is the right size and alignment.
10229   // Look at the alignment that the user wrote on the alloca instead of looking
10230   // at the stack object.
10231   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
10232   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
10233   const AllocaInst *AI = ArgCopyIter->second.first;
10234   int FixedIndex = FINode->getIndex();
10235   int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
10236   int OldIndex = AllocaIndex;
10237   MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
10238   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
10239     LLVM_DEBUG(
10240         dbgs() << "  argument copy elision failed due to bad fixed stack "
10241                   "object size\n");
10242     return;
10243   }
10244   Align RequiredAlignment = AI->getAlign();
10245   if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
10246     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
10247                          "greater than stack argument alignment ("
10248                       << DebugStr(RequiredAlignment) << " vs "
10249                       << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
10250     return;
10251   }
10252 
10253   // Perform the elision. Delete the old stack object and replace its only use
10254   // in the variable info map. Mark the stack object as mutable.
10255   LLVM_DEBUG({
10256     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
10257            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
10258            << '\n';
10259   });
10260   MFI.RemoveStackObject(OldIndex);
10261   MFI.setIsImmutableObjectIndex(FixedIndex, false);
10262   AllocaIndex = FixedIndex;
10263   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
10264   Chains.push_back(ArgVal.getValue(1));
10265 
10266   // Avoid emitting code for the store implementing the copy.
10267   const StoreInst *SI = ArgCopyIter->second.second;
10268   ElidedArgCopyInstrs.insert(SI);
10269 
10270   // Check for uses of the argument again so that we can avoid exporting ArgVal
10271   // if it is't used by anything other than the store.
10272   for (const Value *U : Arg.users()) {
10273     if (U != SI) {
10274       ArgHasUses = true;
10275       break;
10276     }
10277   }
10278 }
10279 
10280 void SelectionDAGISel::LowerArguments(const Function &F) {
10281   SelectionDAG &DAG = SDB->DAG;
10282   SDLoc dl = SDB->getCurSDLoc();
10283   const DataLayout &DL = DAG.getDataLayout();
10284   SmallVector<ISD::InputArg, 16> Ins;
10285 
10286   // In Naked functions we aren't going to save any registers.
10287   if (F.hasFnAttribute(Attribute::Naked))
10288     return;
10289 
10290   if (!FuncInfo->CanLowerReturn) {
10291     // Put in an sret pointer parameter before all the other parameters.
10292     SmallVector<EVT, 1> ValueVTs;
10293     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10294                     F.getReturnType()->getPointerTo(
10295                         DAG.getDataLayout().getAllocaAddrSpace()),
10296                     ValueVTs);
10297 
10298     // NOTE: Assuming that a pointer will never break down to more than one VT
10299     // or one register.
10300     ISD::ArgFlagsTy Flags;
10301     Flags.setSRet();
10302     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
10303     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
10304                          ISD::InputArg::NoArgIndex, 0);
10305     Ins.push_back(RetArg);
10306   }
10307 
10308   // Look for stores of arguments to static allocas. Mark such arguments with a
10309   // flag to ask the target to give us the memory location of that argument if
10310   // available.
10311   ArgCopyElisionMapTy ArgCopyElisionCandidates;
10312   findArgumentCopyElisionCandidates(DL, FuncInfo.get(),
10313                                     ArgCopyElisionCandidates);
10314 
10315   // Set up the incoming argument description vector.
10316   for (const Argument &Arg : F.args()) {
10317     unsigned ArgNo = Arg.getArgNo();
10318     SmallVector<EVT, 4> ValueVTs;
10319     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10320     bool isArgValueUsed = !Arg.use_empty();
10321     unsigned PartBase = 0;
10322     Type *FinalType = Arg.getType();
10323     if (Arg.hasAttribute(Attribute::ByVal))
10324       FinalType = Arg.getParamByValType();
10325     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
10326         FinalType, F.getCallingConv(), F.isVarArg(), DL);
10327     for (unsigned Value = 0, NumValues = ValueVTs.size();
10328          Value != NumValues; ++Value) {
10329       EVT VT = ValueVTs[Value];
10330       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
10331       ISD::ArgFlagsTy Flags;
10332 
10333 
10334       if (Arg.getType()->isPointerTy()) {
10335         Flags.setPointer();
10336         Flags.setPointerAddrSpace(
10337             cast<PointerType>(Arg.getType())->getAddressSpace());
10338       }
10339       if (Arg.hasAttribute(Attribute::ZExt))
10340         Flags.setZExt();
10341       if (Arg.hasAttribute(Attribute::SExt))
10342         Flags.setSExt();
10343       if (Arg.hasAttribute(Attribute::InReg)) {
10344         // If we are using vectorcall calling convention, a structure that is
10345         // passed InReg - is surely an HVA
10346         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
10347             isa<StructType>(Arg.getType())) {
10348           // The first value of a structure is marked
10349           if (0 == Value)
10350             Flags.setHvaStart();
10351           Flags.setHva();
10352         }
10353         // Set InReg Flag
10354         Flags.setInReg();
10355       }
10356       if (Arg.hasAttribute(Attribute::StructRet))
10357         Flags.setSRet();
10358       if (Arg.hasAttribute(Attribute::SwiftSelf))
10359         Flags.setSwiftSelf();
10360       if (Arg.hasAttribute(Attribute::SwiftAsync))
10361         Flags.setSwiftAsync();
10362       if (Arg.hasAttribute(Attribute::SwiftError))
10363         Flags.setSwiftError();
10364       if (Arg.hasAttribute(Attribute::ByVal))
10365         Flags.setByVal();
10366       if (Arg.hasAttribute(Attribute::ByRef))
10367         Flags.setByRef();
10368       if (Arg.hasAttribute(Attribute::InAlloca)) {
10369         Flags.setInAlloca();
10370         // Set the byval flag for CCAssignFn callbacks that don't know about
10371         // inalloca.  This way we can know how many bytes we should've allocated
10372         // and how many bytes a callee cleanup function will pop.  If we port
10373         // inalloca to more targets, we'll have to add custom inalloca handling
10374         // in the various CC lowering callbacks.
10375         Flags.setByVal();
10376       }
10377       if (Arg.hasAttribute(Attribute::Preallocated)) {
10378         Flags.setPreallocated();
10379         // Set the byval flag for CCAssignFn callbacks that don't know about
10380         // preallocated.  This way we can know how many bytes we should've
10381         // allocated and how many bytes a callee cleanup function will pop.  If
10382         // we port preallocated to more targets, we'll have to add custom
10383         // preallocated handling in the various CC lowering callbacks.
10384         Flags.setByVal();
10385       }
10386 
10387       // Certain targets (such as MIPS), may have a different ABI alignment
10388       // for a type depending on the context. Give the target a chance to
10389       // specify the alignment it wants.
10390       const Align OriginalAlignment(
10391           TLI->getABIAlignmentForCallingConv(ArgTy, DL));
10392       Flags.setOrigAlign(OriginalAlignment);
10393 
10394       Align MemAlign;
10395       Type *ArgMemTy = nullptr;
10396       if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
10397           Flags.isByRef()) {
10398         if (!ArgMemTy)
10399           ArgMemTy = Arg.getPointeeInMemoryValueType();
10400 
10401         uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
10402 
10403         // For in-memory arguments, size and alignment should be passed from FE.
10404         // BE will guess if this info is not there but there are cases it cannot
10405         // get right.
10406         if (auto ParamAlign = Arg.getParamStackAlign())
10407           MemAlign = *ParamAlign;
10408         else if ((ParamAlign = Arg.getParamAlign()))
10409           MemAlign = *ParamAlign;
10410         else
10411           MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
10412         if (Flags.isByRef())
10413           Flags.setByRefSize(MemSize);
10414         else
10415           Flags.setByValSize(MemSize);
10416       } else if (auto ParamAlign = Arg.getParamStackAlign()) {
10417         MemAlign = *ParamAlign;
10418       } else {
10419         MemAlign = OriginalAlignment;
10420       }
10421       Flags.setMemAlign(MemAlign);
10422 
10423       if (Arg.hasAttribute(Attribute::Nest))
10424         Flags.setNest();
10425       if (NeedsRegBlock)
10426         Flags.setInConsecutiveRegs();
10427       if (ArgCopyElisionCandidates.count(&Arg))
10428         Flags.setCopyElisionCandidate();
10429       if (Arg.hasAttribute(Attribute::Returned))
10430         Flags.setReturned();
10431 
10432       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
10433           *CurDAG->getContext(), F.getCallingConv(), VT);
10434       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
10435           *CurDAG->getContext(), F.getCallingConv(), VT);
10436       for (unsigned i = 0; i != NumRegs; ++i) {
10437         // For scalable vectors, use the minimum size; individual targets
10438         // are responsible for handling scalable vector arguments and
10439         // return values.
10440         ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
10441                  ArgNo, PartBase+i*RegisterVT.getStoreSize().getKnownMinSize());
10442         if (NumRegs > 1 && i == 0)
10443           MyFlags.Flags.setSplit();
10444         // if it isn't first piece, alignment must be 1
10445         else if (i > 0) {
10446           MyFlags.Flags.setOrigAlign(Align(1));
10447           if (i == NumRegs - 1)
10448             MyFlags.Flags.setSplitEnd();
10449         }
10450         Ins.push_back(MyFlags);
10451       }
10452       if (NeedsRegBlock && Value == NumValues - 1)
10453         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
10454       PartBase += VT.getStoreSize().getKnownMinSize();
10455     }
10456   }
10457 
10458   // Call the target to set up the argument values.
10459   SmallVector<SDValue, 8> InVals;
10460   SDValue NewRoot = TLI->LowerFormalArguments(
10461       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
10462 
10463   // Verify that the target's LowerFormalArguments behaved as expected.
10464   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
10465          "LowerFormalArguments didn't return a valid chain!");
10466   assert(InVals.size() == Ins.size() &&
10467          "LowerFormalArguments didn't emit the correct number of values!");
10468   LLVM_DEBUG({
10469     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10470       assert(InVals[i].getNode() &&
10471              "LowerFormalArguments emitted a null value!");
10472       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
10473              "LowerFormalArguments emitted a value with the wrong type!");
10474     }
10475   });
10476 
10477   // Update the DAG with the new chain value resulting from argument lowering.
10478   DAG.setRoot(NewRoot);
10479 
10480   // Set up the argument values.
10481   unsigned i = 0;
10482   if (!FuncInfo->CanLowerReturn) {
10483     // Create a virtual register for the sret pointer, and put in a copy
10484     // from the sret argument into it.
10485     SmallVector<EVT, 1> ValueVTs;
10486     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10487                     F.getReturnType()->getPointerTo(
10488                         DAG.getDataLayout().getAllocaAddrSpace()),
10489                     ValueVTs);
10490     MVT VT = ValueVTs[0].getSimpleVT();
10491     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
10492     Optional<ISD::NodeType> AssertOp = None;
10493     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
10494                                         nullptr, F.getCallingConv(), AssertOp);
10495 
10496     MachineFunction& MF = SDB->DAG.getMachineFunction();
10497     MachineRegisterInfo& RegInfo = MF.getRegInfo();
10498     Register SRetReg =
10499         RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
10500     FuncInfo->DemoteRegister = SRetReg;
10501     NewRoot =
10502         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
10503     DAG.setRoot(NewRoot);
10504 
10505     // i indexes lowered arguments.  Bump it past the hidden sret argument.
10506     ++i;
10507   }
10508 
10509   SmallVector<SDValue, 4> Chains;
10510   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
10511   for (const Argument &Arg : F.args()) {
10512     SmallVector<SDValue, 4> ArgValues;
10513     SmallVector<EVT, 4> ValueVTs;
10514     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10515     unsigned NumValues = ValueVTs.size();
10516     if (NumValues == 0)
10517       continue;
10518 
10519     bool ArgHasUses = !Arg.use_empty();
10520 
10521     // Elide the copying store if the target loaded this argument from a
10522     // suitable fixed stack object.
10523     if (Ins[i].Flags.isCopyElisionCandidate()) {
10524       tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
10525                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
10526                              InVals[i], ArgHasUses);
10527     }
10528 
10529     // If this argument is unused then remember its value. It is used to generate
10530     // debugging information.
10531     bool isSwiftErrorArg =
10532         TLI->supportSwiftError() &&
10533         Arg.hasAttribute(Attribute::SwiftError);
10534     if (!ArgHasUses && !isSwiftErrorArg) {
10535       SDB->setUnusedArgValue(&Arg, InVals[i]);
10536 
10537       // Also remember any frame index for use in FastISel.
10538       if (FrameIndexSDNode *FI =
10539           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
10540         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10541     }
10542 
10543     for (unsigned Val = 0; Val != NumValues; ++Val) {
10544       EVT VT = ValueVTs[Val];
10545       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
10546                                                       F.getCallingConv(), VT);
10547       unsigned NumParts = TLI->getNumRegistersForCallingConv(
10548           *CurDAG->getContext(), F.getCallingConv(), VT);
10549 
10550       // Even an apparent 'unused' swifterror argument needs to be returned. So
10551       // we do generate a copy for it that can be used on return from the
10552       // function.
10553       if (ArgHasUses || isSwiftErrorArg) {
10554         Optional<ISD::NodeType> AssertOp;
10555         if (Arg.hasAttribute(Attribute::SExt))
10556           AssertOp = ISD::AssertSext;
10557         else if (Arg.hasAttribute(Attribute::ZExt))
10558           AssertOp = ISD::AssertZext;
10559 
10560         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
10561                                              PartVT, VT, nullptr,
10562                                              F.getCallingConv(), AssertOp));
10563       }
10564 
10565       i += NumParts;
10566     }
10567 
10568     // We don't need to do anything else for unused arguments.
10569     if (ArgValues.empty())
10570       continue;
10571 
10572     // Note down frame index.
10573     if (FrameIndexSDNode *FI =
10574         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
10575       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10576 
10577     SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
10578                                      SDB->getCurSDLoc());
10579 
10580     SDB->setValue(&Arg, Res);
10581     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
10582       // We want to associate the argument with the frame index, among
10583       // involved operands, that correspond to the lowest address. The
10584       // getCopyFromParts function, called earlier, is swapping the order of
10585       // the operands to BUILD_PAIR depending on endianness. The result of
10586       // that swapping is that the least significant bits of the argument will
10587       // be in the first operand of the BUILD_PAIR node, and the most
10588       // significant bits will be in the second operand.
10589       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
10590       if (LoadSDNode *LNode =
10591           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
10592         if (FrameIndexSDNode *FI =
10593             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
10594           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10595     }
10596 
10597     // Analyses past this point are naive and don't expect an assertion.
10598     if (Res.getOpcode() == ISD::AssertZext)
10599       Res = Res.getOperand(0);
10600 
10601     // Update the SwiftErrorVRegDefMap.
10602     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
10603       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10604       if (Register::isVirtualRegister(Reg))
10605         SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
10606                                    Reg);
10607     }
10608 
10609     // If this argument is live outside of the entry block, insert a copy from
10610     // wherever we got it to the vreg that other BB's will reference it as.
10611     if (Res.getOpcode() == ISD::CopyFromReg) {
10612       // If we can, though, try to skip creating an unnecessary vreg.
10613       // FIXME: This isn't very clean... it would be nice to make this more
10614       // general.
10615       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10616       if (Register::isVirtualRegister(Reg)) {
10617         FuncInfo->ValueMap[&Arg] = Reg;
10618         continue;
10619       }
10620     }
10621     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
10622       FuncInfo->InitializeRegForValue(&Arg);
10623       SDB->CopyToExportRegsIfNeeded(&Arg);
10624     }
10625   }
10626 
10627   if (!Chains.empty()) {
10628     Chains.push_back(NewRoot);
10629     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
10630   }
10631 
10632   DAG.setRoot(NewRoot);
10633 
10634   assert(i == InVals.size() && "Argument register count mismatch!");
10635 
10636   // If any argument copy elisions occurred and we have debug info, update the
10637   // stale frame indices used in the dbg.declare variable info table.
10638   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
10639   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
10640     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
10641       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
10642       if (I != ArgCopyElisionFrameIndexMap.end())
10643         VI.Slot = I->second;
10644     }
10645   }
10646 
10647   // Finally, if the target has anything special to do, allow it to do so.
10648   emitFunctionEntryCode();
10649 }
10650 
10651 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
10652 /// ensure constants are generated when needed.  Remember the virtual registers
10653 /// that need to be added to the Machine PHI nodes as input.  We cannot just
10654 /// directly add them, because expansion might result in multiple MBB's for one
10655 /// BB.  As such, the start of the BB might correspond to a different MBB than
10656 /// the end.
10657 void
10658 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
10659   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10660   const Instruction *TI = LLVMBB->getTerminator();
10661 
10662   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
10663 
10664   // Check PHI nodes in successors that expect a value to be available from this
10665   // block.
10666   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
10667     const BasicBlock *SuccBB = TI->getSuccessor(succ);
10668     if (!isa<PHINode>(SuccBB->begin())) continue;
10669     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
10670 
10671     // If this terminator has multiple identical successors (common for
10672     // switches), only handle each succ once.
10673     if (!SuccsHandled.insert(SuccMBB).second)
10674       continue;
10675 
10676     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
10677 
10678     // At this point we know that there is a 1-1 correspondence between LLVM PHI
10679     // nodes and Machine PHI nodes, but the incoming operands have not been
10680     // emitted yet.
10681     for (const PHINode &PN : SuccBB->phis()) {
10682       // Ignore dead phi's.
10683       if (PN.use_empty())
10684         continue;
10685 
10686       // Skip empty types
10687       if (PN.getType()->isEmptyTy())
10688         continue;
10689 
10690       unsigned Reg;
10691       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
10692 
10693       if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
10694         unsigned &RegOut = ConstantsOut[C];
10695         if (RegOut == 0) {
10696           RegOut = FuncInfo.CreateRegs(C);
10697           // We need to zero/sign extend ConstantInt phi operands to match
10698           // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
10699           ISD::NodeType ExtendType = ISD::ANY_EXTEND;
10700           if (auto *CI = dyn_cast<ConstantInt>(C))
10701             ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
10702                                                     : ISD::ZERO_EXTEND;
10703           CopyValueToVirtualRegister(C, RegOut, ExtendType);
10704         }
10705         Reg = RegOut;
10706       } else {
10707         DenseMap<const Value *, Register>::iterator I =
10708           FuncInfo.ValueMap.find(PHIOp);
10709         if (I != FuncInfo.ValueMap.end())
10710           Reg = I->second;
10711         else {
10712           assert(isa<AllocaInst>(PHIOp) &&
10713                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
10714                  "Didn't codegen value into a register!??");
10715           Reg = FuncInfo.CreateRegs(PHIOp);
10716           CopyValueToVirtualRegister(PHIOp, Reg);
10717         }
10718       }
10719 
10720       // Remember that this register needs to added to the machine PHI node as
10721       // the input for this MBB.
10722       SmallVector<EVT, 4> ValueVTs;
10723       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
10724       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
10725         EVT VT = ValueVTs[vti];
10726         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
10727         for (unsigned i = 0, e = NumRegisters; i != e; ++i)
10728           FuncInfo.PHINodesToUpdate.push_back(
10729               std::make_pair(&*MBBI++, Reg + i));
10730         Reg += NumRegisters;
10731       }
10732     }
10733   }
10734 
10735   ConstantsOut.clear();
10736 }
10737 
10738 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
10739   MachineFunction::iterator I(MBB);
10740   if (++I == FuncInfo.MF->end())
10741     return nullptr;
10742   return &*I;
10743 }
10744 
10745 /// During lowering new call nodes can be created (such as memset, etc.).
10746 /// Those will become new roots of the current DAG, but complications arise
10747 /// when they are tail calls. In such cases, the call lowering will update
10748 /// the root, but the builder still needs to know that a tail call has been
10749 /// lowered in order to avoid generating an additional return.
10750 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
10751   // If the node is null, we do have a tail call.
10752   if (MaybeTC.getNode() != nullptr)
10753     DAG.setRoot(MaybeTC);
10754   else
10755     HasTailCall = true;
10756 }
10757 
10758 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
10759                                         MachineBasicBlock *SwitchMBB,
10760                                         MachineBasicBlock *DefaultMBB) {
10761   MachineFunction *CurMF = FuncInfo.MF;
10762   MachineBasicBlock *NextMBB = nullptr;
10763   MachineFunction::iterator BBI(W.MBB);
10764   if (++BBI != FuncInfo.MF->end())
10765     NextMBB = &*BBI;
10766 
10767   unsigned Size = W.LastCluster - W.FirstCluster + 1;
10768 
10769   BranchProbabilityInfo *BPI = FuncInfo.BPI;
10770 
10771   if (Size == 2 && W.MBB == SwitchMBB) {
10772     // If any two of the cases has the same destination, and if one value
10773     // is the same as the other, but has one bit unset that the other has set,
10774     // use bit manipulation to do two compares at once.  For example:
10775     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
10776     // TODO: This could be extended to merge any 2 cases in switches with 3
10777     // cases.
10778     // TODO: Handle cases where W.CaseBB != SwitchBB.
10779     CaseCluster &Small = *W.FirstCluster;
10780     CaseCluster &Big = *W.LastCluster;
10781 
10782     if (Small.Low == Small.High && Big.Low == Big.High &&
10783         Small.MBB == Big.MBB) {
10784       const APInt &SmallValue = Small.Low->getValue();
10785       const APInt &BigValue = Big.Low->getValue();
10786 
10787       // Check that there is only one bit different.
10788       APInt CommonBit = BigValue ^ SmallValue;
10789       if (CommonBit.isPowerOf2()) {
10790         SDValue CondLHS = getValue(Cond);
10791         EVT VT = CondLHS.getValueType();
10792         SDLoc DL = getCurSDLoc();
10793 
10794         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
10795                                  DAG.getConstant(CommonBit, DL, VT));
10796         SDValue Cond = DAG.getSetCC(
10797             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
10798             ISD::SETEQ);
10799 
10800         // Update successor info.
10801         // Both Small and Big will jump to Small.BB, so we sum up the
10802         // probabilities.
10803         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
10804         if (BPI)
10805           addSuccessorWithProb(
10806               SwitchMBB, DefaultMBB,
10807               // The default destination is the first successor in IR.
10808               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
10809         else
10810           addSuccessorWithProb(SwitchMBB, DefaultMBB);
10811 
10812         // Insert the true branch.
10813         SDValue BrCond =
10814             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
10815                         DAG.getBasicBlock(Small.MBB));
10816         // Insert the false branch.
10817         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
10818                              DAG.getBasicBlock(DefaultMBB));
10819 
10820         DAG.setRoot(BrCond);
10821         return;
10822       }
10823     }
10824   }
10825 
10826   if (TM.getOptLevel() != CodeGenOpt::None) {
10827     // Here, we order cases by probability so the most likely case will be
10828     // checked first. However, two clusters can have the same probability in
10829     // which case their relative ordering is non-deterministic. So we use Low
10830     // as a tie-breaker as clusters are guaranteed to never overlap.
10831     llvm::sort(W.FirstCluster, W.LastCluster + 1,
10832                [](const CaseCluster &a, const CaseCluster &b) {
10833       return a.Prob != b.Prob ?
10834              a.Prob > b.Prob :
10835              a.Low->getValue().slt(b.Low->getValue());
10836     });
10837 
10838     // Rearrange the case blocks so that the last one falls through if possible
10839     // without changing the order of probabilities.
10840     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
10841       --I;
10842       if (I->Prob > W.LastCluster->Prob)
10843         break;
10844       if (I->Kind == CC_Range && I->MBB == NextMBB) {
10845         std::swap(*I, *W.LastCluster);
10846         break;
10847       }
10848     }
10849   }
10850 
10851   // Compute total probability.
10852   BranchProbability DefaultProb = W.DefaultProb;
10853   BranchProbability UnhandledProbs = DefaultProb;
10854   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
10855     UnhandledProbs += I->Prob;
10856 
10857   MachineBasicBlock *CurMBB = W.MBB;
10858   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
10859     bool FallthroughUnreachable = false;
10860     MachineBasicBlock *Fallthrough;
10861     if (I == W.LastCluster) {
10862       // For the last cluster, fall through to the default destination.
10863       Fallthrough = DefaultMBB;
10864       FallthroughUnreachable = isa<UnreachableInst>(
10865           DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
10866     } else {
10867       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
10868       CurMF->insert(BBI, Fallthrough);
10869       // Put Cond in a virtual register to make it available from the new blocks.
10870       ExportFromCurrentBlock(Cond);
10871     }
10872     UnhandledProbs -= I->Prob;
10873 
10874     switch (I->Kind) {
10875       case CC_JumpTable: {
10876         // FIXME: Optimize away range check based on pivot comparisons.
10877         JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
10878         SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
10879 
10880         // The jump block hasn't been inserted yet; insert it here.
10881         MachineBasicBlock *JumpMBB = JT->MBB;
10882         CurMF->insert(BBI, JumpMBB);
10883 
10884         auto JumpProb = I->Prob;
10885         auto FallthroughProb = UnhandledProbs;
10886 
10887         // If the default statement is a target of the jump table, we evenly
10888         // distribute the default probability to successors of CurMBB. Also
10889         // update the probability on the edge from JumpMBB to Fallthrough.
10890         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
10891                                               SE = JumpMBB->succ_end();
10892              SI != SE; ++SI) {
10893           if (*SI == DefaultMBB) {
10894             JumpProb += DefaultProb / 2;
10895             FallthroughProb -= DefaultProb / 2;
10896             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
10897             JumpMBB->normalizeSuccProbs();
10898             break;
10899           }
10900         }
10901 
10902         if (FallthroughUnreachable)
10903           JTH->FallthroughUnreachable = true;
10904 
10905         if (!JTH->FallthroughUnreachable)
10906           addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
10907         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
10908         CurMBB->normalizeSuccProbs();
10909 
10910         // The jump table header will be inserted in our current block, do the
10911         // range check, and fall through to our fallthrough block.
10912         JTH->HeaderBB = CurMBB;
10913         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
10914 
10915         // If we're in the right place, emit the jump table header right now.
10916         if (CurMBB == SwitchMBB) {
10917           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
10918           JTH->Emitted = true;
10919         }
10920         break;
10921       }
10922       case CC_BitTests: {
10923         // FIXME: Optimize away range check based on pivot comparisons.
10924         BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
10925 
10926         // The bit test blocks haven't been inserted yet; insert them here.
10927         for (BitTestCase &BTC : BTB->Cases)
10928           CurMF->insert(BBI, BTC.ThisBB);
10929 
10930         // Fill in fields of the BitTestBlock.
10931         BTB->Parent = CurMBB;
10932         BTB->Default = Fallthrough;
10933 
10934         BTB->DefaultProb = UnhandledProbs;
10935         // If the cases in bit test don't form a contiguous range, we evenly
10936         // distribute the probability on the edge to Fallthrough to two
10937         // successors of CurMBB.
10938         if (!BTB->ContiguousRange) {
10939           BTB->Prob += DefaultProb / 2;
10940           BTB->DefaultProb -= DefaultProb / 2;
10941         }
10942 
10943         if (FallthroughUnreachable)
10944           BTB->FallthroughUnreachable = true;
10945 
10946         // If we're in the right place, emit the bit test header right now.
10947         if (CurMBB == SwitchMBB) {
10948           visitBitTestHeader(*BTB, SwitchMBB);
10949           BTB->Emitted = true;
10950         }
10951         break;
10952       }
10953       case CC_Range: {
10954         const Value *RHS, *LHS, *MHS;
10955         ISD::CondCode CC;
10956         if (I->Low == I->High) {
10957           // Check Cond == I->Low.
10958           CC = ISD::SETEQ;
10959           LHS = Cond;
10960           RHS=I->Low;
10961           MHS = nullptr;
10962         } else {
10963           // Check I->Low <= Cond <= I->High.
10964           CC = ISD::SETLE;
10965           LHS = I->Low;
10966           MHS = Cond;
10967           RHS = I->High;
10968         }
10969 
10970         // If Fallthrough is unreachable, fold away the comparison.
10971         if (FallthroughUnreachable)
10972           CC = ISD::SETTRUE;
10973 
10974         // The false probability is the sum of all unhandled cases.
10975         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
10976                      getCurSDLoc(), I->Prob, UnhandledProbs);
10977 
10978         if (CurMBB == SwitchMBB)
10979           visitSwitchCase(CB, SwitchMBB);
10980         else
10981           SL->SwitchCases.push_back(CB);
10982 
10983         break;
10984       }
10985     }
10986     CurMBB = Fallthrough;
10987   }
10988 }
10989 
10990 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
10991                                               CaseClusterIt First,
10992                                               CaseClusterIt Last) {
10993   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
10994     if (X.Prob != CC.Prob)
10995       return X.Prob > CC.Prob;
10996 
10997     // Ties are broken by comparing the case value.
10998     return X.Low->getValue().slt(CC.Low->getValue());
10999   });
11000 }
11001 
11002 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11003                                         const SwitchWorkListItem &W,
11004                                         Value *Cond,
11005                                         MachineBasicBlock *SwitchMBB) {
11006   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11007          "Clusters not sorted?");
11008 
11009   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11010 
11011   // Balance the tree based on branch probabilities to create a near-optimal (in
11012   // terms of search time given key frequency) binary search tree. See e.g. Kurt
11013   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
11014   CaseClusterIt LastLeft = W.FirstCluster;
11015   CaseClusterIt FirstRight = W.LastCluster;
11016   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
11017   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
11018 
11019   // Move LastLeft and FirstRight towards each other from opposite directions to
11020   // find a partitioning of the clusters which balances the probability on both
11021   // sides. If LeftProb and RightProb are equal, alternate which side is
11022   // taken to ensure 0-probability nodes are distributed evenly.
11023   unsigned I = 0;
11024   while (LastLeft + 1 < FirstRight) {
11025     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
11026       LeftProb += (++LastLeft)->Prob;
11027     else
11028       RightProb += (--FirstRight)->Prob;
11029     I++;
11030   }
11031 
11032   while (true) {
11033     // Our binary search tree differs from a typical BST in that ours can have up
11034     // to three values in each leaf. The pivot selection above doesn't take that
11035     // into account, which means the tree might require more nodes and be less
11036     // efficient. We compensate for this here.
11037 
11038     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
11039     unsigned NumRight = W.LastCluster - FirstRight + 1;
11040 
11041     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
11042       // If one side has less than 3 clusters, and the other has more than 3,
11043       // consider taking a cluster from the other side.
11044 
11045       if (NumLeft < NumRight) {
11046         // Consider moving the first cluster on the right to the left side.
11047         CaseCluster &CC = *FirstRight;
11048         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11049         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11050         if (LeftSideRank <= RightSideRank) {
11051           // Moving the cluster to the left does not demote it.
11052           ++LastLeft;
11053           ++FirstRight;
11054           continue;
11055         }
11056       } else {
11057         assert(NumRight < NumLeft);
11058         // Consider moving the last element on the left to the right side.
11059         CaseCluster &CC = *LastLeft;
11060         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11061         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11062         if (RightSideRank <= LeftSideRank) {
11063           // Moving the cluster to the right does not demot it.
11064           --LastLeft;
11065           --FirstRight;
11066           continue;
11067         }
11068       }
11069     }
11070     break;
11071   }
11072 
11073   assert(LastLeft + 1 == FirstRight);
11074   assert(LastLeft >= W.FirstCluster);
11075   assert(FirstRight <= W.LastCluster);
11076 
11077   // Use the first element on the right as pivot since we will make less-than
11078   // comparisons against it.
11079   CaseClusterIt PivotCluster = FirstRight;
11080   assert(PivotCluster > W.FirstCluster);
11081   assert(PivotCluster <= W.LastCluster);
11082 
11083   CaseClusterIt FirstLeft = W.FirstCluster;
11084   CaseClusterIt LastRight = W.LastCluster;
11085 
11086   const ConstantInt *Pivot = PivotCluster->Low;
11087 
11088   // New blocks will be inserted immediately after the current one.
11089   MachineFunction::iterator BBI(W.MBB);
11090   ++BBI;
11091 
11092   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11093   // we can branch to its destination directly if it's squeezed exactly in
11094   // between the known lower bound and Pivot - 1.
11095   MachineBasicBlock *LeftMBB;
11096   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11097       FirstLeft->Low == W.GE &&
11098       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11099     LeftMBB = FirstLeft->MBB;
11100   } else {
11101     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11102     FuncInfo.MF->insert(BBI, LeftMBB);
11103     WorkList.push_back(
11104         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11105     // Put Cond in a virtual register to make it available from the new blocks.
11106     ExportFromCurrentBlock(Cond);
11107   }
11108 
11109   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11110   // single cluster, RHS.Low == Pivot, and we can branch to its destination
11111   // directly if RHS.High equals the current upper bound.
11112   MachineBasicBlock *RightMBB;
11113   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11114       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11115     RightMBB = FirstRight->MBB;
11116   } else {
11117     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11118     FuncInfo.MF->insert(BBI, RightMBB);
11119     WorkList.push_back(
11120         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11121     // Put Cond in a virtual register to make it available from the new blocks.
11122     ExportFromCurrentBlock(Cond);
11123   }
11124 
11125   // Create the CaseBlock record that will be used to lower the branch.
11126   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11127                getCurSDLoc(), LeftProb, RightProb);
11128 
11129   if (W.MBB == SwitchMBB)
11130     visitSwitchCase(CB, SwitchMBB);
11131   else
11132     SL->SwitchCases.push_back(CB);
11133 }
11134 
11135 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11136 // from the swith statement.
11137 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
11138                                             BranchProbability PeeledCaseProb) {
11139   if (PeeledCaseProb == BranchProbability::getOne())
11140     return BranchProbability::getZero();
11141   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11142 
11143   uint32_t Numerator = CaseProb.getNumerator();
11144   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11145   return BranchProbability(Numerator, std::max(Numerator, Denominator));
11146 }
11147 
11148 // Try to peel the top probability case if it exceeds the threshold.
11149 // Return current MachineBasicBlock for the switch statement if the peeling
11150 // does not occur.
11151 // If the peeling is performed, return the newly created MachineBasicBlock
11152 // for the peeled switch statement. Also update Clusters to remove the peeled
11153 // case. PeeledCaseProb is the BranchProbability for the peeled case.
11154 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11155     const SwitchInst &SI, CaseClusterVector &Clusters,
11156     BranchProbability &PeeledCaseProb) {
11157   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11158   // Don't perform if there is only one cluster or optimizing for size.
11159   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11160       TM.getOptLevel() == CodeGenOpt::None ||
11161       SwitchMBB->getParent()->getFunction().hasMinSize())
11162     return SwitchMBB;
11163 
11164   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
11165   unsigned PeeledCaseIndex = 0;
11166   bool SwitchPeeled = false;
11167   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11168     CaseCluster &CC = Clusters[Index];
11169     if (CC.Prob < TopCaseProb)
11170       continue;
11171     TopCaseProb = CC.Prob;
11172     PeeledCaseIndex = Index;
11173     SwitchPeeled = true;
11174   }
11175   if (!SwitchPeeled)
11176     return SwitchMBB;
11177 
11178   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11179                     << TopCaseProb << "\n");
11180 
11181   // Record the MBB for the peeled switch statement.
11182   MachineFunction::iterator BBI(SwitchMBB);
11183   ++BBI;
11184   MachineBasicBlock *PeeledSwitchMBB =
11185       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
11186   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
11187 
11188   ExportFromCurrentBlock(SI.getCondition());
11189   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
11190   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
11191                           nullptr,   nullptr,      TopCaseProb.getCompl()};
11192   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
11193 
11194   Clusters.erase(PeeledCaseIt);
11195   for (CaseCluster &CC : Clusters) {
11196     LLVM_DEBUG(
11197         dbgs() << "Scale the probablity for one cluster, before scaling: "
11198                << CC.Prob << "\n");
11199     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
11200     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
11201   }
11202   PeeledCaseProb = TopCaseProb;
11203   return PeeledSwitchMBB;
11204 }
11205 
11206 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
11207   // Extract cases from the switch.
11208   BranchProbabilityInfo *BPI = FuncInfo.BPI;
11209   CaseClusterVector Clusters;
11210   Clusters.reserve(SI.getNumCases());
11211   for (auto I : SI.cases()) {
11212     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
11213     const ConstantInt *CaseVal = I.getCaseValue();
11214     BranchProbability Prob =
11215         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
11216             : BranchProbability(1, SI.getNumCases() + 1);
11217     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
11218   }
11219 
11220   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
11221 
11222   // Cluster adjacent cases with the same destination. We do this at all
11223   // optimization levels because it's cheap to do and will make codegen faster
11224   // if there are many clusters.
11225   sortAndRangeify(Clusters);
11226 
11227   // The branch probablity of the peeled case.
11228   BranchProbability PeeledCaseProb = BranchProbability::getZero();
11229   MachineBasicBlock *PeeledSwitchMBB =
11230       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
11231 
11232   // If there is only the default destination, jump there directly.
11233   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11234   if (Clusters.empty()) {
11235     assert(PeeledSwitchMBB == SwitchMBB);
11236     SwitchMBB->addSuccessor(DefaultMBB);
11237     if (DefaultMBB != NextBlock(SwitchMBB)) {
11238       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
11239                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
11240     }
11241     return;
11242   }
11243 
11244   SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI());
11245   SL->findBitTestClusters(Clusters, &SI);
11246 
11247   LLVM_DEBUG({
11248     dbgs() << "Case clusters: ";
11249     for (const CaseCluster &C : Clusters) {
11250       if (C.Kind == CC_JumpTable)
11251         dbgs() << "JT:";
11252       if (C.Kind == CC_BitTests)
11253         dbgs() << "BT:";
11254 
11255       C.Low->getValue().print(dbgs(), true);
11256       if (C.Low != C.High) {
11257         dbgs() << '-';
11258         C.High->getValue().print(dbgs(), true);
11259       }
11260       dbgs() << ' ';
11261     }
11262     dbgs() << '\n';
11263   });
11264 
11265   assert(!Clusters.empty());
11266   SwitchWorkList WorkList;
11267   CaseClusterIt First = Clusters.begin();
11268   CaseClusterIt Last = Clusters.end() - 1;
11269   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
11270   // Scale the branchprobability for DefaultMBB if the peel occurs and
11271   // DefaultMBB is not replaced.
11272   if (PeeledCaseProb != BranchProbability::getZero() &&
11273       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
11274     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
11275   WorkList.push_back(
11276       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
11277 
11278   while (!WorkList.empty()) {
11279     SwitchWorkListItem W = WorkList.pop_back_val();
11280     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
11281 
11282     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
11283         !DefaultMBB->getParent()->getFunction().hasMinSize()) {
11284       // For optimized builds, lower large range as a balanced binary tree.
11285       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
11286       continue;
11287     }
11288 
11289     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
11290   }
11291 }
11292 
11293 void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
11294   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11295   auto DL = getCurSDLoc();
11296   EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11297   setValue(&I, DAG.getStepVector(DL, ResultVT));
11298 }
11299 
11300 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
11301   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11302   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11303 
11304   SDLoc DL = getCurSDLoc();
11305   SDValue V = getValue(I.getOperand(0));
11306   assert(VT == V.getValueType() && "Malformed vector.reverse!");
11307 
11308   if (VT.isScalableVector()) {
11309     setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
11310     return;
11311   }
11312 
11313   // Use VECTOR_SHUFFLE for the fixed-length vector
11314   // to maintain existing behavior.
11315   SmallVector<int, 8> Mask;
11316   unsigned NumElts = VT.getVectorMinNumElements();
11317   for (unsigned i = 0; i != NumElts; ++i)
11318     Mask.push_back(NumElts - 1 - i);
11319 
11320   setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
11321 }
11322 
11323 void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
11324   SmallVector<EVT, 4> ValueVTs;
11325   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
11326                   ValueVTs);
11327   unsigned NumValues = ValueVTs.size();
11328   if (NumValues == 0) return;
11329 
11330   SmallVector<SDValue, 4> Values(NumValues);
11331   SDValue Op = getValue(I.getOperand(0));
11332 
11333   for (unsigned i = 0; i != NumValues; ++i)
11334     Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
11335                             SDValue(Op.getNode(), Op.getResNo() + i));
11336 
11337   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
11338                            DAG.getVTList(ValueVTs), Values));
11339 }
11340 
11341 void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
11342   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11343   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11344 
11345   SDLoc DL = getCurSDLoc();
11346   SDValue V1 = getValue(I.getOperand(0));
11347   SDValue V2 = getValue(I.getOperand(1));
11348   int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
11349 
11350   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
11351   if (VT.isScalableVector()) {
11352     MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
11353     setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
11354                              DAG.getConstant(Imm, DL, IdxVT)));
11355     return;
11356   }
11357 
11358   unsigned NumElts = VT.getVectorNumElements();
11359 
11360   uint64_t Idx = (NumElts + Imm) % NumElts;
11361 
11362   // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
11363   SmallVector<int, 8> Mask;
11364   for (unsigned i = 0; i < NumElts; ++i)
11365     Mask.push_back(Idx + i);
11366   setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
11367 }
11368