xref: /llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision 9727c77d58ac920a4158d08c15659470e52ddda4)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/Triple.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/AliasAnalysis.h"
27 #include "llvm/Analysis/BranchProbabilityInfo.h"
28 #include "llvm/Analysis/ConstantFolding.h"
29 #include "llvm/Analysis/EHPersonalities.h"
30 #include "llvm/Analysis/MemoryLocation.h"
31 #include "llvm/Analysis/TargetLibraryInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/Analysis.h"
34 #include "llvm/CodeGen/FunctionLoweringInfo.h"
35 #include "llvm/CodeGen/GCMetadata.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFrameInfo.h"
38 #include "llvm/CodeGen/MachineFunction.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineInstrBundleIterator.h"
41 #include "llvm/CodeGen/MachineMemOperand.h"
42 #include "llvm/CodeGen/MachineModuleInfo.h"
43 #include "llvm/CodeGen/MachineOperand.h"
44 #include "llvm/CodeGen/MachineRegisterInfo.h"
45 #include "llvm/CodeGen/RuntimeLibcalls.h"
46 #include "llvm/CodeGen/SelectionDAG.h"
47 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
48 #include "llvm/CodeGen/StackMaps.h"
49 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
50 #include "llvm/CodeGen/TargetFrameLowering.h"
51 #include "llvm/CodeGen/TargetInstrInfo.h"
52 #include "llvm/CodeGen/TargetOpcodes.h"
53 #include "llvm/CodeGen/TargetRegisterInfo.h"
54 #include "llvm/CodeGen/TargetSubtargetInfo.h"
55 #include "llvm/CodeGen/WinEHFuncInfo.h"
56 #include "llvm/IR/Argument.h"
57 #include "llvm/IR/Attributes.h"
58 #include "llvm/IR/BasicBlock.h"
59 #include "llvm/IR/CFG.h"
60 #include "llvm/IR/CallingConv.h"
61 #include "llvm/IR/Constant.h"
62 #include "llvm/IR/ConstantRange.h"
63 #include "llvm/IR/Constants.h"
64 #include "llvm/IR/DataLayout.h"
65 #include "llvm/IR/DebugInfoMetadata.h"
66 #include "llvm/IR/DerivedTypes.h"
67 #include "llvm/IR/DiagnosticInfo.h"
68 #include "llvm/IR/Function.h"
69 #include "llvm/IR/GetElementPtrTypeIterator.h"
70 #include "llvm/IR/InlineAsm.h"
71 #include "llvm/IR/InstrTypes.h"
72 #include "llvm/IR/Instructions.h"
73 #include "llvm/IR/IntrinsicInst.h"
74 #include "llvm/IR/Intrinsics.h"
75 #include "llvm/IR/IntrinsicsAArch64.h"
76 #include "llvm/IR/IntrinsicsWebAssembly.h"
77 #include "llvm/IR/LLVMContext.h"
78 #include "llvm/IR/Metadata.h"
79 #include "llvm/IR/Module.h"
80 #include "llvm/IR/Operator.h"
81 #include "llvm/IR/PatternMatch.h"
82 #include "llvm/IR/Statepoint.h"
83 #include "llvm/IR/Type.h"
84 #include "llvm/IR/User.h"
85 #include "llvm/IR/Value.h"
86 #include "llvm/MC/MCContext.h"
87 #include "llvm/Support/AtomicOrdering.h"
88 #include "llvm/Support/Casting.h"
89 #include "llvm/Support/CommandLine.h"
90 #include "llvm/Support/Compiler.h"
91 #include "llvm/Support/Debug.h"
92 #include "llvm/Support/MathExtras.h"
93 #include "llvm/Support/raw_ostream.h"
94 #include "llvm/Target/TargetIntrinsicInfo.h"
95 #include "llvm/Target/TargetMachine.h"
96 #include "llvm/Target/TargetOptions.h"
97 #include "llvm/Transforms/Utils/Local.h"
98 #include <cstddef>
99 #include <iterator>
100 #include <limits>
101 #include <tuple>
102 
103 using namespace llvm;
104 using namespace PatternMatch;
105 using namespace SwitchCG;
106 
107 #define DEBUG_TYPE "isel"
108 
109 /// LimitFloatPrecision - Generate low-precision inline sequences for
110 /// some float libcalls (6, 8 or 12 bits).
111 static unsigned LimitFloatPrecision;
112 
113 static cl::opt<bool>
114     InsertAssertAlign("insert-assert-align", cl::init(true),
115                       cl::desc("Insert the experimental `assertalign` node."),
116                       cl::ReallyHidden);
117 
118 static cl::opt<unsigned, true>
119     LimitFPPrecision("limit-float-precision",
120                      cl::desc("Generate low-precision inline sequences "
121                               "for some float libcalls"),
122                      cl::location(LimitFloatPrecision), cl::Hidden,
123                      cl::init(0));
124 
125 static cl::opt<unsigned> SwitchPeelThreshold(
126     "switch-peel-threshold", cl::Hidden, cl::init(66),
127     cl::desc("Set the case probability threshold for peeling the case from a "
128              "switch statement. A value greater than 100 will void this "
129              "optimization"));
130 
131 // Limit the width of DAG chains. This is important in general to prevent
132 // DAG-based analysis from blowing up. For example, alias analysis and
133 // load clustering may not complete in reasonable time. It is difficult to
134 // recognize and avoid this situation within each individual analysis, and
135 // future analyses are likely to have the same behavior. Limiting DAG width is
136 // the safe approach and will be especially important with global DAGs.
137 //
138 // MaxParallelChains default is arbitrarily high to avoid affecting
139 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
140 // sequence over this should have been converted to llvm.memcpy by the
141 // frontend. It is easy to induce this behavior with .ll code such as:
142 // %buffer = alloca [4096 x i8]
143 // %data = load [4096 x i8]* %argPtr
144 // store [4096 x i8] %data, [4096 x i8]* %buffer
145 static const unsigned MaxParallelChains = 64;
146 
147 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
148                                       const SDValue *Parts, unsigned NumParts,
149                                       MVT PartVT, EVT ValueVT, const Value *V,
150                                       Optional<CallingConv::ID> CC);
151 
152 /// getCopyFromParts - Create a value that contains the specified legal parts
153 /// combined into the value they represent.  If the parts combine to a type
154 /// larger than ValueVT then AssertOp can be used to specify whether the extra
155 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
156 /// (ISD::AssertSext).
157 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
158                                 const SDValue *Parts, unsigned NumParts,
159                                 MVT PartVT, EVT ValueVT, const Value *V,
160                                 Optional<CallingConv::ID> CC = None,
161                                 Optional<ISD::NodeType> AssertOp = None) {
162   // Let the target assemble the parts if it wants to
163   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
164   if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
165                                                    PartVT, ValueVT, CC))
166     return Val;
167 
168   if (ValueVT.isVector())
169     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
170                                   CC);
171 
172   assert(NumParts > 0 && "No parts to assemble!");
173   SDValue Val = Parts[0];
174 
175   if (NumParts > 1) {
176     // Assemble the value from multiple parts.
177     if (ValueVT.isInteger()) {
178       unsigned PartBits = PartVT.getSizeInBits();
179       unsigned ValueBits = ValueVT.getSizeInBits();
180 
181       // Assemble the power of 2 part.
182       unsigned RoundParts =
183           (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
184       unsigned RoundBits = PartBits * RoundParts;
185       EVT RoundVT = RoundBits == ValueBits ?
186         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
187       SDValue Lo, Hi;
188 
189       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
190 
191       if (RoundParts > 2) {
192         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
193                               PartVT, HalfVT, V);
194         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
195                               RoundParts / 2, PartVT, HalfVT, V);
196       } else {
197         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
198         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
199       }
200 
201       if (DAG.getDataLayout().isBigEndian())
202         std::swap(Lo, Hi);
203 
204       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
205 
206       if (RoundParts < NumParts) {
207         // Assemble the trailing non-power-of-2 part.
208         unsigned OddParts = NumParts - RoundParts;
209         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
210         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
211                               OddVT, V, CC);
212 
213         // Combine the round and odd parts.
214         Lo = Val;
215         if (DAG.getDataLayout().isBigEndian())
216           std::swap(Lo, Hi);
217         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
218         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
219         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
220                          DAG.getConstant(Lo.getValueSizeInBits(), DL,
221                                          TLI.getShiftAmountTy(
222                                              TotalVT, DAG.getDataLayout())));
223         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
224         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
225       }
226     } else if (PartVT.isFloatingPoint()) {
227       // FP split into multiple FP parts (for ppcf128)
228       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
229              "Unexpected split");
230       SDValue Lo, Hi;
231       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
232       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
233       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
234         std::swap(Lo, Hi);
235       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
236     } else {
237       // FP split into integer parts (soft fp)
238       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
239              !PartVT.isVector() && "Unexpected split");
240       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
241       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
242     }
243   }
244 
245   // There is now one part, held in Val.  Correct it to match ValueVT.
246   // PartEVT is the type of the register class that holds the value.
247   // ValueVT is the type of the inline asm operation.
248   EVT PartEVT = Val.getValueType();
249 
250   if (PartEVT == ValueVT)
251     return Val;
252 
253   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
254       ValueVT.bitsLT(PartEVT)) {
255     // For an FP value in an integer part, we need to truncate to the right
256     // width first.
257     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
258     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
259   }
260 
261   // Handle types that have the same size.
262   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
263     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
264 
265   // Handle types with different sizes.
266   if (PartEVT.isInteger() && ValueVT.isInteger()) {
267     if (ValueVT.bitsLT(PartEVT)) {
268       // For a truncate, see if we have any information to
269       // indicate whether the truncated bits will always be
270       // zero or sign-extension.
271       if (AssertOp.hasValue())
272         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
273                           DAG.getValueType(ValueVT));
274       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
275     }
276     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
277   }
278 
279   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
280     // FP_ROUND's are always exact here.
281     if (ValueVT.bitsLT(Val.getValueType()))
282       return DAG.getNode(
283           ISD::FP_ROUND, DL, ValueVT, Val,
284           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
285 
286     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
287   }
288 
289   // Handle MMX to a narrower integer type by bitcasting MMX to integer and
290   // then truncating.
291   if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
292       ValueVT.bitsLT(PartEVT)) {
293     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
294     return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
295   }
296 
297   report_fatal_error("Unknown mismatch in getCopyFromParts!");
298 }
299 
300 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
301                                               const Twine &ErrMsg) {
302   const Instruction *I = dyn_cast_or_null<Instruction>(V);
303   if (!V)
304     return Ctx.emitError(ErrMsg);
305 
306   const char *AsmError = ", possible invalid constraint for vector type";
307   if (const CallInst *CI = dyn_cast<CallInst>(I))
308     if (CI->isInlineAsm())
309       return Ctx.emitError(I, ErrMsg + AsmError);
310 
311   return Ctx.emitError(I, ErrMsg);
312 }
313 
314 /// getCopyFromPartsVector - Create a value that contains the specified legal
315 /// parts combined into the value they represent.  If the parts combine to a
316 /// type larger than ValueVT then AssertOp can be used to specify whether the
317 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
318 /// ValueVT (ISD::AssertSext).
319 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
320                                       const SDValue *Parts, unsigned NumParts,
321                                       MVT PartVT, EVT ValueVT, const Value *V,
322                                       Optional<CallingConv::ID> CallConv) {
323   assert(ValueVT.isVector() && "Not a vector value");
324   assert(NumParts > 0 && "No parts to assemble!");
325   const bool IsABIRegCopy = CallConv.hasValue();
326 
327   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
328   SDValue Val = Parts[0];
329 
330   // Handle a multi-element vector.
331   if (NumParts > 1) {
332     EVT IntermediateVT;
333     MVT RegisterVT;
334     unsigned NumIntermediates;
335     unsigned NumRegs;
336 
337     if (IsABIRegCopy) {
338       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
339           *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
340           NumIntermediates, RegisterVT);
341     } else {
342       NumRegs =
343           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
344                                      NumIntermediates, RegisterVT);
345     }
346 
347     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
348     NumParts = NumRegs; // Silence a compiler warning.
349     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
350     assert(RegisterVT.getSizeInBits() ==
351            Parts[0].getSimpleValueType().getSizeInBits() &&
352            "Part type sizes don't match!");
353 
354     // Assemble the parts into intermediate operands.
355     SmallVector<SDValue, 8> Ops(NumIntermediates);
356     if (NumIntermediates == NumParts) {
357       // If the register was not expanded, truncate or copy the value,
358       // as appropriate.
359       for (unsigned i = 0; i != NumParts; ++i)
360         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
361                                   PartVT, IntermediateVT, V, CallConv);
362     } else if (NumParts > 0) {
363       // If the intermediate type was expanded, build the intermediate
364       // operands from the parts.
365       assert(NumParts % NumIntermediates == 0 &&
366              "Must expand into a divisible number of parts!");
367       unsigned Factor = NumParts / NumIntermediates;
368       for (unsigned i = 0; i != NumIntermediates; ++i)
369         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
370                                   PartVT, IntermediateVT, V, CallConv);
371     }
372 
373     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
374     // intermediate operands.
375     EVT BuiltVectorTy =
376         IntermediateVT.isVector()
377             ? EVT::getVectorVT(
378                   *DAG.getContext(), IntermediateVT.getScalarType(),
379                   IntermediateVT.getVectorElementCount() * NumParts)
380             : EVT::getVectorVT(*DAG.getContext(),
381                                IntermediateVT.getScalarType(),
382                                NumIntermediates);
383     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
384                                                 : ISD::BUILD_VECTOR,
385                       DL, BuiltVectorTy, Ops);
386   }
387 
388   // There is now one part, held in Val.  Correct it to match ValueVT.
389   EVT PartEVT = Val.getValueType();
390 
391   if (PartEVT == ValueVT)
392     return Val;
393 
394   if (PartEVT.isVector()) {
395     // Vector/Vector bitcast.
396     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
397       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
398 
399     // If the element type of the source/dest vectors are the same, but the
400     // parts vector has more elements than the value vector, then we have a
401     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
402     // elements we want.
403     if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
404       assert((PartEVT.getVectorElementCount().getKnownMinValue() >
405               ValueVT.getVectorElementCount().getKnownMinValue()) &&
406              (PartEVT.getVectorElementCount().isScalable() ==
407               ValueVT.getVectorElementCount().isScalable()) &&
408              "Cannot narrow, it would be a lossy transformation");
409       PartEVT =
410           EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
411                            ValueVT.getVectorElementCount());
412       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
413                         DAG.getVectorIdxConstant(0, DL));
414       if (PartEVT == ValueVT)
415         return Val;
416     }
417 
418     // Promoted vector extract
419     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
420   }
421 
422   // Trivial bitcast if the types are the same size and the destination
423   // vector type is legal.
424   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
425       TLI.isTypeLegal(ValueVT))
426     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
427 
428   if (ValueVT.getVectorNumElements() != 1) {
429      // Certain ABIs require that vectors are passed as integers. For vectors
430      // are the same size, this is an obvious bitcast.
431      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
432        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433      } else if (ValueVT.bitsLT(PartEVT)) {
434        const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
435        EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
436        // Drop the extra bits.
437        Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
438        return DAG.getBitcast(ValueVT, Val);
439      }
440 
441      diagnosePossiblyInvalidConstraint(
442          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
443      return DAG.getUNDEF(ValueVT);
444   }
445 
446   // Handle cases such as i8 -> <1 x i1>
447   EVT ValueSVT = ValueVT.getVectorElementType();
448   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
449     if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())
450       Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
451     else
452       Val = ValueVT.isFloatingPoint()
453                 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
454                 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
455   }
456 
457   return DAG.getBuildVector(ValueVT, DL, Val);
458 }
459 
460 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
461                                  SDValue Val, SDValue *Parts, unsigned NumParts,
462                                  MVT PartVT, const Value *V,
463                                  Optional<CallingConv::ID> CallConv);
464 
465 /// getCopyToParts - Create a series of nodes that contain the specified value
466 /// split into legal parts.  If the parts contain more bits than Val, then, for
467 /// integers, ExtendKind can be used to specify how to generate the extra bits.
468 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
469                            SDValue *Parts, unsigned NumParts, MVT PartVT,
470                            const Value *V,
471                            Optional<CallingConv::ID> CallConv = None,
472                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
473   // Let the target split the parts if it wants to
474   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
475   if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
476                                       CallConv))
477     return;
478   EVT ValueVT = Val.getValueType();
479 
480   // Handle the vector case separately.
481   if (ValueVT.isVector())
482     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
483                                 CallConv);
484 
485   unsigned PartBits = PartVT.getSizeInBits();
486   unsigned OrigNumParts = NumParts;
487   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
488          "Copying to an illegal type!");
489 
490   if (NumParts == 0)
491     return;
492 
493   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
494   EVT PartEVT = PartVT;
495   if (PartEVT == ValueVT) {
496     assert(NumParts == 1 && "No-op copy with multiple parts!");
497     Parts[0] = Val;
498     return;
499   }
500 
501   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
502     // If the parts cover more bits than the value has, promote the value.
503     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
504       assert(NumParts == 1 && "Do not know what to promote to!");
505       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
506     } else {
507       if (ValueVT.isFloatingPoint()) {
508         // FP values need to be bitcast, then extended if they are being put
509         // into a larger container.
510         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
511         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
512       }
513       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
514              ValueVT.isInteger() &&
515              "Unknown mismatch!");
516       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
517       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
518       if (PartVT == MVT::x86mmx)
519         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
520     }
521   } else if (PartBits == ValueVT.getSizeInBits()) {
522     // Different types of the same size.
523     assert(NumParts == 1 && PartEVT != ValueVT);
524     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
525   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
526     // If the parts cover less bits than value has, truncate the value.
527     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
528            ValueVT.isInteger() &&
529            "Unknown mismatch!");
530     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
531     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
532     if (PartVT == MVT::x86mmx)
533       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
534   }
535 
536   // The value may have changed - recompute ValueVT.
537   ValueVT = Val.getValueType();
538   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
539          "Failed to tile the value with PartVT!");
540 
541   if (NumParts == 1) {
542     if (PartEVT != ValueVT) {
543       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
544                                         "scalar-to-vector conversion failed");
545       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
546     }
547 
548     Parts[0] = Val;
549     return;
550   }
551 
552   // Expand the value into multiple parts.
553   if (NumParts & (NumParts - 1)) {
554     // The number of parts is not a power of 2.  Split off and copy the tail.
555     assert(PartVT.isInteger() && ValueVT.isInteger() &&
556            "Do not know what to expand to!");
557     unsigned RoundParts = 1 << Log2_32(NumParts);
558     unsigned RoundBits = RoundParts * PartBits;
559     unsigned OddParts = NumParts - RoundParts;
560     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
561       DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
562 
563     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
564                    CallConv);
565 
566     if (DAG.getDataLayout().isBigEndian())
567       // The odd parts were reversed by getCopyToParts - unreverse them.
568       std::reverse(Parts + RoundParts, Parts + NumParts);
569 
570     NumParts = RoundParts;
571     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
572     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
573   }
574 
575   // The number of parts is a power of 2.  Repeatedly bisect the value using
576   // EXTRACT_ELEMENT.
577   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
578                          EVT::getIntegerVT(*DAG.getContext(),
579                                            ValueVT.getSizeInBits()),
580                          Val);
581 
582   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
583     for (unsigned i = 0; i < NumParts; i += StepSize) {
584       unsigned ThisBits = StepSize * PartBits / 2;
585       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
586       SDValue &Part0 = Parts[i];
587       SDValue &Part1 = Parts[i+StepSize/2];
588 
589       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
590                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
591       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
592                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
593 
594       if (ThisBits == PartBits && ThisVT != PartVT) {
595         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
596         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
597       }
598     }
599   }
600 
601   if (DAG.getDataLayout().isBigEndian())
602     std::reverse(Parts, Parts + OrigNumParts);
603 }
604 
605 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
606                                      const SDLoc &DL, EVT PartVT) {
607   if (!PartVT.isVector())
608     return SDValue();
609 
610   EVT ValueVT = Val.getValueType();
611   ElementCount PartNumElts = PartVT.getVectorElementCount();
612   ElementCount ValueNumElts = ValueVT.getVectorElementCount();
613 
614   // We only support widening vectors with equivalent element types and
615   // fixed/scalable properties. If a target needs to widen a fixed-length type
616   // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
617   if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
618       PartNumElts.isScalable() != ValueNumElts.isScalable() ||
619       PartVT.getVectorElementType() != ValueVT.getVectorElementType())
620     return SDValue();
621 
622   // Widening a scalable vector to another scalable vector is done by inserting
623   // the vector into a larger undef one.
624   if (PartNumElts.isScalable())
625     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
626                        Val, DAG.getVectorIdxConstant(0, DL));
627 
628   EVT ElementVT = PartVT.getVectorElementType();
629   // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
630   // undef elements.
631   SmallVector<SDValue, 16> Ops;
632   DAG.ExtractVectorElements(Val, Ops);
633   SDValue EltUndef = DAG.getUNDEF(ElementVT);
634   Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
635 
636   // FIXME: Use CONCAT for 2x -> 4x.
637   return DAG.getBuildVector(PartVT, DL, Ops);
638 }
639 
640 /// getCopyToPartsVector - Create a series of nodes that contain the specified
641 /// value split into legal parts.
642 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
643                                  SDValue Val, SDValue *Parts, unsigned NumParts,
644                                  MVT PartVT, const Value *V,
645                                  Optional<CallingConv::ID> CallConv) {
646   EVT ValueVT = Val.getValueType();
647   assert(ValueVT.isVector() && "Not a vector");
648   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
649   const bool IsABIRegCopy = CallConv.hasValue();
650 
651   if (NumParts == 1) {
652     EVT PartEVT = PartVT;
653     if (PartEVT == ValueVT) {
654       // Nothing to do.
655     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
656       // Bitconvert vector->vector case.
657       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
658     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
659       Val = Widened;
660     } else if (PartVT.isVector() &&
661                PartEVT.getVectorElementType().bitsGE(
662                    ValueVT.getVectorElementType()) &&
663                PartEVT.getVectorElementCount() ==
664                    ValueVT.getVectorElementCount()) {
665 
666       // Promoted vector extract
667       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
668     } else if (PartEVT.isVector() &&
669                PartEVT.getVectorElementType() !=
670                    ValueVT.getVectorElementType() &&
671                TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
672                    TargetLowering::TypeWidenVector) {
673       // Combination of widening and promotion.
674       EVT WidenVT =
675           EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
676                            PartVT.getVectorElementCount());
677       SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
678       Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
679     } else {
680       if (ValueVT.getVectorElementCount().isScalar()) {
681         Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
682                           DAG.getVectorIdxConstant(0, DL));
683       } else {
684         uint64_t ValueSize = ValueVT.getFixedSizeInBits();
685         assert(PartVT.getFixedSizeInBits() > ValueSize &&
686                "lossy conversion of vector to scalar type");
687         EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
688         Val = DAG.getBitcast(IntermediateType, Val);
689         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
690       }
691     }
692 
693     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
694     Parts[0] = Val;
695     return;
696   }
697 
698   // Handle a multi-element vector.
699   EVT IntermediateVT;
700   MVT RegisterVT;
701   unsigned NumIntermediates;
702   unsigned NumRegs;
703   if (IsABIRegCopy) {
704     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
705         *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
706         NumIntermediates, RegisterVT);
707   } else {
708     NumRegs =
709         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
710                                    NumIntermediates, RegisterVT);
711   }
712 
713   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
714   NumParts = NumRegs; // Silence a compiler warning.
715   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
716 
717   assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
718          "Mixing scalable and fixed vectors when copying in parts");
719 
720   Optional<ElementCount> DestEltCnt;
721 
722   if (IntermediateVT.isVector())
723     DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
724   else
725     DestEltCnt = ElementCount::getFixed(NumIntermediates);
726 
727   EVT BuiltVectorTy = EVT::getVectorVT(
728       *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());
729 
730   if (ValueVT == BuiltVectorTy) {
731     // Nothing to do.
732   } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
733     // Bitconvert vector->vector case.
734     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
735   } else {
736     if (BuiltVectorTy.getVectorElementType().bitsGT(
737             ValueVT.getVectorElementType())) {
738       // Integer promotion.
739       ValueVT = EVT::getVectorVT(*DAG.getContext(),
740                                  BuiltVectorTy.getVectorElementType(),
741                                  ValueVT.getVectorElementCount());
742       Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
743     }
744 
745     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
746       Val = Widened;
747     }
748   }
749 
750   assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
751 
752   // Split the vector into intermediate operands.
753   SmallVector<SDValue, 8> Ops(NumIntermediates);
754   for (unsigned i = 0; i != NumIntermediates; ++i) {
755     if (IntermediateVT.isVector()) {
756       // This does something sensible for scalable vectors - see the
757       // definition of EXTRACT_SUBVECTOR for further details.
758       unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
759       Ops[i] =
760           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
761                       DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
762     } else {
763       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
764                            DAG.getVectorIdxConstant(i, DL));
765     }
766   }
767 
768   // Split the intermediate operands into legal parts.
769   if (NumParts == NumIntermediates) {
770     // If the register was not expanded, promote or copy the value,
771     // as appropriate.
772     for (unsigned i = 0; i != NumParts; ++i)
773       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
774   } else if (NumParts > 0) {
775     // If the intermediate type was expanded, split each the value into
776     // legal parts.
777     assert(NumIntermediates != 0 && "division by zero");
778     assert(NumParts % NumIntermediates == 0 &&
779            "Must expand into a divisible number of parts!");
780     unsigned Factor = NumParts / NumIntermediates;
781     for (unsigned i = 0; i != NumIntermediates; ++i)
782       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
783                      CallConv);
784   }
785 }
786 
787 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
788                            EVT valuevt, Optional<CallingConv::ID> CC)
789     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
790       RegCount(1, regs.size()), CallConv(CC) {}
791 
792 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
793                            const DataLayout &DL, unsigned Reg, Type *Ty,
794                            Optional<CallingConv::ID> CC) {
795   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
796 
797   CallConv = CC;
798 
799   for (EVT ValueVT : ValueVTs) {
800     unsigned NumRegs =
801         isABIMangled()
802             ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
803             : TLI.getNumRegisters(Context, ValueVT);
804     MVT RegisterVT =
805         isABIMangled()
806             ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
807             : TLI.getRegisterType(Context, ValueVT);
808     for (unsigned i = 0; i != NumRegs; ++i)
809       Regs.push_back(Reg + i);
810     RegVTs.push_back(RegisterVT);
811     RegCount.push_back(NumRegs);
812     Reg += NumRegs;
813   }
814 }
815 
816 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
817                                       FunctionLoweringInfo &FuncInfo,
818                                       const SDLoc &dl, SDValue &Chain,
819                                       SDValue *Flag, const Value *V) const {
820   // A Value with type {} or [0 x %t] needs no registers.
821   if (ValueVTs.empty())
822     return SDValue();
823 
824   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
825 
826   // Assemble the legal parts into the final values.
827   SmallVector<SDValue, 4> Values(ValueVTs.size());
828   SmallVector<SDValue, 8> Parts;
829   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
830     // Copy the legal parts from the registers.
831     EVT ValueVT = ValueVTs[Value];
832     unsigned NumRegs = RegCount[Value];
833     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
834                                           *DAG.getContext(),
835                                           CallConv.getValue(), RegVTs[Value])
836                                     : RegVTs[Value];
837 
838     Parts.resize(NumRegs);
839     for (unsigned i = 0; i != NumRegs; ++i) {
840       SDValue P;
841       if (!Flag) {
842         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
843       } else {
844         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
845         *Flag = P.getValue(2);
846       }
847 
848       Chain = P.getValue(1);
849       Parts[i] = P;
850 
851       // If the source register was virtual and if we know something about it,
852       // add an assert node.
853       if (!Register::isVirtualRegister(Regs[Part + i]) ||
854           !RegisterVT.isInteger())
855         continue;
856 
857       const FunctionLoweringInfo::LiveOutInfo *LOI =
858         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
859       if (!LOI)
860         continue;
861 
862       unsigned RegSize = RegisterVT.getScalarSizeInBits();
863       unsigned NumSignBits = LOI->NumSignBits;
864       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
865 
866       if (NumZeroBits == RegSize) {
867         // The current value is a zero.
868         // Explicitly express that as it would be easier for
869         // optimizations to kick in.
870         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
871         continue;
872       }
873 
874       // FIXME: We capture more information than the dag can represent.  For
875       // now, just use the tightest assertzext/assertsext possible.
876       bool isSExt;
877       EVT FromVT(MVT::Other);
878       if (NumZeroBits) {
879         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
880         isSExt = false;
881       } else if (NumSignBits > 1) {
882         FromVT =
883             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
884         isSExt = true;
885       } else {
886         continue;
887       }
888       // Add an assertion node.
889       assert(FromVT != MVT::Other);
890       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
891                              RegisterVT, P, DAG.getValueType(FromVT));
892     }
893 
894     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
895                                      RegisterVT, ValueVT, V, CallConv);
896     Part += NumRegs;
897     Parts.clear();
898   }
899 
900   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
901 }
902 
903 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
904                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
905                                  const Value *V,
906                                  ISD::NodeType PreferredExtendType) const {
907   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
908   ISD::NodeType ExtendKind = PreferredExtendType;
909 
910   // Get the list of the values's legal parts.
911   unsigned NumRegs = Regs.size();
912   SmallVector<SDValue, 8> Parts(NumRegs);
913   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
914     unsigned NumParts = RegCount[Value];
915 
916     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
917                                           *DAG.getContext(),
918                                           CallConv.getValue(), RegVTs[Value])
919                                     : RegVTs[Value];
920 
921     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
922       ExtendKind = ISD::ZERO_EXTEND;
923 
924     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
925                    NumParts, RegisterVT, V, CallConv, ExtendKind);
926     Part += NumParts;
927   }
928 
929   // Copy the parts into the registers.
930   SmallVector<SDValue, 8> Chains(NumRegs);
931   for (unsigned i = 0; i != NumRegs; ++i) {
932     SDValue Part;
933     if (!Flag) {
934       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
935     } else {
936       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
937       *Flag = Part.getValue(1);
938     }
939 
940     Chains[i] = Part.getValue(0);
941   }
942 
943   if (NumRegs == 1 || Flag)
944     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
945     // flagged to it. That is the CopyToReg nodes and the user are considered
946     // a single scheduling unit. If we create a TokenFactor and return it as
947     // chain, then the TokenFactor is both a predecessor (operand) of the
948     // user as well as a successor (the TF operands are flagged to the user).
949     // c1, f1 = CopyToReg
950     // c2, f2 = CopyToReg
951     // c3     = TokenFactor c1, c2
952     // ...
953     //        = op c3, ..., f2
954     Chain = Chains[NumRegs-1];
955   else
956     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
957 }
958 
959 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
960                                         unsigned MatchingIdx, const SDLoc &dl,
961                                         SelectionDAG &DAG,
962                                         std::vector<SDValue> &Ops) const {
963   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
964 
965   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
966   if (HasMatching)
967     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
968   else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
969     // Put the register class of the virtual registers in the flag word.  That
970     // way, later passes can recompute register class constraints for inline
971     // assembly as well as normal instructions.
972     // Don't do this for tied operands that can use the regclass information
973     // from the def.
974     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
975     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
976     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
977   }
978 
979   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
980   Ops.push_back(Res);
981 
982   if (Code == InlineAsm::Kind_Clobber) {
983     // Clobbers should always have a 1:1 mapping with registers, and may
984     // reference registers that have illegal (e.g. vector) types. Hence, we
985     // shouldn't try to apply any sort of splitting logic to them.
986     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
987            "No 1:1 mapping from clobbers to regs?");
988     Register SP = TLI.getStackPointerRegisterToSaveRestore();
989     (void)SP;
990     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
991       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
992       assert(
993           (Regs[I] != SP ||
994            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
995           "If we clobbered the stack pointer, MFI should know about it.");
996     }
997     return;
998   }
999 
1000   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1001     MVT RegisterVT = RegVTs[Value];
1002     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1003                                            RegisterVT);
1004     for (unsigned i = 0; i != NumRegs; ++i) {
1005       assert(Reg < Regs.size() && "Mismatch in # registers expected");
1006       unsigned TheReg = Regs[Reg++];
1007       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1008     }
1009   }
1010 }
1011 
1012 SmallVector<std::pair<unsigned, TypeSize>, 4>
1013 RegsForValue::getRegsAndSizes() const {
1014   SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
1015   unsigned I = 0;
1016   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1017     unsigned RegCount = std::get<0>(CountAndVT);
1018     MVT RegisterVT = std::get<1>(CountAndVT);
1019     TypeSize RegisterSize = RegisterVT.getSizeInBits();
1020     for (unsigned E = I + RegCount; I != E; ++I)
1021       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1022   }
1023   return OutVec;
1024 }
1025 
1026 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1027                                const TargetLibraryInfo *li) {
1028   AA = aa;
1029   GFI = gfi;
1030   LibInfo = li;
1031   Context = DAG.getContext();
1032   LPadToCallSiteMap.clear();
1033   SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1034 }
1035 
1036 void SelectionDAGBuilder::clear() {
1037   NodeMap.clear();
1038   UnusedArgNodeMap.clear();
1039   PendingLoads.clear();
1040   PendingExports.clear();
1041   PendingConstrainedFP.clear();
1042   PendingConstrainedFPStrict.clear();
1043   CurInst = nullptr;
1044   HasTailCall = false;
1045   SDNodeOrder = LowestSDNodeOrder;
1046   StatepointLowering.clear();
1047 }
1048 
1049 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1050   DanglingDebugInfoMap.clear();
1051 }
1052 
1053 // Update DAG root to include dependencies on Pending chains.
1054 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1055   SDValue Root = DAG.getRoot();
1056 
1057   if (Pending.empty())
1058     return Root;
1059 
1060   // Add current root to PendingChains, unless we already indirectly
1061   // depend on it.
1062   if (Root.getOpcode() != ISD::EntryToken) {
1063     unsigned i = 0, e = Pending.size();
1064     for (; i != e; ++i) {
1065       assert(Pending[i].getNode()->getNumOperands() > 1);
1066       if (Pending[i].getNode()->getOperand(0) == Root)
1067         break;  // Don't add the root if we already indirectly depend on it.
1068     }
1069 
1070     if (i == e)
1071       Pending.push_back(Root);
1072   }
1073 
1074   if (Pending.size() == 1)
1075     Root = Pending[0];
1076   else
1077     Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1078 
1079   DAG.setRoot(Root);
1080   Pending.clear();
1081   return Root;
1082 }
1083 
1084 SDValue SelectionDAGBuilder::getMemoryRoot() {
1085   return updateRoot(PendingLoads);
1086 }
1087 
1088 SDValue SelectionDAGBuilder::getRoot() {
1089   // Chain up all pending constrained intrinsics together with all
1090   // pending loads, by simply appending them to PendingLoads and
1091   // then calling getMemoryRoot().
1092   PendingLoads.reserve(PendingLoads.size() +
1093                        PendingConstrainedFP.size() +
1094                        PendingConstrainedFPStrict.size());
1095   PendingLoads.append(PendingConstrainedFP.begin(),
1096                       PendingConstrainedFP.end());
1097   PendingLoads.append(PendingConstrainedFPStrict.begin(),
1098                       PendingConstrainedFPStrict.end());
1099   PendingConstrainedFP.clear();
1100   PendingConstrainedFPStrict.clear();
1101   return getMemoryRoot();
1102 }
1103 
1104 SDValue SelectionDAGBuilder::getControlRoot() {
1105   // We need to emit pending fpexcept.strict constrained intrinsics,
1106   // so append them to the PendingExports list.
1107   PendingExports.append(PendingConstrainedFPStrict.begin(),
1108                         PendingConstrainedFPStrict.end());
1109   PendingConstrainedFPStrict.clear();
1110   return updateRoot(PendingExports);
1111 }
1112 
1113 void SelectionDAGBuilder::visit(const Instruction &I) {
1114   // Set up outgoing PHI node register values before emitting the terminator.
1115   if (I.isTerminator()) {
1116     HandlePHINodesInSuccessorBlocks(I.getParent());
1117   }
1118 
1119   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1120   if (!isa<DbgInfoIntrinsic>(I))
1121     ++SDNodeOrder;
1122 
1123   CurInst = &I;
1124 
1125   visit(I.getOpcode(), I);
1126 
1127   if (!I.isTerminator() && !HasTailCall &&
1128       !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1129     CopyToExportRegsIfNeeded(&I);
1130 
1131   CurInst = nullptr;
1132 }
1133 
1134 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1135   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1136 }
1137 
1138 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1139   // Note: this doesn't use InstVisitor, because it has to work with
1140   // ConstantExpr's in addition to instructions.
1141   switch (Opcode) {
1142   default: llvm_unreachable("Unknown instruction type encountered!");
1143     // Build the switch statement using the Instruction.def file.
1144 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1145     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1146 #include "llvm/IR/Instruction.def"
1147   }
1148 }
1149 
1150 void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
1151                                                DebugLoc DL, unsigned Order) {
1152   // We treat variadic dbg_values differently at this stage.
1153   if (DI->hasArgList()) {
1154     // For variadic dbg_values we will now insert an undef.
1155     // FIXME: We can potentially recover these!
1156     SmallVector<SDDbgOperand, 2> Locs;
1157     for (const Value *V : DI->getValues()) {
1158       auto Undef = UndefValue::get(V->getType());
1159       Locs.push_back(SDDbgOperand::fromConst(Undef));
1160     }
1161     SDDbgValue *SDV = DAG.getDbgValueList(
1162         DI->getVariable(), DI->getExpression(), Locs, {},
1163         /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);
1164     DAG.AddDbgValue(SDV, /*isParameter=*/false);
1165   } else {
1166     // TODO: Dangling debug info will eventually either be resolved or produce
1167     // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1168     // between the original dbg.value location and its resolved DBG_VALUE,
1169     // which we should ideally fill with an extra Undef DBG_VALUE.
1170     assert(DI->getNumVariableLocationOps() == 1 &&
1171            "DbgValueInst without an ArgList should have a single location "
1172            "operand.");
1173     DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);
1174   }
1175 }
1176 
1177 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1178                                                 const DIExpression *Expr) {
1179   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1180     const DbgValueInst *DI = DDI.getDI();
1181     DIVariable *DanglingVariable = DI->getVariable();
1182     DIExpression *DanglingExpr = DI->getExpression();
1183     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1184       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1185       return true;
1186     }
1187     return false;
1188   };
1189 
1190   for (auto &DDIMI : DanglingDebugInfoMap) {
1191     DanglingDebugInfoVector &DDIV = DDIMI.second;
1192 
1193     // If debug info is to be dropped, run it through final checks to see
1194     // whether it can be salvaged.
1195     for (auto &DDI : DDIV)
1196       if (isMatchingDbgValue(DDI))
1197         salvageUnresolvedDbgValue(DDI);
1198 
1199     erase_if(DDIV, isMatchingDbgValue);
1200   }
1201 }
1202 
1203 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1204 // generate the debug data structures now that we've seen its definition.
1205 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1206                                                    SDValue Val) {
1207   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1208   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1209     return;
1210 
1211   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1212   for (auto &DDI : DDIV) {
1213     const DbgValueInst *DI = DDI.getDI();
1214     assert(!DI->hasArgList() && "Not implemented for variadic dbg_values");
1215     assert(DI && "Ill-formed DanglingDebugInfo");
1216     DebugLoc dl = DDI.getdl();
1217     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1218     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1219     DILocalVariable *Variable = DI->getVariable();
1220     DIExpression *Expr = DI->getExpression();
1221     assert(Variable->isValidLocationForIntrinsic(dl) &&
1222            "Expected inlined-at fields to agree");
1223     SDDbgValue *SDV;
1224     if (Val.getNode()) {
1225       // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1226       // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1227       // we couldn't resolve it directly when examining the DbgValue intrinsic
1228       // in the first place we should not be more successful here). Unless we
1229       // have some test case that prove this to be correct we should avoid
1230       // calling EmitFuncArgumentDbgValue here.
1231       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl,
1232                                     FuncArgumentDbgValueKind::Value, Val)) {
1233         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1234                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1235         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1236         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1237         // inserted after the definition of Val when emitting the instructions
1238         // after ISel. An alternative could be to teach
1239         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1240         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1241                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1242                    << ValSDNodeOrder << "\n");
1243         SDV = getDbgValue(Val, Variable, Expr, dl,
1244                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1245         DAG.AddDbgValue(SDV, false);
1246       } else
1247         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1248                           << "in EmitFuncArgumentDbgValue\n");
1249     } else {
1250       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1251       auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1252       auto SDV =
1253           DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1254       DAG.AddDbgValue(SDV, false);
1255     }
1256   }
1257   DDIV.clear();
1258 }
1259 
1260 void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1261   // TODO: For the variadic implementation, instead of only checking the fail
1262   // state of `handleDebugValue`, we need know specifically which values were
1263   // invalid, so that we attempt to salvage only those values when processing
1264   // a DIArgList.
1265   assert(!DDI.getDI()->hasArgList() &&
1266          "Not implemented for variadic dbg_values");
1267   Value *V = DDI.getDI()->getValue(0);
1268   DILocalVariable *Var = DDI.getDI()->getVariable();
1269   DIExpression *Expr = DDI.getDI()->getExpression();
1270   DebugLoc DL = DDI.getdl();
1271   DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1272   unsigned SDOrder = DDI.getSDNodeOrder();
1273   // Currently we consider only dbg.value intrinsics -- we tell the salvager
1274   // that DW_OP_stack_value is desired.
1275   assert(isa<DbgValueInst>(DDI.getDI()));
1276   bool StackValue = true;
1277 
1278   // Can this Value can be encoded without any further work?
1279   if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))
1280     return;
1281 
1282   // Attempt to salvage back through as many instructions as possible. Bail if
1283   // a non-instruction is seen, such as a constant expression or global
1284   // variable. FIXME: Further work could recover those too.
1285   while (isa<Instruction>(V)) {
1286     Instruction &VAsInst = *cast<Instruction>(V);
1287     // Temporary "0", awaiting real implementation.
1288     SmallVector<uint64_t, 16> Ops;
1289     SmallVector<Value *, 4> AdditionalValues;
1290     V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1291                              AdditionalValues);
1292     // If we cannot salvage any further, and haven't yet found a suitable debug
1293     // expression, bail out.
1294     if (!V)
1295       break;
1296 
1297     // TODO: If AdditionalValues isn't empty, then the salvage can only be
1298     // represented with a DBG_VALUE_LIST, so we give up. When we have support
1299     // here for variadic dbg_values, remove that condition.
1300     if (!AdditionalValues.empty())
1301       break;
1302 
1303     // New value and expr now represent this debuginfo.
1304     Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1305 
1306     // Some kind of simplification occurred: check whether the operand of the
1307     // salvaged debug expression can be encoded in this DAG.
1308     if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,
1309                          /*IsVariadic=*/false)) {
1310       LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n  "
1311                         << DDI.getDI() << "\nBy stripping back to:\n  " << V);
1312       return;
1313     }
1314   }
1315 
1316   // This was the final opportunity to salvage this debug information, and it
1317   // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1318   // any earlier variable location.
1319   auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1320   auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1321   DAG.AddDbgValue(SDV, false);
1322 
1323   LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << DDI.getDI()
1324                     << "\n");
1325   LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *DDI.getDI()->getOperand(0)
1326                     << "\n");
1327 }
1328 
1329 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1330                                            DILocalVariable *Var,
1331                                            DIExpression *Expr, DebugLoc dl,
1332                                            DebugLoc InstDL, unsigned Order,
1333                                            bool IsVariadic) {
1334   if (Values.empty())
1335     return true;
1336   SmallVector<SDDbgOperand> LocationOps;
1337   SmallVector<SDNode *> Dependencies;
1338   for (const Value *V : Values) {
1339     // Constant value.
1340     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1341         isa<ConstantPointerNull>(V)) {
1342       LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1343       continue;
1344     }
1345 
1346     // If the Value is a frame index, we can create a FrameIndex debug value
1347     // without relying on the DAG at all.
1348     if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1349       auto SI = FuncInfo.StaticAllocaMap.find(AI);
1350       if (SI != FuncInfo.StaticAllocaMap.end()) {
1351         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1352         continue;
1353       }
1354     }
1355 
1356     // Do not use getValue() in here; we don't want to generate code at
1357     // this point if it hasn't been done yet.
1358     SDValue N = NodeMap[V];
1359     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1360       N = UnusedArgNodeMap[V];
1361     if (N.getNode()) {
1362       // Only emit func arg dbg value for non-variadic dbg.values for now.
1363       if (!IsVariadic &&
1364           EmitFuncArgumentDbgValue(V, Var, Expr, dl,
1365                                    FuncArgumentDbgValueKind::Value, N))
1366         return true;
1367       if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1368         // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1369         // describe stack slot locations.
1370         //
1371         // Consider "int x = 0; int *px = &x;". There are two kinds of
1372         // interesting debug values here after optimization:
1373         //
1374         //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
1375         //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1376         //
1377         // Both describe the direct values of their associated variables.
1378         Dependencies.push_back(N.getNode());
1379         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1380         continue;
1381       }
1382       LocationOps.emplace_back(
1383           SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1384       continue;
1385     }
1386 
1387     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1388     // Special rules apply for the first dbg.values of parameter variables in a
1389     // function. Identify them by the fact they reference Argument Values, that
1390     // they're parameters, and they are parameters of the current function. We
1391     // need to let them dangle until they get an SDNode.
1392     bool IsParamOfFunc =
1393         isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();
1394     if (IsParamOfFunc)
1395       return false;
1396 
1397     // The value is not used in this block yet (or it would have an SDNode).
1398     // We still want the value to appear for the user if possible -- if it has
1399     // an associated VReg, we can refer to that instead.
1400     auto VMI = FuncInfo.ValueMap.find(V);
1401     if (VMI != FuncInfo.ValueMap.end()) {
1402       unsigned Reg = VMI->second;
1403       // If this is a PHI node, it may be split up into several MI PHI nodes
1404       // (in FunctionLoweringInfo::set).
1405       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1406                        V->getType(), None);
1407       if (RFV.occupiesMultipleRegs()) {
1408         // FIXME: We could potentially support variadic dbg_values here.
1409         if (IsVariadic)
1410           return false;
1411         unsigned Offset = 0;
1412         unsigned BitsToDescribe = 0;
1413         if (auto VarSize = Var->getSizeInBits())
1414           BitsToDescribe = *VarSize;
1415         if (auto Fragment = Expr->getFragmentInfo())
1416           BitsToDescribe = Fragment->SizeInBits;
1417         for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1418           // Bail out if all bits are described already.
1419           if (Offset >= BitsToDescribe)
1420             break;
1421           // TODO: handle scalable vectors.
1422           unsigned RegisterSize = RegAndSize.second;
1423           unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1424                                       ? BitsToDescribe - Offset
1425                                       : RegisterSize;
1426           auto FragmentExpr = DIExpression::createFragmentExpression(
1427               Expr, Offset, FragmentSize);
1428           if (!FragmentExpr)
1429             continue;
1430           SDDbgValue *SDV = DAG.getVRegDbgValue(
1431               Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);
1432           DAG.AddDbgValue(SDV, false);
1433           Offset += RegisterSize;
1434         }
1435         return true;
1436       }
1437       // We can use simple vreg locations for variadic dbg_values as well.
1438       LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1439       continue;
1440     }
1441     // We failed to create a SDDbgOperand for V.
1442     return false;
1443   }
1444 
1445   // We have created a SDDbgOperand for each Value in Values.
1446   // Should use Order instead of SDNodeOrder?
1447   assert(!LocationOps.empty());
1448   SDDbgValue *SDV =
1449       DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1450                           /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);
1451   DAG.AddDbgValue(SDV, /*isParameter=*/false);
1452   return true;
1453 }
1454 
1455 void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1456   // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1457   for (auto &Pair : DanglingDebugInfoMap)
1458     for (auto &DDI : Pair.second)
1459       salvageUnresolvedDbgValue(DDI);
1460   clearDanglingDebugInfo();
1461 }
1462 
1463 /// getCopyFromRegs - If there was virtual register allocated for the value V
1464 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1465 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1466   DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
1467   SDValue Result;
1468 
1469   if (It != FuncInfo.ValueMap.end()) {
1470     Register InReg = It->second;
1471 
1472     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1473                      DAG.getDataLayout(), InReg, Ty,
1474                      None); // This is not an ABI copy.
1475     SDValue Chain = DAG.getEntryNode();
1476     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1477                                  V);
1478     resolveDanglingDebugInfo(V, Result);
1479   }
1480 
1481   return Result;
1482 }
1483 
1484 /// getValue - Return an SDValue for the given Value.
1485 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1486   // If we already have an SDValue for this value, use it. It's important
1487   // to do this first, so that we don't create a CopyFromReg if we already
1488   // have a regular SDValue.
1489   SDValue &N = NodeMap[V];
1490   if (N.getNode()) return N;
1491 
1492   // If there's a virtual register allocated and initialized for this
1493   // value, use it.
1494   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1495     return copyFromReg;
1496 
1497   // Otherwise create a new SDValue and remember it.
1498   SDValue Val = getValueImpl(V);
1499   NodeMap[V] = Val;
1500   resolveDanglingDebugInfo(V, Val);
1501   return Val;
1502 }
1503 
1504 /// getNonRegisterValue - Return an SDValue for the given Value, but
1505 /// don't look in FuncInfo.ValueMap for a virtual register.
1506 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1507   // If we already have an SDValue for this value, use it.
1508   SDValue &N = NodeMap[V];
1509   if (N.getNode()) {
1510     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1511       // Remove the debug location from the node as the node is about to be used
1512       // in a location which may differ from the original debug location.  This
1513       // is relevant to Constant and ConstantFP nodes because they can appear
1514       // as constant expressions inside PHI nodes.
1515       N->setDebugLoc(DebugLoc());
1516     }
1517     return N;
1518   }
1519 
1520   // Otherwise create a new SDValue and remember it.
1521   SDValue Val = getValueImpl(V);
1522   NodeMap[V] = Val;
1523   resolveDanglingDebugInfo(V, Val);
1524   return Val;
1525 }
1526 
1527 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1528 /// Create an SDValue for the given value.
1529 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1530   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1531 
1532   if (const Constant *C = dyn_cast<Constant>(V)) {
1533     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1534 
1535     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1536       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1537 
1538     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1539       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1540 
1541     if (isa<ConstantPointerNull>(C)) {
1542       unsigned AS = V->getType()->getPointerAddressSpace();
1543       return DAG.getConstant(0, getCurSDLoc(),
1544                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1545     }
1546 
1547     if (match(C, m_VScale(DAG.getDataLayout())))
1548       return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1549 
1550     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1551       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1552 
1553     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1554       return DAG.getUNDEF(VT);
1555 
1556     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1557       visit(CE->getOpcode(), *CE);
1558       SDValue N1 = NodeMap[V];
1559       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1560       return N1;
1561     }
1562 
1563     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1564       SmallVector<SDValue, 4> Constants;
1565       for (const Use &U : C->operands()) {
1566         SDNode *Val = getValue(U).getNode();
1567         // If the operand is an empty aggregate, there are no values.
1568         if (!Val) continue;
1569         // Add each leaf value from the operand to the Constants list
1570         // to form a flattened list of all the values.
1571         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1572           Constants.push_back(SDValue(Val, i));
1573       }
1574 
1575       return DAG.getMergeValues(Constants, getCurSDLoc());
1576     }
1577 
1578     if (const ConstantDataSequential *CDS =
1579           dyn_cast<ConstantDataSequential>(C)) {
1580       SmallVector<SDValue, 4> Ops;
1581       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1582         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1583         // Add each leaf value from the operand to the Constants list
1584         // to form a flattened list of all the values.
1585         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1586           Ops.push_back(SDValue(Val, i));
1587       }
1588 
1589       if (isa<ArrayType>(CDS->getType()))
1590         return DAG.getMergeValues(Ops, getCurSDLoc());
1591       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1592     }
1593 
1594     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1595       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1596              "Unknown struct or array constant!");
1597 
1598       SmallVector<EVT, 4> ValueVTs;
1599       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1600       unsigned NumElts = ValueVTs.size();
1601       if (NumElts == 0)
1602         return SDValue(); // empty struct
1603       SmallVector<SDValue, 4> Constants(NumElts);
1604       for (unsigned i = 0; i != NumElts; ++i) {
1605         EVT EltVT = ValueVTs[i];
1606         if (isa<UndefValue>(C))
1607           Constants[i] = DAG.getUNDEF(EltVT);
1608         else if (EltVT.isFloatingPoint())
1609           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1610         else
1611           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1612       }
1613 
1614       return DAG.getMergeValues(Constants, getCurSDLoc());
1615     }
1616 
1617     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1618       return DAG.getBlockAddress(BA, VT);
1619 
1620     if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1621       return getValue(Equiv->getGlobalValue());
1622 
1623     if (const auto *NC = dyn_cast<NoCFIValue>(C))
1624       return getValue(NC->getGlobalValue());
1625 
1626     VectorType *VecTy = cast<VectorType>(V->getType());
1627 
1628     // Now that we know the number and type of the elements, get that number of
1629     // elements into the Ops array based on what kind of constant it is.
1630     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1631       SmallVector<SDValue, 16> Ops;
1632       unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1633       for (unsigned i = 0; i != NumElements; ++i)
1634         Ops.push_back(getValue(CV->getOperand(i)));
1635 
1636       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1637     }
1638 
1639     if (isa<ConstantAggregateZero>(C)) {
1640       EVT EltVT =
1641           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1642 
1643       SDValue Op;
1644       if (EltVT.isFloatingPoint())
1645         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1646       else
1647         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1648 
1649       if (isa<ScalableVectorType>(VecTy))
1650         return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);
1651 
1652       SmallVector<SDValue, 16> Ops;
1653       Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);
1654       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1655     }
1656 
1657     llvm_unreachable("Unknown vector constant");
1658   }
1659 
1660   // If this is a static alloca, generate it as the frameindex instead of
1661   // computation.
1662   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1663     DenseMap<const AllocaInst*, int>::iterator SI =
1664       FuncInfo.StaticAllocaMap.find(AI);
1665     if (SI != FuncInfo.StaticAllocaMap.end())
1666       return DAG.getFrameIndex(SI->second,
1667                                TLI.getFrameIndexTy(DAG.getDataLayout()));
1668   }
1669 
1670   // If this is an instruction which fast-isel has deferred, select it now.
1671   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1672     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1673 
1674     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1675                      Inst->getType(), None);
1676     SDValue Chain = DAG.getEntryNode();
1677     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1678   }
1679 
1680   if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1681     return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1682 
1683   if (const auto *BB = dyn_cast<BasicBlock>(V))
1684     return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1685 
1686   llvm_unreachable("Can't get register for value!");
1687 }
1688 
1689 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1690   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1691   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1692   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1693   bool IsSEH = isAsynchronousEHPersonality(Pers);
1694   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1695   if (!IsSEH)
1696     CatchPadMBB->setIsEHScopeEntry();
1697   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1698   if (IsMSVCCXX || IsCoreCLR)
1699     CatchPadMBB->setIsEHFuncletEntry();
1700 }
1701 
1702 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1703   // Update machine-CFG edge.
1704   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1705   FuncInfo.MBB->addSuccessor(TargetMBB);
1706   TargetMBB->setIsEHCatchretTarget(true);
1707   DAG.getMachineFunction().setHasEHCatchret(true);
1708 
1709   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1710   bool IsSEH = isAsynchronousEHPersonality(Pers);
1711   if (IsSEH) {
1712     // If this is not a fall-through branch or optimizations are switched off,
1713     // emit the branch.
1714     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1715         TM.getOptLevel() == CodeGenOpt::None)
1716       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1717                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1718     return;
1719   }
1720 
1721   // Figure out the funclet membership for the catchret's successor.
1722   // This will be used by the FuncletLayout pass to determine how to order the
1723   // BB's.
1724   // A 'catchret' returns to the outer scope's color.
1725   Value *ParentPad = I.getCatchSwitchParentPad();
1726   const BasicBlock *SuccessorColor;
1727   if (isa<ConstantTokenNone>(ParentPad))
1728     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1729   else
1730     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1731   assert(SuccessorColor && "No parent funclet for catchret!");
1732   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1733   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1734 
1735   // Create the terminator node.
1736   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1737                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1738                             DAG.getBasicBlock(SuccessorColorMBB));
1739   DAG.setRoot(Ret);
1740 }
1741 
1742 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1743   // Don't emit any special code for the cleanuppad instruction. It just marks
1744   // the start of an EH scope/funclet.
1745   FuncInfo.MBB->setIsEHScopeEntry();
1746   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1747   if (Pers != EHPersonality::Wasm_CXX) {
1748     FuncInfo.MBB->setIsEHFuncletEntry();
1749     FuncInfo.MBB->setIsCleanupFuncletEntry();
1750   }
1751 }
1752 
1753 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1754 // not match, it is OK to add only the first unwind destination catchpad to the
1755 // successors, because there will be at least one invoke instruction within the
1756 // catch scope that points to the next unwind destination, if one exists, so
1757 // CFGSort cannot mess up with BB sorting order.
1758 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1759 // call within them, and catchpads only consisting of 'catch (...)' have a
1760 // '__cxa_end_catch' call within them, both of which generate invokes in case
1761 // the next unwind destination exists, i.e., the next unwind destination is not
1762 // the caller.)
1763 //
1764 // Having at most one EH pad successor is also simpler and helps later
1765 // transformations.
1766 //
1767 // For example,
1768 // current:
1769 //   invoke void @foo to ... unwind label %catch.dispatch
1770 // catch.dispatch:
1771 //   %0 = catchswitch within ... [label %catch.start] unwind label %next
1772 // catch.start:
1773 //   ...
1774 //   ... in this BB or some other child BB dominated by this BB there will be an
1775 //   invoke that points to 'next' BB as an unwind destination
1776 //
1777 // next: ; We don't need to add this to 'current' BB's successor
1778 //   ...
1779 static void findWasmUnwindDestinations(
1780     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1781     BranchProbability Prob,
1782     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1783         &UnwindDests) {
1784   while (EHPadBB) {
1785     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1786     if (isa<CleanupPadInst>(Pad)) {
1787       // Stop on cleanup pads.
1788       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1789       UnwindDests.back().first->setIsEHScopeEntry();
1790       break;
1791     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1792       // Add the catchpad handlers to the possible destinations. We don't
1793       // continue to the unwind destination of the catchswitch for wasm.
1794       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1795         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1796         UnwindDests.back().first->setIsEHScopeEntry();
1797       }
1798       break;
1799     } else {
1800       continue;
1801     }
1802   }
1803 }
1804 
1805 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1806 /// many places it could ultimately go. In the IR, we have a single unwind
1807 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1808 /// This function skips over imaginary basic blocks that hold catchswitch
1809 /// instructions, and finds all the "real" machine
1810 /// basic block destinations. As those destinations may not be successors of
1811 /// EHPadBB, here we also calculate the edge probability to those destinations.
1812 /// The passed-in Prob is the edge probability to EHPadBB.
1813 static void findUnwindDestinations(
1814     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1815     BranchProbability Prob,
1816     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1817         &UnwindDests) {
1818   EHPersonality Personality =
1819     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1820   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1821   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1822   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1823   bool IsSEH = isAsynchronousEHPersonality(Personality);
1824 
1825   if (IsWasmCXX) {
1826     findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1827     assert(UnwindDests.size() <= 1 &&
1828            "There should be at most one unwind destination for wasm");
1829     return;
1830   }
1831 
1832   while (EHPadBB) {
1833     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1834     BasicBlock *NewEHPadBB = nullptr;
1835     if (isa<LandingPadInst>(Pad)) {
1836       // Stop on landingpads. They are not funclets.
1837       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1838       break;
1839     } else if (isa<CleanupPadInst>(Pad)) {
1840       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1841       // personalities.
1842       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1843       UnwindDests.back().first->setIsEHScopeEntry();
1844       UnwindDests.back().first->setIsEHFuncletEntry();
1845       break;
1846     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1847       // Add the catchpad handlers to the possible destinations.
1848       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1849         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1850         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1851         if (IsMSVCCXX || IsCoreCLR)
1852           UnwindDests.back().first->setIsEHFuncletEntry();
1853         if (!IsSEH)
1854           UnwindDests.back().first->setIsEHScopeEntry();
1855       }
1856       NewEHPadBB = CatchSwitch->getUnwindDest();
1857     } else {
1858       continue;
1859     }
1860 
1861     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1862     if (BPI && NewEHPadBB)
1863       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1864     EHPadBB = NewEHPadBB;
1865   }
1866 }
1867 
1868 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1869   // Update successor info.
1870   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1871   auto UnwindDest = I.getUnwindDest();
1872   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1873   BranchProbability UnwindDestProb =
1874       (BPI && UnwindDest)
1875           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1876           : BranchProbability::getZero();
1877   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1878   for (auto &UnwindDest : UnwindDests) {
1879     UnwindDest.first->setIsEHPad();
1880     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1881   }
1882   FuncInfo.MBB->normalizeSuccProbs();
1883 
1884   // Create the terminator node.
1885   SDValue Ret =
1886       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1887   DAG.setRoot(Ret);
1888 }
1889 
1890 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1891   report_fatal_error("visitCatchSwitch not yet implemented!");
1892 }
1893 
1894 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1895   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1896   auto &DL = DAG.getDataLayout();
1897   SDValue Chain = getControlRoot();
1898   SmallVector<ISD::OutputArg, 8> Outs;
1899   SmallVector<SDValue, 8> OutVals;
1900 
1901   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1902   // lower
1903   //
1904   //   %val = call <ty> @llvm.experimental.deoptimize()
1905   //   ret <ty> %val
1906   //
1907   // differently.
1908   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1909     LowerDeoptimizingReturn();
1910     return;
1911   }
1912 
1913   if (!FuncInfo.CanLowerReturn) {
1914     unsigned DemoteReg = FuncInfo.DemoteRegister;
1915     const Function *F = I.getParent()->getParent();
1916 
1917     // Emit a store of the return value through the virtual register.
1918     // Leave Outs empty so that LowerReturn won't try to load return
1919     // registers the usual way.
1920     SmallVector<EVT, 1> PtrValueVTs;
1921     ComputeValueVTs(TLI, DL,
1922                     F->getReturnType()->getPointerTo(
1923                         DAG.getDataLayout().getAllocaAddrSpace()),
1924                     PtrValueVTs);
1925 
1926     SDValue RetPtr =
1927         DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
1928     SDValue RetOp = getValue(I.getOperand(0));
1929 
1930     SmallVector<EVT, 4> ValueVTs, MemVTs;
1931     SmallVector<uint64_t, 4> Offsets;
1932     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1933                     &Offsets);
1934     unsigned NumValues = ValueVTs.size();
1935 
1936     SmallVector<SDValue, 4> Chains(NumValues);
1937     Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1938     for (unsigned i = 0; i != NumValues; ++i) {
1939       // An aggregate return value cannot wrap around the address space, so
1940       // offsets to its parts don't wrap either.
1941       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
1942                                            TypeSize::Fixed(Offsets[i]));
1943 
1944       SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1945       if (MemVTs[i] != ValueVTs[i])
1946         Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1947       Chains[i] = DAG.getStore(
1948           Chain, getCurSDLoc(), Val,
1949           // FIXME: better loc info would be nice.
1950           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
1951           commonAlignment(BaseAlign, Offsets[i]));
1952     }
1953 
1954     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1955                         MVT::Other, Chains);
1956   } else if (I.getNumOperands() != 0) {
1957     SmallVector<EVT, 4> ValueVTs;
1958     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1959     unsigned NumValues = ValueVTs.size();
1960     if (NumValues) {
1961       SDValue RetOp = getValue(I.getOperand(0));
1962 
1963       const Function *F = I.getParent()->getParent();
1964 
1965       bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1966           I.getOperand(0)->getType(), F->getCallingConv(),
1967           /*IsVarArg*/ false, DL);
1968 
1969       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1970       if (F->getAttributes().hasRetAttr(Attribute::SExt))
1971         ExtendKind = ISD::SIGN_EXTEND;
1972       else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
1973         ExtendKind = ISD::ZERO_EXTEND;
1974 
1975       LLVMContext &Context = F->getContext();
1976       bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
1977 
1978       for (unsigned j = 0; j != NumValues; ++j) {
1979         EVT VT = ValueVTs[j];
1980 
1981         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1982           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1983 
1984         CallingConv::ID CC = F->getCallingConv();
1985 
1986         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1987         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1988         SmallVector<SDValue, 4> Parts(NumParts);
1989         getCopyToParts(DAG, getCurSDLoc(),
1990                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1991                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1992 
1993         // 'inreg' on function refers to return value
1994         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1995         if (RetInReg)
1996           Flags.setInReg();
1997 
1998         if (I.getOperand(0)->getType()->isPointerTy()) {
1999           Flags.setPointer();
2000           Flags.setPointerAddrSpace(
2001               cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2002         }
2003 
2004         if (NeedsRegBlock) {
2005           Flags.setInConsecutiveRegs();
2006           if (j == NumValues - 1)
2007             Flags.setInConsecutiveRegsLast();
2008         }
2009 
2010         // Propagate extension type if any
2011         if (ExtendKind == ISD::SIGN_EXTEND)
2012           Flags.setSExt();
2013         else if (ExtendKind == ISD::ZERO_EXTEND)
2014           Flags.setZExt();
2015 
2016         for (unsigned i = 0; i < NumParts; ++i) {
2017           Outs.push_back(ISD::OutputArg(Flags,
2018                                         Parts[i].getValueType().getSimpleVT(),
2019                                         VT, /*isfixed=*/true, 0, 0));
2020           OutVals.push_back(Parts[i]);
2021         }
2022       }
2023     }
2024   }
2025 
2026   // Push in swifterror virtual register as the last element of Outs. This makes
2027   // sure swifterror virtual register will be returned in the swifterror
2028   // physical register.
2029   const Function *F = I.getParent()->getParent();
2030   if (TLI.supportSwiftError() &&
2031       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2032     assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2033     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2034     Flags.setSwiftError();
2035     Outs.push_back(ISD::OutputArg(
2036         Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2037         /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2038     // Create SDNode for the swifterror virtual register.
2039     OutVals.push_back(
2040         DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2041                             &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2042                         EVT(TLI.getPointerTy(DL))));
2043   }
2044 
2045   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2046   CallingConv::ID CallConv =
2047     DAG.getMachineFunction().getFunction().getCallingConv();
2048   Chain = DAG.getTargetLoweringInfo().LowerReturn(
2049       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2050 
2051   // Verify that the target's LowerReturn behaved as expected.
2052   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2053          "LowerReturn didn't return a valid chain!");
2054 
2055   // Update the DAG with the new chain value resulting from return lowering.
2056   DAG.setRoot(Chain);
2057 }
2058 
2059 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2060 /// created for it, emit nodes to copy the value into the virtual
2061 /// registers.
2062 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2063   // Skip empty types
2064   if (V->getType()->isEmptyTy())
2065     return;
2066 
2067   DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
2068   if (VMI != FuncInfo.ValueMap.end()) {
2069     assert(!V->use_empty() && "Unused value assigned virtual registers!");
2070     CopyValueToVirtualRegister(V, VMI->second);
2071   }
2072 }
2073 
2074 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2075 /// the current basic block, add it to ValueMap now so that we'll get a
2076 /// CopyTo/FromReg.
2077 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2078   // No need to export constants.
2079   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2080 
2081   // Already exported?
2082   if (FuncInfo.isExportedInst(V)) return;
2083 
2084   unsigned Reg = FuncInfo.InitializeRegForValue(V);
2085   CopyValueToVirtualRegister(V, Reg);
2086 }
2087 
2088 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2089                                                      const BasicBlock *FromBB) {
2090   // The operands of the setcc have to be in this block.  We don't know
2091   // how to export them from some other block.
2092   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2093     // Can export from current BB.
2094     if (VI->getParent() == FromBB)
2095       return true;
2096 
2097     // Is already exported, noop.
2098     return FuncInfo.isExportedInst(V);
2099   }
2100 
2101   // If this is an argument, we can export it if the BB is the entry block or
2102   // if it is already exported.
2103   if (isa<Argument>(V)) {
2104     if (FromBB->isEntryBlock())
2105       return true;
2106 
2107     // Otherwise, can only export this if it is already exported.
2108     return FuncInfo.isExportedInst(V);
2109   }
2110 
2111   // Otherwise, constants can always be exported.
2112   return true;
2113 }
2114 
2115 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2116 BranchProbability
2117 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2118                                         const MachineBasicBlock *Dst) const {
2119   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2120   const BasicBlock *SrcBB = Src->getBasicBlock();
2121   const BasicBlock *DstBB = Dst->getBasicBlock();
2122   if (!BPI) {
2123     // If BPI is not available, set the default probability as 1 / N, where N is
2124     // the number of successors.
2125     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2126     return BranchProbability(1, SuccSize);
2127   }
2128   return BPI->getEdgeProbability(SrcBB, DstBB);
2129 }
2130 
2131 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2132                                                MachineBasicBlock *Dst,
2133                                                BranchProbability Prob) {
2134   if (!FuncInfo.BPI)
2135     Src->addSuccessorWithoutProb(Dst);
2136   else {
2137     if (Prob.isUnknown())
2138       Prob = getEdgeProbability(Src, Dst);
2139     Src->addSuccessor(Dst, Prob);
2140   }
2141 }
2142 
2143 static bool InBlock(const Value *V, const BasicBlock *BB) {
2144   if (const Instruction *I = dyn_cast<Instruction>(V))
2145     return I->getParent() == BB;
2146   return true;
2147 }
2148 
2149 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2150 /// This function emits a branch and is used at the leaves of an OR or an
2151 /// AND operator tree.
2152 void
2153 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2154                                                   MachineBasicBlock *TBB,
2155                                                   MachineBasicBlock *FBB,
2156                                                   MachineBasicBlock *CurBB,
2157                                                   MachineBasicBlock *SwitchBB,
2158                                                   BranchProbability TProb,
2159                                                   BranchProbability FProb,
2160                                                   bool InvertCond) {
2161   const BasicBlock *BB = CurBB->getBasicBlock();
2162 
2163   // If the leaf of the tree is a comparison, merge the condition into
2164   // the caseblock.
2165   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2166     // The operands of the cmp have to be in this block.  We don't know
2167     // how to export them from some other block.  If this is the first block
2168     // of the sequence, no exporting is needed.
2169     if (CurBB == SwitchBB ||
2170         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2171          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2172       ISD::CondCode Condition;
2173       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2174         ICmpInst::Predicate Pred =
2175             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2176         Condition = getICmpCondCode(Pred);
2177       } else {
2178         const FCmpInst *FC = cast<FCmpInst>(Cond);
2179         FCmpInst::Predicate Pred =
2180             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2181         Condition = getFCmpCondCode(Pred);
2182         if (TM.Options.NoNaNsFPMath)
2183           Condition = getFCmpCodeWithoutNaN(Condition);
2184       }
2185 
2186       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2187                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2188       SL->SwitchCases.push_back(CB);
2189       return;
2190     }
2191   }
2192 
2193   // Create a CaseBlock record representing this branch.
2194   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2195   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2196                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2197   SL->SwitchCases.push_back(CB);
2198 }
2199 
2200 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2201                                                MachineBasicBlock *TBB,
2202                                                MachineBasicBlock *FBB,
2203                                                MachineBasicBlock *CurBB,
2204                                                MachineBasicBlock *SwitchBB,
2205                                                Instruction::BinaryOps Opc,
2206                                                BranchProbability TProb,
2207                                                BranchProbability FProb,
2208                                                bool InvertCond) {
2209   // Skip over not part of the tree and remember to invert op and operands at
2210   // next level.
2211   Value *NotCond;
2212   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2213       InBlock(NotCond, CurBB->getBasicBlock())) {
2214     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2215                          !InvertCond);
2216     return;
2217   }
2218 
2219   const Instruction *BOp = dyn_cast<Instruction>(Cond);
2220   const Value *BOpOp0, *BOpOp1;
2221   // Compute the effective opcode for Cond, taking into account whether it needs
2222   // to be inverted, e.g.
2223   //   and (not (or A, B)), C
2224   // gets lowered as
2225   //   and (and (not A, not B), C)
2226   Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2227   if (BOp) {
2228     BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2229                ? Instruction::And
2230                : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2231                       ? Instruction::Or
2232                       : (Instruction::BinaryOps)0);
2233     if (InvertCond) {
2234       if (BOpc == Instruction::And)
2235         BOpc = Instruction::Or;
2236       else if (BOpc == Instruction::Or)
2237         BOpc = Instruction::And;
2238     }
2239   }
2240 
2241   // If this node is not part of the or/and tree, emit it as a branch.
2242   // Note that all nodes in the tree should have same opcode.
2243   bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2244   if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2245       !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2246       !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2247     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2248                                  TProb, FProb, InvertCond);
2249     return;
2250   }
2251 
2252   //  Create TmpBB after CurBB.
2253   MachineFunction::iterator BBI(CurBB);
2254   MachineFunction &MF = DAG.getMachineFunction();
2255   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2256   CurBB->getParent()->insert(++BBI, TmpBB);
2257 
2258   if (Opc == Instruction::Or) {
2259     // Codegen X | Y as:
2260     // BB1:
2261     //   jmp_if_X TBB
2262     //   jmp TmpBB
2263     // TmpBB:
2264     //   jmp_if_Y TBB
2265     //   jmp FBB
2266     //
2267 
2268     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2269     // The requirement is that
2270     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2271     //     = TrueProb for original BB.
2272     // Assuming the original probabilities are A and B, one choice is to set
2273     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2274     // A/(1+B) and 2B/(1+B). This choice assumes that
2275     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2276     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2277     // TmpBB, but the math is more complicated.
2278 
2279     auto NewTrueProb = TProb / 2;
2280     auto NewFalseProb = TProb / 2 + FProb;
2281     // Emit the LHS condition.
2282     FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2283                          NewFalseProb, InvertCond);
2284 
2285     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2286     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2287     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2288     // Emit the RHS condition into TmpBB.
2289     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2290                          Probs[1], InvertCond);
2291   } else {
2292     assert(Opc == Instruction::And && "Unknown merge op!");
2293     // Codegen X & Y as:
2294     // BB1:
2295     //   jmp_if_X TmpBB
2296     //   jmp FBB
2297     // TmpBB:
2298     //   jmp_if_Y TBB
2299     //   jmp FBB
2300     //
2301     //  This requires creation of TmpBB after CurBB.
2302 
2303     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2304     // The requirement is that
2305     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2306     //     = FalseProb for original BB.
2307     // Assuming the original probabilities are A and B, one choice is to set
2308     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2309     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2310     // TrueProb for BB1 * FalseProb for TmpBB.
2311 
2312     auto NewTrueProb = TProb + FProb / 2;
2313     auto NewFalseProb = FProb / 2;
2314     // Emit the LHS condition.
2315     FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2316                          NewFalseProb, InvertCond);
2317 
2318     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2319     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2320     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2321     // Emit the RHS condition into TmpBB.
2322     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2323                          Probs[1], InvertCond);
2324   }
2325 }
2326 
2327 /// If the set of cases should be emitted as a series of branches, return true.
2328 /// If we should emit this as a bunch of and/or'd together conditions, return
2329 /// false.
2330 bool
2331 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2332   if (Cases.size() != 2) return true;
2333 
2334   // If this is two comparisons of the same values or'd or and'd together, they
2335   // will get folded into a single comparison, so don't emit two blocks.
2336   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2337        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2338       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2339        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2340     return false;
2341   }
2342 
2343   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2344   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2345   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2346       Cases[0].CC == Cases[1].CC &&
2347       isa<Constant>(Cases[0].CmpRHS) &&
2348       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2349     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2350       return false;
2351     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2352       return false;
2353   }
2354 
2355   return true;
2356 }
2357 
2358 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2359   MachineBasicBlock *BrMBB = FuncInfo.MBB;
2360 
2361   // Update machine-CFG edges.
2362   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2363 
2364   if (I.isUnconditional()) {
2365     // Update machine-CFG edges.
2366     BrMBB->addSuccessor(Succ0MBB);
2367 
2368     // If this is not a fall-through branch or optimizations are switched off,
2369     // emit the branch.
2370     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2371       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2372                               MVT::Other, getControlRoot(),
2373                               DAG.getBasicBlock(Succ0MBB)));
2374 
2375     return;
2376   }
2377 
2378   // If this condition is one of the special cases we handle, do special stuff
2379   // now.
2380   const Value *CondVal = I.getCondition();
2381   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2382 
2383   // If this is a series of conditions that are or'd or and'd together, emit
2384   // this as a sequence of branches instead of setcc's with and/or operations.
2385   // As long as jumps are not expensive (exceptions for multi-use logic ops,
2386   // unpredictable branches, and vector extracts because those jumps are likely
2387   // expensive for any target), this should improve performance.
2388   // For example, instead of something like:
2389   //     cmp A, B
2390   //     C = seteq
2391   //     cmp D, E
2392   //     F = setle
2393   //     or C, F
2394   //     jnz foo
2395   // Emit:
2396   //     cmp A, B
2397   //     je foo
2398   //     cmp D, E
2399   //     jle foo
2400   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2401   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2402       BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2403     Value *Vec;
2404     const Value *BOp0, *BOp1;
2405     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2406     if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2407       Opcode = Instruction::And;
2408     else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2409       Opcode = Instruction::Or;
2410 
2411     if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2412                     match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2413       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2414                            getEdgeProbability(BrMBB, Succ0MBB),
2415                            getEdgeProbability(BrMBB, Succ1MBB),
2416                            /*InvertCond=*/false);
2417       // If the compares in later blocks need to use values not currently
2418       // exported from this block, export them now.  This block should always
2419       // be the first entry.
2420       assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2421 
2422       // Allow some cases to be rejected.
2423       if (ShouldEmitAsBranches(SL->SwitchCases)) {
2424         for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2425           ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2426           ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2427         }
2428 
2429         // Emit the branch for this block.
2430         visitSwitchCase(SL->SwitchCases[0], BrMBB);
2431         SL->SwitchCases.erase(SL->SwitchCases.begin());
2432         return;
2433       }
2434 
2435       // Okay, we decided not to do this, remove any inserted MBB's and clear
2436       // SwitchCases.
2437       for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2438         FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2439 
2440       SL->SwitchCases.clear();
2441     }
2442   }
2443 
2444   // Create a CaseBlock record representing this branch.
2445   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2446                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2447 
2448   // Use visitSwitchCase to actually insert the fast branch sequence for this
2449   // cond branch.
2450   visitSwitchCase(CB, BrMBB);
2451 }
2452 
2453 /// visitSwitchCase - Emits the necessary code to represent a single node in
2454 /// the binary search tree resulting from lowering a switch instruction.
2455 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2456                                           MachineBasicBlock *SwitchBB) {
2457   SDValue Cond;
2458   SDValue CondLHS = getValue(CB.CmpLHS);
2459   SDLoc dl = CB.DL;
2460 
2461   if (CB.CC == ISD::SETTRUE) {
2462     // Branch or fall through to TrueBB.
2463     addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2464     SwitchBB->normalizeSuccProbs();
2465     if (CB.TrueBB != NextBlock(SwitchBB)) {
2466       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2467                               DAG.getBasicBlock(CB.TrueBB)));
2468     }
2469     return;
2470   }
2471 
2472   auto &TLI = DAG.getTargetLoweringInfo();
2473   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2474 
2475   // Build the setcc now.
2476   if (!CB.CmpMHS) {
2477     // Fold "(X == true)" to X and "(X == false)" to !X to
2478     // handle common cases produced by branch lowering.
2479     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2480         CB.CC == ISD::SETEQ)
2481       Cond = CondLHS;
2482     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2483              CB.CC == ISD::SETEQ) {
2484       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2485       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2486     } else {
2487       SDValue CondRHS = getValue(CB.CmpRHS);
2488 
2489       // If a pointer's DAG type is larger than its memory type then the DAG
2490       // values are zero-extended. This breaks signed comparisons so truncate
2491       // back to the underlying type before doing the compare.
2492       if (CondLHS.getValueType() != MemVT) {
2493         CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2494         CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2495       }
2496       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2497     }
2498   } else {
2499     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2500 
2501     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2502     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2503 
2504     SDValue CmpOp = getValue(CB.CmpMHS);
2505     EVT VT = CmpOp.getValueType();
2506 
2507     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2508       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2509                           ISD::SETLE);
2510     } else {
2511       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2512                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2513       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2514                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2515     }
2516   }
2517 
2518   // Update successor info
2519   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2520   // TrueBB and FalseBB are always different unless the incoming IR is
2521   // degenerate. This only happens when running llc on weird IR.
2522   if (CB.TrueBB != CB.FalseBB)
2523     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2524   SwitchBB->normalizeSuccProbs();
2525 
2526   // If the lhs block is the next block, invert the condition so that we can
2527   // fall through to the lhs instead of the rhs block.
2528   if (CB.TrueBB == NextBlock(SwitchBB)) {
2529     std::swap(CB.TrueBB, CB.FalseBB);
2530     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2531     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2532   }
2533 
2534   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2535                                MVT::Other, getControlRoot(), Cond,
2536                                DAG.getBasicBlock(CB.TrueBB));
2537 
2538   // Insert the false branch. Do this even if it's a fall through branch,
2539   // this makes it easier to do DAG optimizations which require inverting
2540   // the branch condition.
2541   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2542                        DAG.getBasicBlock(CB.FalseBB));
2543 
2544   DAG.setRoot(BrCond);
2545 }
2546 
2547 /// visitJumpTable - Emit JumpTable node in the current MBB
2548 void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2549   // Emit the code for the jump table
2550   assert(JT.Reg != -1U && "Should lower JT Header first!");
2551   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2552   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2553                                      JT.Reg, PTy);
2554   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2555   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2556                                     MVT::Other, Index.getValue(1),
2557                                     Table, Index);
2558   DAG.setRoot(BrJumpTable);
2559 }
2560 
2561 /// visitJumpTableHeader - This function emits necessary code to produce index
2562 /// in the JumpTable from switch case.
2563 void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2564                                                JumpTableHeader &JTH,
2565                                                MachineBasicBlock *SwitchBB) {
2566   SDLoc dl = getCurSDLoc();
2567 
2568   // Subtract the lowest switch case value from the value being switched on.
2569   SDValue SwitchOp = getValue(JTH.SValue);
2570   EVT VT = SwitchOp.getValueType();
2571   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2572                             DAG.getConstant(JTH.First, dl, VT));
2573 
2574   // The SDNode we just created, which holds the value being switched on minus
2575   // the smallest case value, needs to be copied to a virtual register so it
2576   // can be used as an index into the jump table in a subsequent basic block.
2577   // This value may be smaller or larger than the target's pointer type, and
2578   // therefore require extension or truncating.
2579   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2580   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2581 
2582   unsigned JumpTableReg =
2583       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2584   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2585                                     JumpTableReg, SwitchOp);
2586   JT.Reg = JumpTableReg;
2587 
2588   if (!JTH.FallthroughUnreachable) {
2589     // Emit the range check for the jump table, and branch to the default block
2590     // for the switch statement if the value being switched on exceeds the
2591     // largest case in the switch.
2592     SDValue CMP = DAG.getSetCC(
2593         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2594                                    Sub.getValueType()),
2595         Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2596 
2597     SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2598                                  MVT::Other, CopyTo, CMP,
2599                                  DAG.getBasicBlock(JT.Default));
2600 
2601     // Avoid emitting unnecessary branches to the next block.
2602     if (JT.MBB != NextBlock(SwitchBB))
2603       BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2604                            DAG.getBasicBlock(JT.MBB));
2605 
2606     DAG.setRoot(BrCond);
2607   } else {
2608     // Avoid emitting unnecessary branches to the next block.
2609     if (JT.MBB != NextBlock(SwitchBB))
2610       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2611                               DAG.getBasicBlock(JT.MBB)));
2612     else
2613       DAG.setRoot(CopyTo);
2614   }
2615 }
2616 
2617 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2618 /// variable if there exists one.
2619 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2620                                  SDValue &Chain) {
2621   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2622   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2623   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2624   MachineFunction &MF = DAG.getMachineFunction();
2625   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2626   MachineSDNode *Node =
2627       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2628   if (Global) {
2629     MachinePointerInfo MPInfo(Global);
2630     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2631                  MachineMemOperand::MODereferenceable;
2632     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2633         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2634     DAG.setNodeMemRefs(Node, {MemRef});
2635   }
2636   if (PtrTy != PtrMemTy)
2637     return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2638   return SDValue(Node, 0);
2639 }
2640 
2641 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2642 /// tail spliced into a stack protector check success bb.
2643 ///
2644 /// For a high level explanation of how this fits into the stack protector
2645 /// generation see the comment on the declaration of class
2646 /// StackProtectorDescriptor.
2647 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2648                                                   MachineBasicBlock *ParentBB) {
2649 
2650   // First create the loads to the guard/stack slot for the comparison.
2651   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2652   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2653   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2654 
2655   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2656   int FI = MFI.getStackProtectorIndex();
2657 
2658   SDValue Guard;
2659   SDLoc dl = getCurSDLoc();
2660   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2661   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2662   Align Align =
2663       DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2664 
2665   // Generate code to load the content of the guard slot.
2666   SDValue GuardVal = DAG.getLoad(
2667       PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2668       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2669       MachineMemOperand::MOVolatile);
2670 
2671   if (TLI.useStackGuardXorFP())
2672     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2673 
2674   // Retrieve guard check function, nullptr if instrumentation is inlined.
2675   if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2676     // The target provides a guard check function to validate the guard value.
2677     // Generate a call to that function with the content of the guard slot as
2678     // argument.
2679     FunctionType *FnTy = GuardCheckFn->getFunctionType();
2680     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2681 
2682     TargetLowering::ArgListTy Args;
2683     TargetLowering::ArgListEntry Entry;
2684     Entry.Node = GuardVal;
2685     Entry.Ty = FnTy->getParamType(0);
2686     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2687       Entry.IsInReg = true;
2688     Args.push_back(Entry);
2689 
2690     TargetLowering::CallLoweringInfo CLI(DAG);
2691     CLI.setDebugLoc(getCurSDLoc())
2692         .setChain(DAG.getEntryNode())
2693         .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2694                    getValue(GuardCheckFn), std::move(Args));
2695 
2696     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2697     DAG.setRoot(Result.second);
2698     return;
2699   }
2700 
2701   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2702   // Otherwise, emit a volatile load to retrieve the stack guard value.
2703   SDValue Chain = DAG.getEntryNode();
2704   if (TLI.useLoadStackGuardNode()) {
2705     Guard = getLoadStackGuard(DAG, dl, Chain);
2706   } else {
2707     const Value *IRGuard = TLI.getSDagStackGuard(M);
2708     SDValue GuardPtr = getValue(IRGuard);
2709 
2710     Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2711                         MachinePointerInfo(IRGuard, 0), Align,
2712                         MachineMemOperand::MOVolatile);
2713   }
2714 
2715   // Perform the comparison via a getsetcc.
2716   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2717                                                         *DAG.getContext(),
2718                                                         Guard.getValueType()),
2719                              Guard, GuardVal, ISD::SETNE);
2720 
2721   // If the guard/stackslot do not equal, branch to failure MBB.
2722   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2723                                MVT::Other, GuardVal.getOperand(0),
2724                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2725   // Otherwise branch to success MBB.
2726   SDValue Br = DAG.getNode(ISD::BR, dl,
2727                            MVT::Other, BrCond,
2728                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2729 
2730   DAG.setRoot(Br);
2731 }
2732 
2733 /// Codegen the failure basic block for a stack protector check.
2734 ///
2735 /// A failure stack protector machine basic block consists simply of a call to
2736 /// __stack_chk_fail().
2737 ///
2738 /// For a high level explanation of how this fits into the stack protector
2739 /// generation see the comment on the declaration of class
2740 /// StackProtectorDescriptor.
2741 void
2742 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2743   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2744   TargetLowering::MakeLibCallOptions CallOptions;
2745   CallOptions.setDiscardResult(true);
2746   SDValue Chain =
2747       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2748                       None, CallOptions, getCurSDLoc()).second;
2749   // On PS4, the "return address" must still be within the calling function,
2750   // even if it's at the very end, so emit an explicit TRAP here.
2751   // Passing 'true' for doesNotReturn above won't generate the trap for us.
2752   if (TM.getTargetTriple().isPS4())
2753     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2754   // WebAssembly needs an unreachable instruction after a non-returning call,
2755   // because the function return type can be different from __stack_chk_fail's
2756   // return type (void).
2757   if (TM.getTargetTriple().isWasm())
2758     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2759 
2760   DAG.setRoot(Chain);
2761 }
2762 
2763 /// visitBitTestHeader - This function emits necessary code to produce value
2764 /// suitable for "bit tests"
2765 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2766                                              MachineBasicBlock *SwitchBB) {
2767   SDLoc dl = getCurSDLoc();
2768 
2769   // Subtract the minimum value.
2770   SDValue SwitchOp = getValue(B.SValue);
2771   EVT VT = SwitchOp.getValueType();
2772   SDValue RangeSub =
2773       DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2774 
2775   // Determine the type of the test operands.
2776   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2777   bool UsePtrType = false;
2778   if (!TLI.isTypeLegal(VT)) {
2779     UsePtrType = true;
2780   } else {
2781     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2782       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2783         // Switch table case range are encoded into series of masks.
2784         // Just use pointer type, it's guaranteed to fit.
2785         UsePtrType = true;
2786         break;
2787       }
2788   }
2789   SDValue Sub = RangeSub;
2790   if (UsePtrType) {
2791     VT = TLI.getPointerTy(DAG.getDataLayout());
2792     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2793   }
2794 
2795   B.RegVT = VT.getSimpleVT();
2796   B.Reg = FuncInfo.CreateReg(B.RegVT);
2797   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2798 
2799   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2800 
2801   if (!B.FallthroughUnreachable)
2802     addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2803   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2804   SwitchBB->normalizeSuccProbs();
2805 
2806   SDValue Root = CopyTo;
2807   if (!B.FallthroughUnreachable) {
2808     // Conditional branch to the default block.
2809     SDValue RangeCmp = DAG.getSetCC(dl,
2810         TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2811                                RangeSub.getValueType()),
2812         RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2813         ISD::SETUGT);
2814 
2815     Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2816                        DAG.getBasicBlock(B.Default));
2817   }
2818 
2819   // Avoid emitting unnecessary branches to the next block.
2820   if (MBB != NextBlock(SwitchBB))
2821     Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2822 
2823   DAG.setRoot(Root);
2824 }
2825 
2826 /// visitBitTestCase - this function produces one "bit test"
2827 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2828                                            MachineBasicBlock* NextMBB,
2829                                            BranchProbability BranchProbToNext,
2830                                            unsigned Reg,
2831                                            BitTestCase &B,
2832                                            MachineBasicBlock *SwitchBB) {
2833   SDLoc dl = getCurSDLoc();
2834   MVT VT = BB.RegVT;
2835   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2836   SDValue Cmp;
2837   unsigned PopCount = countPopulation(B.Mask);
2838   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2839   if (PopCount == 1) {
2840     // Testing for a single bit; just compare the shift count with what it
2841     // would need to be to shift a 1 bit in that position.
2842     Cmp = DAG.getSetCC(
2843         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2844         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2845         ISD::SETEQ);
2846   } else if (PopCount == BB.Range) {
2847     // There is only one zero bit in the range, test for it directly.
2848     Cmp = DAG.getSetCC(
2849         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2850         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2851         ISD::SETNE);
2852   } else {
2853     // Make desired shift
2854     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2855                                     DAG.getConstant(1, dl, VT), ShiftOp);
2856 
2857     // Emit bit tests and jumps
2858     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2859                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2860     Cmp = DAG.getSetCC(
2861         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2862         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2863   }
2864 
2865   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2866   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2867   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2868   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2869   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2870   // one as they are relative probabilities (and thus work more like weights),
2871   // and hence we need to normalize them to let the sum of them become one.
2872   SwitchBB->normalizeSuccProbs();
2873 
2874   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2875                               MVT::Other, getControlRoot(),
2876                               Cmp, DAG.getBasicBlock(B.TargetBB));
2877 
2878   // Avoid emitting unnecessary branches to the next block.
2879   if (NextMBB != NextBlock(SwitchBB))
2880     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2881                         DAG.getBasicBlock(NextMBB));
2882 
2883   DAG.setRoot(BrAnd);
2884 }
2885 
2886 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2887   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2888 
2889   // Retrieve successors. Look through artificial IR level blocks like
2890   // catchswitch for successors.
2891   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2892   const BasicBlock *EHPadBB = I.getSuccessor(1);
2893 
2894   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2895   // have to do anything here to lower funclet bundles.
2896   assert(!I.hasOperandBundlesOtherThan(
2897              {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2898               LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2899               LLVMContext::OB_cfguardtarget,
2900               LLVMContext::OB_clang_arc_attachedcall}) &&
2901          "Cannot lower invokes with arbitrary operand bundles yet!");
2902 
2903   const Value *Callee(I.getCalledOperand());
2904   const Function *Fn = dyn_cast<Function>(Callee);
2905   if (isa<InlineAsm>(Callee))
2906     visitInlineAsm(I, EHPadBB);
2907   else if (Fn && Fn->isIntrinsic()) {
2908     switch (Fn->getIntrinsicID()) {
2909     default:
2910       llvm_unreachable("Cannot invoke this intrinsic");
2911     case Intrinsic::donothing:
2912       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2913     case Intrinsic::seh_try_begin:
2914     case Intrinsic::seh_scope_begin:
2915     case Intrinsic::seh_try_end:
2916     case Intrinsic::seh_scope_end:
2917       break;
2918     case Intrinsic::experimental_patchpoint_void:
2919     case Intrinsic::experimental_patchpoint_i64:
2920       visitPatchpoint(I, EHPadBB);
2921       break;
2922     case Intrinsic::experimental_gc_statepoint:
2923       LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2924       break;
2925     case Intrinsic::wasm_rethrow: {
2926       // This is usually done in visitTargetIntrinsic, but this intrinsic is
2927       // special because it can be invoked, so we manually lower it to a DAG
2928       // node here.
2929       SmallVector<SDValue, 8> Ops;
2930       Ops.push_back(getRoot()); // inchain
2931       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2932       Ops.push_back(
2933           DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2934                                 TLI.getPointerTy(DAG.getDataLayout())));
2935       SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2936       DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2937       break;
2938     }
2939     }
2940   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2941     // Currently we do not lower any intrinsic calls with deopt operand bundles.
2942     // Eventually we will support lowering the @llvm.experimental.deoptimize
2943     // intrinsic, and right now there are no plans to support other intrinsics
2944     // with deopt state.
2945     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2946   } else {
2947     LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2948   }
2949 
2950   // If the value of the invoke is used outside of its defining block, make it
2951   // available as a virtual register.
2952   // We already took care of the exported value for the statepoint instruction
2953   // during call to the LowerStatepoint.
2954   if (!isa<GCStatepointInst>(I)) {
2955     CopyToExportRegsIfNeeded(&I);
2956   }
2957 
2958   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2959   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2960   BranchProbability EHPadBBProb =
2961       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2962           : BranchProbability::getZero();
2963   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2964 
2965   // Update successor info.
2966   addSuccessorWithProb(InvokeMBB, Return);
2967   for (auto &UnwindDest : UnwindDests) {
2968     UnwindDest.first->setIsEHPad();
2969     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2970   }
2971   InvokeMBB->normalizeSuccProbs();
2972 
2973   // Drop into normal successor.
2974   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2975                           DAG.getBasicBlock(Return)));
2976 }
2977 
2978 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2979   MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2980 
2981   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2982   // have to do anything here to lower funclet bundles.
2983   assert(!I.hasOperandBundlesOtherThan(
2984              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2985          "Cannot lower callbrs with arbitrary operand bundles yet!");
2986 
2987   assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
2988   visitInlineAsm(I);
2989   CopyToExportRegsIfNeeded(&I);
2990 
2991   // Retrieve successors.
2992   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2993 
2994   // Update successor info.
2995   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
2996   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2997     MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2998     addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
2999     Target->setIsInlineAsmBrIndirectTarget();
3000   }
3001   CallBrMBB->normalizeSuccProbs();
3002 
3003   // Drop into default successor.
3004   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3005                           MVT::Other, getControlRoot(),
3006                           DAG.getBasicBlock(Return)));
3007 }
3008 
3009 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3010   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3011 }
3012 
3013 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3014   assert(FuncInfo.MBB->isEHPad() &&
3015          "Call to landingpad not in landing pad!");
3016 
3017   // If there aren't registers to copy the values into (e.g., during SjLj
3018   // exceptions), then don't bother to create these DAG nodes.
3019   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3020   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3021   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3022       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3023     return;
3024 
3025   // If landingpad's return type is token type, we don't create DAG nodes
3026   // for its exception pointer and selector value. The extraction of exception
3027   // pointer or selector value from token type landingpads is not currently
3028   // supported.
3029   if (LP.getType()->isTokenTy())
3030     return;
3031 
3032   SmallVector<EVT, 2> ValueVTs;
3033   SDLoc dl = getCurSDLoc();
3034   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3035   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3036 
3037   // Get the two live-in registers as SDValues. The physregs have already been
3038   // copied into virtual registers.
3039   SDValue Ops[2];
3040   if (FuncInfo.ExceptionPointerVirtReg) {
3041     Ops[0] = DAG.getZExtOrTrunc(
3042         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3043                            FuncInfo.ExceptionPointerVirtReg,
3044                            TLI.getPointerTy(DAG.getDataLayout())),
3045         dl, ValueVTs[0]);
3046   } else {
3047     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3048   }
3049   Ops[1] = DAG.getZExtOrTrunc(
3050       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3051                          FuncInfo.ExceptionSelectorVirtReg,
3052                          TLI.getPointerTy(DAG.getDataLayout())),
3053       dl, ValueVTs[1]);
3054 
3055   // Merge into one.
3056   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3057                             DAG.getVTList(ValueVTs), Ops);
3058   setValue(&LP, Res);
3059 }
3060 
3061 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3062                                            MachineBasicBlock *Last) {
3063   // Update JTCases.
3064   for (JumpTableBlock &JTB : SL->JTCases)
3065     if (JTB.first.HeaderBB == First)
3066       JTB.first.HeaderBB = Last;
3067 
3068   // Update BitTestCases.
3069   for (BitTestBlock &BTB : SL->BitTestCases)
3070     if (BTB.Parent == First)
3071       BTB.Parent = Last;
3072 }
3073 
3074 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3075   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3076 
3077   // Update machine-CFG edges with unique successors.
3078   SmallSet<BasicBlock*, 32> Done;
3079   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3080     BasicBlock *BB = I.getSuccessor(i);
3081     bool Inserted = Done.insert(BB).second;
3082     if (!Inserted)
3083         continue;
3084 
3085     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3086     addSuccessorWithProb(IndirectBrMBB, Succ);
3087   }
3088   IndirectBrMBB->normalizeSuccProbs();
3089 
3090   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3091                           MVT::Other, getControlRoot(),
3092                           getValue(I.getAddress())));
3093 }
3094 
3095 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3096   if (!DAG.getTarget().Options.TrapUnreachable)
3097     return;
3098 
3099   // We may be able to ignore unreachable behind a noreturn call.
3100   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
3101     const BasicBlock &BB = *I.getParent();
3102     if (&I != &BB.front()) {
3103       BasicBlock::const_iterator PredI =
3104         std::prev(BasicBlock::const_iterator(&I));
3105       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3106         if (Call->doesNotReturn())
3107           return;
3108       }
3109     }
3110   }
3111 
3112   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3113 }
3114 
3115 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3116   SDNodeFlags Flags;
3117   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3118     Flags.copyFMF(*FPOp);
3119 
3120   SDValue Op = getValue(I.getOperand(0));
3121   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3122                                     Op, Flags);
3123   setValue(&I, UnNodeValue);
3124 }
3125 
3126 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3127   SDNodeFlags Flags;
3128   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3129     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3130     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3131   }
3132   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3133     Flags.setExact(ExactOp->isExact());
3134   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3135     Flags.copyFMF(*FPOp);
3136 
3137   SDValue Op1 = getValue(I.getOperand(0));
3138   SDValue Op2 = getValue(I.getOperand(1));
3139   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3140                                      Op1, Op2, Flags);
3141   setValue(&I, BinNodeValue);
3142 }
3143 
3144 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3145   SDValue Op1 = getValue(I.getOperand(0));
3146   SDValue Op2 = getValue(I.getOperand(1));
3147 
3148   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3149       Op1.getValueType(), DAG.getDataLayout());
3150 
3151   // Coerce the shift amount to the right type if we can. This exposes the
3152   // truncate or zext to optimization early.
3153   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3154     assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3155            "Unexpected shift type");
3156     Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3157   }
3158 
3159   bool nuw = false;
3160   bool nsw = false;
3161   bool exact = false;
3162 
3163   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3164 
3165     if (const OverflowingBinaryOperator *OFBinOp =
3166             dyn_cast<const OverflowingBinaryOperator>(&I)) {
3167       nuw = OFBinOp->hasNoUnsignedWrap();
3168       nsw = OFBinOp->hasNoSignedWrap();
3169     }
3170     if (const PossiblyExactOperator *ExactOp =
3171             dyn_cast<const PossiblyExactOperator>(&I))
3172       exact = ExactOp->isExact();
3173   }
3174   SDNodeFlags Flags;
3175   Flags.setExact(exact);
3176   Flags.setNoSignedWrap(nsw);
3177   Flags.setNoUnsignedWrap(nuw);
3178   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3179                             Flags);
3180   setValue(&I, Res);
3181 }
3182 
3183 void SelectionDAGBuilder::visitSDiv(const User &I) {
3184   SDValue Op1 = getValue(I.getOperand(0));
3185   SDValue Op2 = getValue(I.getOperand(1));
3186 
3187   SDNodeFlags Flags;
3188   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3189                  cast<PossiblyExactOperator>(&I)->isExact());
3190   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3191                            Op2, Flags));
3192 }
3193 
3194 void SelectionDAGBuilder::visitICmp(const User &I) {
3195   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3196   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3197     predicate = IC->getPredicate();
3198   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3199     predicate = ICmpInst::Predicate(IC->getPredicate());
3200   SDValue Op1 = getValue(I.getOperand(0));
3201   SDValue Op2 = getValue(I.getOperand(1));
3202   ISD::CondCode Opcode = getICmpCondCode(predicate);
3203 
3204   auto &TLI = DAG.getTargetLoweringInfo();
3205   EVT MemVT =
3206       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3207 
3208   // If a pointer's DAG type is larger than its memory type then the DAG values
3209   // are zero-extended. This breaks signed comparisons so truncate back to the
3210   // underlying type before doing the compare.
3211   if (Op1.getValueType() != MemVT) {
3212     Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3213     Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3214   }
3215 
3216   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3217                                                         I.getType());
3218   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3219 }
3220 
3221 void SelectionDAGBuilder::visitFCmp(const User &I) {
3222   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3223   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3224     predicate = FC->getPredicate();
3225   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3226     predicate = FCmpInst::Predicate(FC->getPredicate());
3227   SDValue Op1 = getValue(I.getOperand(0));
3228   SDValue Op2 = getValue(I.getOperand(1));
3229 
3230   ISD::CondCode Condition = getFCmpCondCode(predicate);
3231   auto *FPMO = cast<FPMathOperator>(&I);
3232   if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3233     Condition = getFCmpCodeWithoutNaN(Condition);
3234 
3235   SDNodeFlags Flags;
3236   Flags.copyFMF(*FPMO);
3237   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3238 
3239   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3240                                                         I.getType());
3241   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3242 }
3243 
3244 // Check if the condition of the select has one use or two users that are both
3245 // selects with the same condition.
3246 static bool hasOnlySelectUsers(const Value *Cond) {
3247   return llvm::all_of(Cond->users(), [](const Value *V) {
3248     return isa<SelectInst>(V);
3249   });
3250 }
3251 
3252 void SelectionDAGBuilder::visitSelect(const User &I) {
3253   SmallVector<EVT, 4> ValueVTs;
3254   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3255                   ValueVTs);
3256   unsigned NumValues = ValueVTs.size();
3257   if (NumValues == 0) return;
3258 
3259   SmallVector<SDValue, 4> Values(NumValues);
3260   SDValue Cond     = getValue(I.getOperand(0));
3261   SDValue LHSVal   = getValue(I.getOperand(1));
3262   SDValue RHSVal   = getValue(I.getOperand(2));
3263   SmallVector<SDValue, 1> BaseOps(1, Cond);
3264   ISD::NodeType OpCode =
3265       Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3266 
3267   bool IsUnaryAbs = false;
3268   bool Negate = false;
3269 
3270   SDNodeFlags Flags;
3271   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3272     Flags.copyFMF(*FPOp);
3273 
3274   // Min/max matching is only viable if all output VTs are the same.
3275   if (is_splat(ValueVTs)) {
3276     EVT VT = ValueVTs[0];
3277     LLVMContext &Ctx = *DAG.getContext();
3278     auto &TLI = DAG.getTargetLoweringInfo();
3279 
3280     // We care about the legality of the operation after it has been type
3281     // legalized.
3282     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3283       VT = TLI.getTypeToTransformTo(Ctx, VT);
3284 
3285     // If the vselect is legal, assume we want to leave this as a vector setcc +
3286     // vselect. Otherwise, if this is going to be scalarized, we want to see if
3287     // min/max is legal on the scalar type.
3288     bool UseScalarMinMax = VT.isVector() &&
3289       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3290 
3291     Value *LHS, *RHS;
3292     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3293     ISD::NodeType Opc = ISD::DELETED_NODE;
3294     switch (SPR.Flavor) {
3295     case SPF_UMAX:    Opc = ISD::UMAX; break;
3296     case SPF_UMIN:    Opc = ISD::UMIN; break;
3297     case SPF_SMAX:    Opc = ISD::SMAX; break;
3298     case SPF_SMIN:    Opc = ISD::SMIN; break;
3299     case SPF_FMINNUM:
3300       switch (SPR.NaNBehavior) {
3301       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3302       case SPNB_RETURNS_NAN:   Opc = ISD::FMINIMUM; break;
3303       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3304       case SPNB_RETURNS_ANY: {
3305         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3306           Opc = ISD::FMINNUM;
3307         else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3308           Opc = ISD::FMINIMUM;
3309         else if (UseScalarMinMax)
3310           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3311             ISD::FMINNUM : ISD::FMINIMUM;
3312         break;
3313       }
3314       }
3315       break;
3316     case SPF_FMAXNUM:
3317       switch (SPR.NaNBehavior) {
3318       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3319       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXIMUM; break;
3320       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3321       case SPNB_RETURNS_ANY:
3322 
3323         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3324           Opc = ISD::FMAXNUM;
3325         else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3326           Opc = ISD::FMAXIMUM;
3327         else if (UseScalarMinMax)
3328           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3329             ISD::FMAXNUM : ISD::FMAXIMUM;
3330         break;
3331       }
3332       break;
3333     case SPF_NABS:
3334       Negate = true;
3335       LLVM_FALLTHROUGH;
3336     case SPF_ABS:
3337       IsUnaryAbs = true;
3338       Opc = ISD::ABS;
3339       break;
3340     default: break;
3341     }
3342 
3343     if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3344         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3345          (UseScalarMinMax &&
3346           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3347         // If the underlying comparison instruction is used by any other
3348         // instruction, the consumed instructions won't be destroyed, so it is
3349         // not profitable to convert to a min/max.
3350         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3351       OpCode = Opc;
3352       LHSVal = getValue(LHS);
3353       RHSVal = getValue(RHS);
3354       BaseOps.clear();
3355     }
3356 
3357     if (IsUnaryAbs) {
3358       OpCode = Opc;
3359       LHSVal = getValue(LHS);
3360       BaseOps.clear();
3361     }
3362   }
3363 
3364   if (IsUnaryAbs) {
3365     for (unsigned i = 0; i != NumValues; ++i) {
3366       SDLoc dl = getCurSDLoc();
3367       EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3368       Values[i] =
3369           DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3370       if (Negate)
3371         Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
3372                                 Values[i]);
3373     }
3374   } else {
3375     for (unsigned i = 0; i != NumValues; ++i) {
3376       SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3377       Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3378       Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3379       Values[i] = DAG.getNode(
3380           OpCode, getCurSDLoc(),
3381           LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3382     }
3383   }
3384 
3385   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3386                            DAG.getVTList(ValueVTs), Values));
3387 }
3388 
3389 void SelectionDAGBuilder::visitTrunc(const User &I) {
3390   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3391   SDValue N = getValue(I.getOperand(0));
3392   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3393                                                         I.getType());
3394   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3395 }
3396 
3397 void SelectionDAGBuilder::visitZExt(const User &I) {
3398   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3399   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3400   SDValue N = getValue(I.getOperand(0));
3401   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3402                                                         I.getType());
3403   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3404 }
3405 
3406 void SelectionDAGBuilder::visitSExt(const User &I) {
3407   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3408   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3409   SDValue N = getValue(I.getOperand(0));
3410   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3411                                                         I.getType());
3412   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3413 }
3414 
3415 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3416   // FPTrunc is never a no-op cast, no need to check
3417   SDValue N = getValue(I.getOperand(0));
3418   SDLoc dl = getCurSDLoc();
3419   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3420   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3421   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3422                            DAG.getTargetConstant(
3423                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3424 }
3425 
3426 void SelectionDAGBuilder::visitFPExt(const User &I) {
3427   // FPExt is never a no-op cast, no need to check
3428   SDValue N = getValue(I.getOperand(0));
3429   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3430                                                         I.getType());
3431   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3432 }
3433 
3434 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3435   // FPToUI is never a no-op cast, no need to check
3436   SDValue N = getValue(I.getOperand(0));
3437   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3438                                                         I.getType());
3439   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3440 }
3441 
3442 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3443   // FPToSI is never a no-op cast, no need to check
3444   SDValue N = getValue(I.getOperand(0));
3445   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3446                                                         I.getType());
3447   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3448 }
3449 
3450 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3451   // UIToFP is never a no-op cast, no need to check
3452   SDValue N = getValue(I.getOperand(0));
3453   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3454                                                         I.getType());
3455   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3456 }
3457 
3458 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3459   // SIToFP is never a no-op cast, no need to check
3460   SDValue N = getValue(I.getOperand(0));
3461   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3462                                                         I.getType());
3463   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3464 }
3465 
3466 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3467   // What to do depends on the size of the integer and the size of the pointer.
3468   // We can either truncate, zero extend, or no-op, accordingly.
3469   SDValue N = getValue(I.getOperand(0));
3470   auto &TLI = DAG.getTargetLoweringInfo();
3471   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3472                                                         I.getType());
3473   EVT PtrMemVT =
3474       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3475   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3476   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3477   setValue(&I, N);
3478 }
3479 
3480 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3481   // What to do depends on the size of the integer and the size of the pointer.
3482   // We can either truncate, zero extend, or no-op, accordingly.
3483   SDValue N = getValue(I.getOperand(0));
3484   auto &TLI = DAG.getTargetLoweringInfo();
3485   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3486   EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3487   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3488   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3489   setValue(&I, N);
3490 }
3491 
3492 void SelectionDAGBuilder::visitBitCast(const User &I) {
3493   SDValue N = getValue(I.getOperand(0));
3494   SDLoc dl = getCurSDLoc();
3495   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3496                                                         I.getType());
3497 
3498   // BitCast assures us that source and destination are the same size so this is
3499   // either a BITCAST or a no-op.
3500   if (DestVT != N.getValueType())
3501     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3502                              DestVT, N)); // convert types.
3503   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3504   // might fold any kind of constant expression to an integer constant and that
3505   // is not what we are looking for. Only recognize a bitcast of a genuine
3506   // constant integer as an opaque constant.
3507   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3508     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3509                                  /*isOpaque*/true));
3510   else
3511     setValue(&I, N);            // noop cast.
3512 }
3513 
3514 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3515   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3516   const Value *SV = I.getOperand(0);
3517   SDValue N = getValue(SV);
3518   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3519 
3520   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3521   unsigned DestAS = I.getType()->getPointerAddressSpace();
3522 
3523   if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3524     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3525 
3526   setValue(&I, N);
3527 }
3528 
3529 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3530   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3531   SDValue InVec = getValue(I.getOperand(0));
3532   SDValue InVal = getValue(I.getOperand(1));
3533   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3534                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3535   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3536                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3537                            InVec, InVal, InIdx));
3538 }
3539 
3540 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3541   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3542   SDValue InVec = getValue(I.getOperand(0));
3543   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3544                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3545   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3546                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3547                            InVec, InIdx));
3548 }
3549 
3550 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3551   SDValue Src1 = getValue(I.getOperand(0));
3552   SDValue Src2 = getValue(I.getOperand(1));
3553   ArrayRef<int> Mask;
3554   if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3555     Mask = SVI->getShuffleMask();
3556   else
3557     Mask = cast<ConstantExpr>(I).getShuffleMask();
3558   SDLoc DL = getCurSDLoc();
3559   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3560   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3561   EVT SrcVT = Src1.getValueType();
3562 
3563   if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3564       VT.isScalableVector()) {
3565     // Canonical splat form of first element of first input vector.
3566     SDValue FirstElt =
3567         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
3568                     DAG.getVectorIdxConstant(0, DL));
3569     setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3570     return;
3571   }
3572 
3573   // For now, we only handle splats for scalable vectors.
3574   // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3575   // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3576   assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3577 
3578   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3579   unsigned MaskNumElts = Mask.size();
3580 
3581   if (SrcNumElts == MaskNumElts) {
3582     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3583     return;
3584   }
3585 
3586   // Normalize the shuffle vector since mask and vector length don't match.
3587   if (SrcNumElts < MaskNumElts) {
3588     // Mask is longer than the source vectors. We can use concatenate vector to
3589     // make the mask and vectors lengths match.
3590 
3591     if (MaskNumElts % SrcNumElts == 0) {
3592       // Mask length is a multiple of the source vector length.
3593       // Check if the shuffle is some kind of concatenation of the input
3594       // vectors.
3595       unsigned NumConcat = MaskNumElts / SrcNumElts;
3596       bool IsConcat = true;
3597       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3598       for (unsigned i = 0; i != MaskNumElts; ++i) {
3599         int Idx = Mask[i];
3600         if (Idx < 0)
3601           continue;
3602         // Ensure the indices in each SrcVT sized piece are sequential and that
3603         // the same source is used for the whole piece.
3604         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3605             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3606              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3607           IsConcat = false;
3608           break;
3609         }
3610         // Remember which source this index came from.
3611         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3612       }
3613 
3614       // The shuffle is concatenating multiple vectors together. Just emit
3615       // a CONCAT_VECTORS operation.
3616       if (IsConcat) {
3617         SmallVector<SDValue, 8> ConcatOps;
3618         for (auto Src : ConcatSrcs) {
3619           if (Src < 0)
3620             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3621           else if (Src == 0)
3622             ConcatOps.push_back(Src1);
3623           else
3624             ConcatOps.push_back(Src2);
3625         }
3626         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3627         return;
3628       }
3629     }
3630 
3631     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3632     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3633     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3634                                     PaddedMaskNumElts);
3635 
3636     // Pad both vectors with undefs to make them the same length as the mask.
3637     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3638 
3639     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3640     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3641     MOps1[0] = Src1;
3642     MOps2[0] = Src2;
3643 
3644     Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3645     Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3646 
3647     // Readjust mask for new input vector length.
3648     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3649     for (unsigned i = 0; i != MaskNumElts; ++i) {
3650       int Idx = Mask[i];
3651       if (Idx >= (int)SrcNumElts)
3652         Idx -= SrcNumElts - PaddedMaskNumElts;
3653       MappedOps[i] = Idx;
3654     }
3655 
3656     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3657 
3658     // If the concatenated vector was padded, extract a subvector with the
3659     // correct number of elements.
3660     if (MaskNumElts != PaddedMaskNumElts)
3661       Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3662                            DAG.getVectorIdxConstant(0, DL));
3663 
3664     setValue(&I, Result);
3665     return;
3666   }
3667 
3668   if (SrcNumElts > MaskNumElts) {
3669     // Analyze the access pattern of the vector to see if we can extract
3670     // two subvectors and do the shuffle.
3671     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3672     bool CanExtract = true;
3673     for (int Idx : Mask) {
3674       unsigned Input = 0;
3675       if (Idx < 0)
3676         continue;
3677 
3678       if (Idx >= (int)SrcNumElts) {
3679         Input = 1;
3680         Idx -= SrcNumElts;
3681       }
3682 
3683       // If all the indices come from the same MaskNumElts sized portion of
3684       // the sources we can use extract. Also make sure the extract wouldn't
3685       // extract past the end of the source.
3686       int NewStartIdx = alignDown(Idx, MaskNumElts);
3687       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3688           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3689         CanExtract = false;
3690       // Make sure we always update StartIdx as we use it to track if all
3691       // elements are undef.
3692       StartIdx[Input] = NewStartIdx;
3693     }
3694 
3695     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3696       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3697       return;
3698     }
3699     if (CanExtract) {
3700       // Extract appropriate subvector and generate a vector shuffle
3701       for (unsigned Input = 0; Input < 2; ++Input) {
3702         SDValue &Src = Input == 0 ? Src1 : Src2;
3703         if (StartIdx[Input] < 0)
3704           Src = DAG.getUNDEF(VT);
3705         else {
3706           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3707                             DAG.getVectorIdxConstant(StartIdx[Input], DL));
3708         }
3709       }
3710 
3711       // Calculate new mask.
3712       SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3713       for (int &Idx : MappedOps) {
3714         if (Idx >= (int)SrcNumElts)
3715           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3716         else if (Idx >= 0)
3717           Idx -= StartIdx[0];
3718       }
3719 
3720       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3721       return;
3722     }
3723   }
3724 
3725   // We can't use either concat vectors or extract subvectors so fall back to
3726   // replacing the shuffle with extract and build vector.
3727   // to insert and build vector.
3728   EVT EltVT = VT.getVectorElementType();
3729   SmallVector<SDValue,8> Ops;
3730   for (int Idx : Mask) {
3731     SDValue Res;
3732 
3733     if (Idx < 0) {
3734       Res = DAG.getUNDEF(EltVT);
3735     } else {
3736       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3737       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3738 
3739       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3740                         DAG.getVectorIdxConstant(Idx, DL));
3741     }
3742 
3743     Ops.push_back(Res);
3744   }
3745 
3746   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3747 }
3748 
3749 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3750   ArrayRef<unsigned> Indices;
3751   if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3752     Indices = IV->getIndices();
3753   else
3754     Indices = cast<ConstantExpr>(&I)->getIndices();
3755 
3756   const Value *Op0 = I.getOperand(0);
3757   const Value *Op1 = I.getOperand(1);
3758   Type *AggTy = I.getType();
3759   Type *ValTy = Op1->getType();
3760   bool IntoUndef = isa<UndefValue>(Op0);
3761   bool FromUndef = isa<UndefValue>(Op1);
3762 
3763   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3764 
3765   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3766   SmallVector<EVT, 4> AggValueVTs;
3767   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3768   SmallVector<EVT, 4> ValValueVTs;
3769   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3770 
3771   unsigned NumAggValues = AggValueVTs.size();
3772   unsigned NumValValues = ValValueVTs.size();
3773   SmallVector<SDValue, 4> Values(NumAggValues);
3774 
3775   // Ignore an insertvalue that produces an empty object
3776   if (!NumAggValues) {
3777     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3778     return;
3779   }
3780 
3781   SDValue Agg = getValue(Op0);
3782   unsigned i = 0;
3783   // Copy the beginning value(s) from the original aggregate.
3784   for (; i != LinearIndex; ++i)
3785     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3786                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3787   // Copy values from the inserted value(s).
3788   if (NumValValues) {
3789     SDValue Val = getValue(Op1);
3790     for (; i != LinearIndex + NumValValues; ++i)
3791       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3792                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3793   }
3794   // Copy remaining value(s) from the original aggregate.
3795   for (; i != NumAggValues; ++i)
3796     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3797                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3798 
3799   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3800                            DAG.getVTList(AggValueVTs), Values));
3801 }
3802 
3803 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3804   ArrayRef<unsigned> Indices;
3805   if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3806     Indices = EV->getIndices();
3807   else
3808     Indices = cast<ConstantExpr>(&I)->getIndices();
3809 
3810   const Value *Op0 = I.getOperand(0);
3811   Type *AggTy = Op0->getType();
3812   Type *ValTy = I.getType();
3813   bool OutOfUndef = isa<UndefValue>(Op0);
3814 
3815   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3816 
3817   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3818   SmallVector<EVT, 4> ValValueVTs;
3819   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3820 
3821   unsigned NumValValues = ValValueVTs.size();
3822 
3823   // Ignore a extractvalue that produces an empty object
3824   if (!NumValValues) {
3825     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3826     return;
3827   }
3828 
3829   SmallVector<SDValue, 4> Values(NumValValues);
3830 
3831   SDValue Agg = getValue(Op0);
3832   // Copy out the selected value(s).
3833   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3834     Values[i - LinearIndex] =
3835       OutOfUndef ?
3836         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3837         SDValue(Agg.getNode(), Agg.getResNo() + i);
3838 
3839   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3840                            DAG.getVTList(ValValueVTs), Values));
3841 }
3842 
3843 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3844   Value *Op0 = I.getOperand(0);
3845   // Note that the pointer operand may be a vector of pointers. Take the scalar
3846   // element which holds a pointer.
3847   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3848   SDValue N = getValue(Op0);
3849   SDLoc dl = getCurSDLoc();
3850   auto &TLI = DAG.getTargetLoweringInfo();
3851 
3852   // Normalize Vector GEP - all scalar operands should be converted to the
3853   // splat vector.
3854   bool IsVectorGEP = I.getType()->isVectorTy();
3855   ElementCount VectorElementCount =
3856       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3857                   : ElementCount::getFixed(0);
3858 
3859   if (IsVectorGEP && !N.getValueType().isVector()) {
3860     LLVMContext &Context = *DAG.getContext();
3861     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3862     if (VectorElementCount.isScalable())
3863       N = DAG.getSplatVector(VT, dl, N);
3864     else
3865       N = DAG.getSplatBuildVector(VT, dl, N);
3866   }
3867 
3868   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3869        GTI != E; ++GTI) {
3870     const Value *Idx = GTI.getOperand();
3871     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3872       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3873       if (Field) {
3874         // N = N + Offset
3875         uint64_t Offset =
3876             DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
3877 
3878         // In an inbounds GEP with an offset that is nonnegative even when
3879         // interpreted as signed, assume there is no unsigned overflow.
3880         SDNodeFlags Flags;
3881         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3882           Flags.setNoUnsignedWrap(true);
3883 
3884         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3885                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3886       }
3887     } else {
3888       // IdxSize is the width of the arithmetic according to IR semantics.
3889       // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3890       // (and fix up the result later).
3891       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3892       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3893       TypeSize ElementSize =
3894           DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3895       // We intentionally mask away the high bits here; ElementSize may not
3896       // fit in IdxTy.
3897       APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3898       bool ElementScalable = ElementSize.isScalable();
3899 
3900       // If this is a scalar constant or a splat vector of constants,
3901       // handle it quickly.
3902       const auto *C = dyn_cast<Constant>(Idx);
3903       if (C && isa<VectorType>(C->getType()))
3904         C = C->getSplatValue();
3905 
3906       const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3907       if (CI && CI->isZero())
3908         continue;
3909       if (CI && !ElementScalable) {
3910         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3911         LLVMContext &Context = *DAG.getContext();
3912         SDValue OffsVal;
3913         if (IsVectorGEP)
3914           OffsVal = DAG.getConstant(
3915               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3916         else
3917           OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3918 
3919         // In an inbounds GEP with an offset that is nonnegative even when
3920         // interpreted as signed, assume there is no unsigned overflow.
3921         SDNodeFlags Flags;
3922         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3923           Flags.setNoUnsignedWrap(true);
3924 
3925         OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3926 
3927         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3928         continue;
3929       }
3930 
3931       // N = N + Idx * ElementMul;
3932       SDValue IdxN = getValue(Idx);
3933 
3934       if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3935         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3936                                   VectorElementCount);
3937         if (VectorElementCount.isScalable())
3938           IdxN = DAG.getSplatVector(VT, dl, IdxN);
3939         else
3940           IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3941       }
3942 
3943       // If the index is smaller or larger than intptr_t, truncate or extend
3944       // it.
3945       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3946 
3947       if (ElementScalable) {
3948         EVT VScaleTy = N.getValueType().getScalarType();
3949         SDValue VScale = DAG.getNode(
3950             ISD::VSCALE, dl, VScaleTy,
3951             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3952         if (IsVectorGEP)
3953           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3954         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3955       } else {
3956         // If this is a multiply by a power of two, turn it into a shl
3957         // immediately.  This is a very common case.
3958         if (ElementMul != 1) {
3959           if (ElementMul.isPowerOf2()) {
3960             unsigned Amt = ElementMul.logBase2();
3961             IdxN = DAG.getNode(ISD::SHL, dl,
3962                                N.getValueType(), IdxN,
3963                                DAG.getConstant(Amt, dl, IdxN.getValueType()));
3964           } else {
3965             SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3966                                             IdxN.getValueType());
3967             IdxN = DAG.getNode(ISD::MUL, dl,
3968                                N.getValueType(), IdxN, Scale);
3969           }
3970         }
3971       }
3972 
3973       N = DAG.getNode(ISD::ADD, dl,
3974                       N.getValueType(), N, IdxN);
3975     }
3976   }
3977 
3978   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3979   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3980   if (IsVectorGEP) {
3981     PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
3982     PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
3983   }
3984 
3985   if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3986     N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3987 
3988   setValue(&I, N);
3989 }
3990 
3991 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3992   // If this is a fixed sized alloca in the entry block of the function,
3993   // allocate it statically on the stack.
3994   if (FuncInfo.StaticAllocaMap.count(&I))
3995     return;   // getValue will auto-populate this.
3996 
3997   SDLoc dl = getCurSDLoc();
3998   Type *Ty = I.getAllocatedType();
3999   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4000   auto &DL = DAG.getDataLayout();
4001   TypeSize TySize = DL.getTypeAllocSize(Ty);
4002   MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4003 
4004   SDValue AllocSize = getValue(I.getArraySize());
4005 
4006   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
4007   if (AllocSize.getValueType() != IntPtr)
4008     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4009 
4010   if (TySize.isScalable())
4011     AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4012                             DAG.getVScale(dl, IntPtr,
4013                                           APInt(IntPtr.getScalarSizeInBits(),
4014                                                 TySize.getKnownMinValue())));
4015   else
4016     AllocSize =
4017         DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4018                     DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4019 
4020   // Handle alignment.  If the requested alignment is less than or equal to
4021   // the stack alignment, ignore it.  If the size is greater than or equal to
4022   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4023   Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4024   if (*Alignment <= StackAlign)
4025     Alignment = None;
4026 
4027   const uint64_t StackAlignMask = StackAlign.value() - 1U;
4028   // Round the size of the allocation up to the stack alignment size
4029   // by add SA-1 to the size. This doesn't overflow because we're computing
4030   // an address inside an alloca.
4031   SDNodeFlags Flags;
4032   Flags.setNoUnsignedWrap(true);
4033   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4034                           DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4035 
4036   // Mask out the low bits for alignment purposes.
4037   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4038                           DAG.getConstant(~StackAlignMask, dl, IntPtr));
4039 
4040   SDValue Ops[] = {
4041       getRoot(), AllocSize,
4042       DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4043   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4044   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4045   setValue(&I, DSA);
4046   DAG.setRoot(DSA.getValue(1));
4047 
4048   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4049 }
4050 
4051 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4052   if (I.isAtomic())
4053     return visitAtomicLoad(I);
4054 
4055   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4056   const Value *SV = I.getOperand(0);
4057   if (TLI.supportSwiftError()) {
4058     // Swifterror values can come from either a function parameter with
4059     // swifterror attribute or an alloca with swifterror attribute.
4060     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4061       if (Arg->hasSwiftErrorAttr())
4062         return visitLoadFromSwiftError(I);
4063     }
4064 
4065     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4066       if (Alloca->isSwiftError())
4067         return visitLoadFromSwiftError(I);
4068     }
4069   }
4070 
4071   SDValue Ptr = getValue(SV);
4072 
4073   Type *Ty = I.getType();
4074   Align Alignment = I.getAlign();
4075 
4076   AAMDNodes AAInfo = I.getAAMetadata();
4077   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4078 
4079   SmallVector<EVT, 4> ValueVTs, MemVTs;
4080   SmallVector<uint64_t, 4> Offsets;
4081   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4082   unsigned NumValues = ValueVTs.size();
4083   if (NumValues == 0)
4084     return;
4085 
4086   bool isVolatile = I.isVolatile();
4087 
4088   SDValue Root;
4089   bool ConstantMemory = false;
4090   if (isVolatile)
4091     // Serialize volatile loads with other side effects.
4092     Root = getRoot();
4093   else if (NumValues > MaxParallelChains)
4094     Root = getMemoryRoot();
4095   else if (AA &&
4096            AA->pointsToConstantMemory(MemoryLocation(
4097                SV,
4098                LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4099                AAInfo))) {
4100     // Do not serialize (non-volatile) loads of constant memory with anything.
4101     Root = DAG.getEntryNode();
4102     ConstantMemory = true;
4103   } else {
4104     // Do not serialize non-volatile loads against each other.
4105     Root = DAG.getRoot();
4106   }
4107 
4108   SDLoc dl = getCurSDLoc();
4109 
4110   if (isVolatile)
4111     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4112 
4113   // An aggregate load cannot wrap around the address space, so offsets to its
4114   // parts don't wrap either.
4115   SDNodeFlags Flags;
4116   Flags.setNoUnsignedWrap(true);
4117 
4118   SmallVector<SDValue, 4> Values(NumValues);
4119   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4120   EVT PtrVT = Ptr.getValueType();
4121 
4122   MachineMemOperand::Flags MMOFlags
4123     = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4124 
4125   unsigned ChainI = 0;
4126   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4127     // Serializing loads here may result in excessive register pressure, and
4128     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4129     // could recover a bit by hoisting nodes upward in the chain by recognizing
4130     // they are side-effect free or do not alias. The optimizer should really
4131     // avoid this case by converting large object/array copies to llvm.memcpy
4132     // (MaxParallelChains should always remain as failsafe).
4133     if (ChainI == MaxParallelChains) {
4134       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4135       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4136                                   makeArrayRef(Chains.data(), ChainI));
4137       Root = Chain;
4138       ChainI = 0;
4139     }
4140     SDValue A = DAG.getNode(ISD::ADD, dl,
4141                             PtrVT, Ptr,
4142                             DAG.getConstant(Offsets[i], dl, PtrVT),
4143                             Flags);
4144 
4145     SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4146                             MachinePointerInfo(SV, Offsets[i]), Alignment,
4147                             MMOFlags, AAInfo, Ranges);
4148     Chains[ChainI] = L.getValue(1);
4149 
4150     if (MemVTs[i] != ValueVTs[i])
4151       L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4152 
4153     Values[i] = L;
4154   }
4155 
4156   if (!ConstantMemory) {
4157     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4158                                 makeArrayRef(Chains.data(), ChainI));
4159     if (isVolatile)
4160       DAG.setRoot(Chain);
4161     else
4162       PendingLoads.push_back(Chain);
4163   }
4164 
4165   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4166                            DAG.getVTList(ValueVTs), Values));
4167 }
4168 
4169 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4170   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4171          "call visitStoreToSwiftError when backend supports swifterror");
4172 
4173   SmallVector<EVT, 4> ValueVTs;
4174   SmallVector<uint64_t, 4> Offsets;
4175   const Value *SrcV = I.getOperand(0);
4176   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4177                   SrcV->getType(), ValueVTs, &Offsets);
4178   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4179          "expect a single EVT for swifterror");
4180 
4181   SDValue Src = getValue(SrcV);
4182   // Create a virtual register, then update the virtual register.
4183   Register VReg =
4184       SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4185   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4186   // Chain can be getRoot or getControlRoot.
4187   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4188                                       SDValue(Src.getNode(), Src.getResNo()));
4189   DAG.setRoot(CopyNode);
4190 }
4191 
4192 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4193   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4194          "call visitLoadFromSwiftError when backend supports swifterror");
4195 
4196   assert(!I.isVolatile() &&
4197          !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4198          !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4199          "Support volatile, non temporal, invariant for load_from_swift_error");
4200 
4201   const Value *SV = I.getOperand(0);
4202   Type *Ty = I.getType();
4203   assert(
4204       (!AA ||
4205        !AA->pointsToConstantMemory(MemoryLocation(
4206            SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4207            I.getAAMetadata()))) &&
4208       "load_from_swift_error should not be constant memory");
4209 
4210   SmallVector<EVT, 4> ValueVTs;
4211   SmallVector<uint64_t, 4> Offsets;
4212   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4213                   ValueVTs, &Offsets);
4214   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4215          "expect a single EVT for swifterror");
4216 
4217   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4218   SDValue L = DAG.getCopyFromReg(
4219       getRoot(), getCurSDLoc(),
4220       SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4221 
4222   setValue(&I, L);
4223 }
4224 
4225 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4226   if (I.isAtomic())
4227     return visitAtomicStore(I);
4228 
4229   const Value *SrcV = I.getOperand(0);
4230   const Value *PtrV = I.getOperand(1);
4231 
4232   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4233   if (TLI.supportSwiftError()) {
4234     // Swifterror values can come from either a function parameter with
4235     // swifterror attribute or an alloca with swifterror attribute.
4236     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4237       if (Arg->hasSwiftErrorAttr())
4238         return visitStoreToSwiftError(I);
4239     }
4240 
4241     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4242       if (Alloca->isSwiftError())
4243         return visitStoreToSwiftError(I);
4244     }
4245   }
4246 
4247   SmallVector<EVT, 4> ValueVTs, MemVTs;
4248   SmallVector<uint64_t, 4> Offsets;
4249   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4250                   SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4251   unsigned NumValues = ValueVTs.size();
4252   if (NumValues == 0)
4253     return;
4254 
4255   // Get the lowered operands. Note that we do this after
4256   // checking if NumResults is zero, because with zero results
4257   // the operands won't have values in the map.
4258   SDValue Src = getValue(SrcV);
4259   SDValue Ptr = getValue(PtrV);
4260 
4261   SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4262   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4263   SDLoc dl = getCurSDLoc();
4264   Align Alignment = I.getAlign();
4265   AAMDNodes AAInfo = I.getAAMetadata();
4266 
4267   auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4268 
4269   // An aggregate load cannot wrap around the address space, so offsets to its
4270   // parts don't wrap either.
4271   SDNodeFlags Flags;
4272   Flags.setNoUnsignedWrap(true);
4273 
4274   unsigned ChainI = 0;
4275   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4276     // See visitLoad comments.
4277     if (ChainI == MaxParallelChains) {
4278       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4279                                   makeArrayRef(Chains.data(), ChainI));
4280       Root = Chain;
4281       ChainI = 0;
4282     }
4283     SDValue Add =
4284         DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4285     SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4286     if (MemVTs[i] != ValueVTs[i])
4287       Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4288     SDValue St =
4289         DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4290                      Alignment, MMOFlags, AAInfo);
4291     Chains[ChainI] = St;
4292   }
4293 
4294   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4295                                   makeArrayRef(Chains.data(), ChainI));
4296   DAG.setRoot(StoreNode);
4297 }
4298 
4299 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4300                                            bool IsCompressing) {
4301   SDLoc sdl = getCurSDLoc();
4302 
4303   auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4304                                MaybeAlign &Alignment) {
4305     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4306     Src0 = I.getArgOperand(0);
4307     Ptr = I.getArgOperand(1);
4308     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4309     Mask = I.getArgOperand(3);
4310   };
4311   auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4312                                     MaybeAlign &Alignment) {
4313     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4314     Src0 = I.getArgOperand(0);
4315     Ptr = I.getArgOperand(1);
4316     Mask = I.getArgOperand(2);
4317     Alignment = None;
4318   };
4319 
4320   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4321   MaybeAlign Alignment;
4322   if (IsCompressing)
4323     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4324   else
4325     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4326 
4327   SDValue Ptr = getValue(PtrOperand);
4328   SDValue Src0 = getValue(Src0Operand);
4329   SDValue Mask = getValue(MaskOperand);
4330   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4331 
4332   EVT VT = Src0.getValueType();
4333   if (!Alignment)
4334     Alignment = DAG.getEVTAlign(VT);
4335 
4336   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4337       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
4338       MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
4339   SDValue StoreNode =
4340       DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4341                          ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4342   DAG.setRoot(StoreNode);
4343   setValue(&I, StoreNode);
4344 }
4345 
4346 // Get a uniform base for the Gather/Scatter intrinsic.
4347 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4348 // We try to represent it as a base pointer + vector of indices.
4349 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4350 // The first operand of the GEP may be a single pointer or a vector of pointers
4351 // Example:
4352 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4353 //  or
4354 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4355 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4356 //
4357 // When the first GEP operand is a single pointer - it is the uniform base we
4358 // are looking for. If first operand of the GEP is a splat vector - we
4359 // extract the splat value and use it as a uniform base.
4360 // In all other cases the function returns 'false'.
4361 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4362                            ISD::MemIndexType &IndexType, SDValue &Scale,
4363                            SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {
4364   SelectionDAG& DAG = SDB->DAG;
4365   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4366   const DataLayout &DL = DAG.getDataLayout();
4367 
4368   assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4369 
4370   // Handle splat constant pointer.
4371   if (auto *C = dyn_cast<Constant>(Ptr)) {
4372     C = C->getSplatValue();
4373     if (!C)
4374       return false;
4375 
4376     Base = SDB->getValue(C);
4377 
4378     ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4379     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4380     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4381     IndexType = ISD::SIGNED_SCALED;
4382     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4383     return true;
4384   }
4385 
4386   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4387   if (!GEP || GEP->getParent() != CurBB)
4388     return false;
4389 
4390   if (GEP->getNumOperands() != 2)
4391     return false;
4392 
4393   const Value *BasePtr = GEP->getPointerOperand();
4394   const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4395 
4396   // Make sure the base is scalar and the index is a vector.
4397   if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4398     return false;
4399 
4400   Base = SDB->getValue(BasePtr);
4401   Index = SDB->getValue(IndexVal);
4402   IndexType = ISD::SIGNED_SCALED;
4403 
4404   // MGATHER/MSCATTER only support scaling by a power-of-two.
4405   uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4406   if (!isPowerOf2_64(ScaleVal))
4407     return false;
4408 
4409   Scale =
4410       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4411   return true;
4412 }
4413 
4414 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4415   SDLoc sdl = getCurSDLoc();
4416 
4417   // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4418   const Value *Ptr = I.getArgOperand(1);
4419   SDValue Src0 = getValue(I.getArgOperand(0));
4420   SDValue Mask = getValue(I.getArgOperand(3));
4421   EVT VT = Src0.getValueType();
4422   Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4423                         ->getMaybeAlignValue()
4424                         .getValueOr(DAG.getEVTAlign(VT.getScalarType()));
4425   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4426 
4427   SDValue Base;
4428   SDValue Index;
4429   ISD::MemIndexType IndexType;
4430   SDValue Scale;
4431   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4432                                     I.getParent());
4433 
4434   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4435   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4436       MachinePointerInfo(AS), MachineMemOperand::MOStore,
4437       // TODO: Make MachineMemOperands aware of scalable
4438       // vectors.
4439       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4440   if (!UniformBase) {
4441     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4442     Index = getValue(Ptr);
4443     IndexType = ISD::SIGNED_UNSCALED;
4444     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4445   }
4446 
4447   EVT IdxVT = Index.getValueType();
4448   EVT EltTy = IdxVT.getVectorElementType();
4449   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4450     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4451     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4452   }
4453 
4454   SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4455   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4456                                          Ops, MMO, IndexType, false);
4457   DAG.setRoot(Scatter);
4458   setValue(&I, Scatter);
4459 }
4460 
4461 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4462   SDLoc sdl = getCurSDLoc();
4463 
4464   auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4465                               MaybeAlign &Alignment) {
4466     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4467     Ptr = I.getArgOperand(0);
4468     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4469     Mask = I.getArgOperand(2);
4470     Src0 = I.getArgOperand(3);
4471   };
4472   auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4473                                  MaybeAlign &Alignment) {
4474     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4475     Ptr = I.getArgOperand(0);
4476     Alignment = None;
4477     Mask = I.getArgOperand(1);
4478     Src0 = I.getArgOperand(2);
4479   };
4480 
4481   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4482   MaybeAlign Alignment;
4483   if (IsExpanding)
4484     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4485   else
4486     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4487 
4488   SDValue Ptr = getValue(PtrOperand);
4489   SDValue Src0 = getValue(Src0Operand);
4490   SDValue Mask = getValue(MaskOperand);
4491   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4492 
4493   EVT VT = Src0.getValueType();
4494   if (!Alignment)
4495     Alignment = DAG.getEVTAlign(VT);
4496 
4497   AAMDNodes AAInfo = I.getAAMetadata();
4498   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4499 
4500   // Do not serialize masked loads of constant memory with anything.
4501   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4502   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4503 
4504   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4505 
4506   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4507       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
4508       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
4509 
4510   SDValue Load =
4511       DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4512                         ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4513   if (AddToChain)
4514     PendingLoads.push_back(Load.getValue(1));
4515   setValue(&I, Load);
4516 }
4517 
4518 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4519   SDLoc sdl = getCurSDLoc();
4520 
4521   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4522   const Value *Ptr = I.getArgOperand(0);
4523   SDValue Src0 = getValue(I.getArgOperand(3));
4524   SDValue Mask = getValue(I.getArgOperand(2));
4525 
4526   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4527   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4528   Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4529                         ->getMaybeAlignValue()
4530                         .getValueOr(DAG.getEVTAlign(VT.getScalarType()));
4531 
4532   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4533 
4534   SDValue Root = DAG.getRoot();
4535   SDValue Base;
4536   SDValue Index;
4537   ISD::MemIndexType IndexType;
4538   SDValue Scale;
4539   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4540                                     I.getParent());
4541   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4542   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4543       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
4544       // TODO: Make MachineMemOperands aware of scalable
4545       // vectors.
4546       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4547 
4548   if (!UniformBase) {
4549     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4550     Index = getValue(Ptr);
4551     IndexType = ISD::SIGNED_UNSCALED;
4552     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4553   }
4554 
4555   EVT IdxVT = Index.getValueType();
4556   EVT EltTy = IdxVT.getVectorElementType();
4557   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4558     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4559     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4560   }
4561 
4562   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4563   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4564                                        Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4565 
4566   PendingLoads.push_back(Gather.getValue(1));
4567   setValue(&I, Gather);
4568 }
4569 
4570 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4571   SDLoc dl = getCurSDLoc();
4572   AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4573   AtomicOrdering FailureOrdering = I.getFailureOrdering();
4574   SyncScope::ID SSID = I.getSyncScopeID();
4575 
4576   SDValue InChain = getRoot();
4577 
4578   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4579   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4580 
4581   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4582   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4583 
4584   MachineFunction &MF = DAG.getMachineFunction();
4585   MachineMemOperand *MMO = MF.getMachineMemOperand(
4586       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4587       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
4588       FailureOrdering);
4589 
4590   SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4591                                    dl, MemVT, VTs, InChain,
4592                                    getValue(I.getPointerOperand()),
4593                                    getValue(I.getCompareOperand()),
4594                                    getValue(I.getNewValOperand()), MMO);
4595 
4596   SDValue OutChain = L.getValue(2);
4597 
4598   setValue(&I, L);
4599   DAG.setRoot(OutChain);
4600 }
4601 
4602 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4603   SDLoc dl = getCurSDLoc();
4604   ISD::NodeType NT;
4605   switch (I.getOperation()) {
4606   default: llvm_unreachable("Unknown atomicrmw operation");
4607   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4608   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4609   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4610   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4611   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4612   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4613   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4614   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4615   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4616   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4617   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4618   case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4619   case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4620   }
4621   AtomicOrdering Ordering = I.getOrdering();
4622   SyncScope::ID SSID = I.getSyncScopeID();
4623 
4624   SDValue InChain = getRoot();
4625 
4626   auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4627   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4628   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4629 
4630   MachineFunction &MF = DAG.getMachineFunction();
4631   MachineMemOperand *MMO = MF.getMachineMemOperand(
4632       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4633       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
4634 
4635   SDValue L =
4636     DAG.getAtomic(NT, dl, MemVT, InChain,
4637                   getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4638                   MMO);
4639 
4640   SDValue OutChain = L.getValue(1);
4641 
4642   setValue(&I, L);
4643   DAG.setRoot(OutChain);
4644 }
4645 
4646 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4647   SDLoc dl = getCurSDLoc();
4648   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4649   SDValue Ops[3];
4650   Ops[0] = getRoot();
4651   Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
4652                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4653   Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
4654                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4655   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4656 }
4657 
4658 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4659   SDLoc dl = getCurSDLoc();
4660   AtomicOrdering Order = I.getOrdering();
4661   SyncScope::ID SSID = I.getSyncScopeID();
4662 
4663   SDValue InChain = getRoot();
4664 
4665   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4666   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4667   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4668 
4669   if (!TLI.supportsUnalignedAtomics() &&
4670       I.getAlignment() < MemVT.getSizeInBits() / 8)
4671     report_fatal_error("Cannot generate unaligned atomic load");
4672 
4673   auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4674 
4675   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4676       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4677       I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
4678 
4679   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4680 
4681   SDValue Ptr = getValue(I.getPointerOperand());
4682 
4683   if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
4684     // TODO: Once this is better exercised by tests, it should be merged with
4685     // the normal path for loads to prevent future divergence.
4686     SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4687     if (MemVT != VT)
4688       L = DAG.getPtrExtOrTrunc(L, dl, VT);
4689 
4690     setValue(&I, L);
4691     SDValue OutChain = L.getValue(1);
4692     if (!I.isUnordered())
4693       DAG.setRoot(OutChain);
4694     else
4695       PendingLoads.push_back(OutChain);
4696     return;
4697   }
4698 
4699   SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4700                             Ptr, MMO);
4701 
4702   SDValue OutChain = L.getValue(1);
4703   if (MemVT != VT)
4704     L = DAG.getPtrExtOrTrunc(L, dl, VT);
4705 
4706   setValue(&I, L);
4707   DAG.setRoot(OutChain);
4708 }
4709 
4710 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4711   SDLoc dl = getCurSDLoc();
4712 
4713   AtomicOrdering Ordering = I.getOrdering();
4714   SyncScope::ID SSID = I.getSyncScopeID();
4715 
4716   SDValue InChain = getRoot();
4717 
4718   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4719   EVT MemVT =
4720       TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4721 
4722   if (I.getAlignment() < MemVT.getSizeInBits() / 8)
4723     report_fatal_error("Cannot generate unaligned atomic store");
4724 
4725   auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4726 
4727   MachineFunction &MF = DAG.getMachineFunction();
4728   MachineMemOperand *MMO = MF.getMachineMemOperand(
4729       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4730       I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
4731 
4732   SDValue Val = getValue(I.getValueOperand());
4733   if (Val.getValueType() != MemVT)
4734     Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4735   SDValue Ptr = getValue(I.getPointerOperand());
4736 
4737   if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
4738     // TODO: Once this is better exercised by tests, it should be merged with
4739     // the normal path for stores to prevent future divergence.
4740     SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4741     DAG.setRoot(S);
4742     return;
4743   }
4744   SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4745                                    Ptr, Val, MMO);
4746 
4747 
4748   DAG.setRoot(OutChain);
4749 }
4750 
4751 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4752 /// node.
4753 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4754                                                unsigned Intrinsic) {
4755   // Ignore the callsite's attributes. A specific call site may be marked with
4756   // readnone, but the lowering code will expect the chain based on the
4757   // definition.
4758   const Function *F = I.getCalledFunction();
4759   bool HasChain = !F->doesNotAccessMemory();
4760   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4761 
4762   // Build the operand list.
4763   SmallVector<SDValue, 8> Ops;
4764   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4765     if (OnlyLoad) {
4766       // We don't need to serialize loads against other loads.
4767       Ops.push_back(DAG.getRoot());
4768     } else {
4769       Ops.push_back(getRoot());
4770     }
4771   }
4772 
4773   // Info is set by getTgtMemIntrinsic
4774   TargetLowering::IntrinsicInfo Info;
4775   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4776   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4777                                                DAG.getMachineFunction(),
4778                                                Intrinsic);
4779 
4780   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4781   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4782       Info.opc == ISD::INTRINSIC_W_CHAIN)
4783     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4784                                         TLI.getPointerTy(DAG.getDataLayout())));
4785 
4786   // Add all operands of the call to the operand list.
4787   for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
4788     const Value *Arg = I.getArgOperand(i);
4789     if (!I.paramHasAttr(i, Attribute::ImmArg)) {
4790       Ops.push_back(getValue(Arg));
4791       continue;
4792     }
4793 
4794     // Use TargetConstant instead of a regular constant for immarg.
4795     EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
4796     if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4797       assert(CI->getBitWidth() <= 64 &&
4798              "large intrinsic immediates not handled");
4799       Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
4800     } else {
4801       Ops.push_back(
4802           DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
4803     }
4804   }
4805 
4806   SmallVector<EVT, 4> ValueVTs;
4807   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4808 
4809   if (HasChain)
4810     ValueVTs.push_back(MVT::Other);
4811 
4812   SDVTList VTs = DAG.getVTList(ValueVTs);
4813 
4814   // Propagate fast-math-flags from IR to node(s).
4815   SDNodeFlags Flags;
4816   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
4817     Flags.copyFMF(*FPMO);
4818   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
4819 
4820   // Create the node.
4821   SDValue Result;
4822   if (IsTgtIntrinsic) {
4823     // This is target intrinsic that touches memory
4824     Result =
4825         DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4826                                 MachinePointerInfo(Info.ptrVal, Info.offset),
4827                                 Info.align, Info.flags, Info.size,
4828                                 I.getAAMetadata());
4829   } else if (!HasChain) {
4830     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4831   } else if (!I.getType()->isVoidTy()) {
4832     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4833   } else {
4834     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4835   }
4836 
4837   if (HasChain) {
4838     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4839     if (OnlyLoad)
4840       PendingLoads.push_back(Chain);
4841     else
4842       DAG.setRoot(Chain);
4843   }
4844 
4845   if (!I.getType()->isVoidTy()) {
4846     if (!isa<VectorType>(I.getType()))
4847       Result = lowerRangeToAssertZExt(DAG, I, Result);
4848 
4849     MaybeAlign Alignment = I.getRetAlign();
4850     if (!Alignment)
4851       Alignment = F->getAttributes().getRetAlignment();
4852     // Insert `assertalign` node if there's an alignment.
4853     if (InsertAssertAlign && Alignment) {
4854       Result =
4855           DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
4856     }
4857 
4858     setValue(&I, Result);
4859   }
4860 }
4861 
4862 /// GetSignificand - Get the significand and build it into a floating-point
4863 /// number with exponent of 1:
4864 ///
4865 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4866 ///
4867 /// where Op is the hexadecimal representation of floating point value.
4868 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4869   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4870                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4871   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4872                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4873   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4874 }
4875 
4876 /// GetExponent - Get the exponent:
4877 ///
4878 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4879 ///
4880 /// where Op is the hexadecimal representation of floating point value.
4881 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4882                            const TargetLowering &TLI, const SDLoc &dl) {
4883   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4884                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4885   SDValue t1 = DAG.getNode(
4886       ISD::SRL, dl, MVT::i32, t0,
4887       DAG.getConstant(23, dl,
4888                       TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
4889   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4890                            DAG.getConstant(127, dl, MVT::i32));
4891   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4892 }
4893 
4894 /// getF32Constant - Get 32-bit floating point constant.
4895 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4896                               const SDLoc &dl) {
4897   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4898                            MVT::f32);
4899 }
4900 
4901 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4902                                        SelectionDAG &DAG) {
4903   // TODO: What fast-math-flags should be set on the floating-point nodes?
4904 
4905   //   IntegerPartOfX = ((int32_t)(t0);
4906   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4907 
4908   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4909   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4910   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4911 
4912   //   IntegerPartOfX <<= 23;
4913   IntegerPartOfX =
4914       DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4915                   DAG.getConstant(23, dl,
4916                                   DAG.getTargetLoweringInfo().getShiftAmountTy(
4917                                       MVT::i32, DAG.getDataLayout())));
4918 
4919   SDValue TwoToFractionalPartOfX;
4920   if (LimitFloatPrecision <= 6) {
4921     // For floating-point precision of 6:
4922     //
4923     //   TwoToFractionalPartOfX =
4924     //     0.997535578f +
4925     //       (0.735607626f + 0.252464424f * x) * x;
4926     //
4927     // error 0.0144103317, which is 6 bits
4928     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4929                              getF32Constant(DAG, 0x3e814304, dl));
4930     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4931                              getF32Constant(DAG, 0x3f3c50c8, dl));
4932     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4933     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4934                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
4935   } else if (LimitFloatPrecision <= 12) {
4936     // For floating-point precision of 12:
4937     //
4938     //   TwoToFractionalPartOfX =
4939     //     0.999892986f +
4940     //       (0.696457318f +
4941     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4942     //
4943     // error 0.000107046256, which is 13 to 14 bits
4944     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4945                              getF32Constant(DAG, 0x3da235e3, dl));
4946     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4947                              getF32Constant(DAG, 0x3e65b8f3, dl));
4948     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4949     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4950                              getF32Constant(DAG, 0x3f324b07, dl));
4951     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4952     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4953                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
4954   } else { // LimitFloatPrecision <= 18
4955     // For floating-point precision of 18:
4956     //
4957     //   TwoToFractionalPartOfX =
4958     //     0.999999982f +
4959     //       (0.693148872f +
4960     //         (0.240227044f +
4961     //           (0.554906021e-1f +
4962     //             (0.961591928e-2f +
4963     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4964     // error 2.47208000*10^(-7), which is better than 18 bits
4965     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4966                              getF32Constant(DAG, 0x3924b03e, dl));
4967     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4968                              getF32Constant(DAG, 0x3ab24b87, dl));
4969     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4970     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4971                              getF32Constant(DAG, 0x3c1d8c17, dl));
4972     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4973     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4974                              getF32Constant(DAG, 0x3d634a1d, dl));
4975     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4976     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4977                              getF32Constant(DAG, 0x3e75fe14, dl));
4978     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4979     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4980                               getF32Constant(DAG, 0x3f317234, dl));
4981     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4982     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4983                                          getF32Constant(DAG, 0x3f800000, dl));
4984   }
4985 
4986   // Add the exponent into the result in integer domain.
4987   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4988   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4989                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4990 }
4991 
4992 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4993 /// limited-precision mode.
4994 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4995                          const TargetLowering &TLI, SDNodeFlags Flags) {
4996   if (Op.getValueType() == MVT::f32 &&
4997       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4998 
4999     // Put the exponent in the right bit position for later addition to the
5000     // final result:
5001     //
5002     // t0 = Op * log2(e)
5003 
5004     // TODO: What fast-math-flags should be set here?
5005     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5006                              DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5007     return getLimitedPrecisionExp2(t0, dl, DAG);
5008   }
5009 
5010   // No special expansion.
5011   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5012 }
5013 
5014 /// expandLog - Lower a log intrinsic. Handles the special sequences for
5015 /// limited-precision mode.
5016 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5017                          const TargetLowering &TLI, SDNodeFlags Flags) {
5018   // TODO: What fast-math-flags should be set on the floating-point nodes?
5019 
5020   if (Op.getValueType() == MVT::f32 &&
5021       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5022     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5023 
5024     // Scale the exponent by log(2).
5025     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5026     SDValue LogOfExponent =
5027         DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5028                     DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5029 
5030     // Get the significand and build it into a floating-point number with
5031     // exponent of 1.
5032     SDValue X = GetSignificand(DAG, Op1, dl);
5033 
5034     SDValue LogOfMantissa;
5035     if (LimitFloatPrecision <= 6) {
5036       // For floating-point precision of 6:
5037       //
5038       //   LogofMantissa =
5039       //     -1.1609546f +
5040       //       (1.4034025f - 0.23903021f * x) * x;
5041       //
5042       // error 0.0034276066, which is better than 8 bits
5043       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5044                                getF32Constant(DAG, 0xbe74c456, dl));
5045       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5046                                getF32Constant(DAG, 0x3fb3a2b1, dl));
5047       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5048       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5049                                   getF32Constant(DAG, 0x3f949a29, dl));
5050     } else if (LimitFloatPrecision <= 12) {
5051       // For floating-point precision of 12:
5052       //
5053       //   LogOfMantissa =
5054       //     -1.7417939f +
5055       //       (2.8212026f +
5056       //         (-1.4699568f +
5057       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5058       //
5059       // error 0.000061011436, which is 14 bits
5060       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5061                                getF32Constant(DAG, 0xbd67b6d6, dl));
5062       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5063                                getF32Constant(DAG, 0x3ee4f4b8, dl));
5064       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5065       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5066                                getF32Constant(DAG, 0x3fbc278b, dl));
5067       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5068       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5069                                getF32Constant(DAG, 0x40348e95, dl));
5070       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5071       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5072                                   getF32Constant(DAG, 0x3fdef31a, dl));
5073     } else { // LimitFloatPrecision <= 18
5074       // For floating-point precision of 18:
5075       //
5076       //   LogOfMantissa =
5077       //     -2.1072184f +
5078       //       (4.2372794f +
5079       //         (-3.7029485f +
5080       //           (2.2781945f +
5081       //             (-0.87823314f +
5082       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5083       //
5084       // error 0.0000023660568, which is better than 18 bits
5085       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5086                                getF32Constant(DAG, 0xbc91e5ac, dl));
5087       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5088                                getF32Constant(DAG, 0x3e4350aa, dl));
5089       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5090       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5091                                getF32Constant(DAG, 0x3f60d3e3, dl));
5092       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5093       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5094                                getF32Constant(DAG, 0x4011cdf0, dl));
5095       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5096       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5097                                getF32Constant(DAG, 0x406cfd1c, dl));
5098       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5099       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5100                                getF32Constant(DAG, 0x408797cb, dl));
5101       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5102       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5103                                   getF32Constant(DAG, 0x4006dcab, dl));
5104     }
5105 
5106     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5107   }
5108 
5109   // No special expansion.
5110   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5111 }
5112 
5113 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5114 /// limited-precision mode.
5115 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5116                           const TargetLowering &TLI, SDNodeFlags Flags) {
5117   // TODO: What fast-math-flags should be set on the floating-point nodes?
5118 
5119   if (Op.getValueType() == MVT::f32 &&
5120       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5121     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5122 
5123     // Get the exponent.
5124     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5125 
5126     // Get the significand and build it into a floating-point number with
5127     // exponent of 1.
5128     SDValue X = GetSignificand(DAG, Op1, dl);
5129 
5130     // Different possible minimax approximations of significand in
5131     // floating-point for various degrees of accuracy over [1,2].
5132     SDValue Log2ofMantissa;
5133     if (LimitFloatPrecision <= 6) {
5134       // For floating-point precision of 6:
5135       //
5136       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5137       //
5138       // error 0.0049451742, which is more than 7 bits
5139       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5140                                getF32Constant(DAG, 0xbeb08fe0, dl));
5141       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5142                                getF32Constant(DAG, 0x40019463, dl));
5143       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5144       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5145                                    getF32Constant(DAG, 0x3fd6633d, dl));
5146     } else if (LimitFloatPrecision <= 12) {
5147       // For floating-point precision of 12:
5148       //
5149       //   Log2ofMantissa =
5150       //     -2.51285454f +
5151       //       (4.07009056f +
5152       //         (-2.12067489f +
5153       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5154       //
5155       // error 0.0000876136000, which is better than 13 bits
5156       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5157                                getF32Constant(DAG, 0xbda7262e, dl));
5158       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5159                                getF32Constant(DAG, 0x3f25280b, dl));
5160       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5161       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5162                                getF32Constant(DAG, 0x4007b923, dl));
5163       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5164       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5165                                getF32Constant(DAG, 0x40823e2f, dl));
5166       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5167       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5168                                    getF32Constant(DAG, 0x4020d29c, dl));
5169     } else { // LimitFloatPrecision <= 18
5170       // For floating-point precision of 18:
5171       //
5172       //   Log2ofMantissa =
5173       //     -3.0400495f +
5174       //       (6.1129976f +
5175       //         (-5.3420409f +
5176       //           (3.2865683f +
5177       //             (-1.2669343f +
5178       //               (0.27515199f -
5179       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5180       //
5181       // error 0.0000018516, which is better than 18 bits
5182       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5183                                getF32Constant(DAG, 0xbcd2769e, dl));
5184       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5185                                getF32Constant(DAG, 0x3e8ce0b9, dl));
5186       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5187       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5188                                getF32Constant(DAG, 0x3fa22ae7, dl));
5189       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5190       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5191                                getF32Constant(DAG, 0x40525723, dl));
5192       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5193       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5194                                getF32Constant(DAG, 0x40aaf200, dl));
5195       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5196       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5197                                getF32Constant(DAG, 0x40c39dad, dl));
5198       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5199       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5200                                    getF32Constant(DAG, 0x4042902c, dl));
5201     }
5202 
5203     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5204   }
5205 
5206   // No special expansion.
5207   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5208 }
5209 
5210 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5211 /// limited-precision mode.
5212 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5213                            const TargetLowering &TLI, SDNodeFlags Flags) {
5214   // TODO: What fast-math-flags should be set on the floating-point nodes?
5215 
5216   if (Op.getValueType() == MVT::f32 &&
5217       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5218     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5219 
5220     // Scale the exponent by log10(2) [0.30102999f].
5221     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5222     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5223                                         getF32Constant(DAG, 0x3e9a209a, dl));
5224 
5225     // Get the significand and build it into a floating-point number with
5226     // exponent of 1.
5227     SDValue X = GetSignificand(DAG, Op1, dl);
5228 
5229     SDValue Log10ofMantissa;
5230     if (LimitFloatPrecision <= 6) {
5231       // For floating-point precision of 6:
5232       //
5233       //   Log10ofMantissa =
5234       //     -0.50419619f +
5235       //       (0.60948995f - 0.10380950f * x) * x;
5236       //
5237       // error 0.0014886165, which is 6 bits
5238       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5239                                getF32Constant(DAG, 0xbdd49a13, dl));
5240       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5241                                getF32Constant(DAG, 0x3f1c0789, dl));
5242       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5243       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5244                                     getF32Constant(DAG, 0x3f011300, dl));
5245     } else if (LimitFloatPrecision <= 12) {
5246       // For floating-point precision of 12:
5247       //
5248       //   Log10ofMantissa =
5249       //     -0.64831180f +
5250       //       (0.91751397f +
5251       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5252       //
5253       // error 0.00019228036, which is better than 12 bits
5254       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5255                                getF32Constant(DAG, 0x3d431f31, dl));
5256       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5257                                getF32Constant(DAG, 0x3ea21fb2, dl));
5258       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5259       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5260                                getF32Constant(DAG, 0x3f6ae232, dl));
5261       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5262       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5263                                     getF32Constant(DAG, 0x3f25f7c3, dl));
5264     } else { // LimitFloatPrecision <= 18
5265       // For floating-point precision of 18:
5266       //
5267       //   Log10ofMantissa =
5268       //     -0.84299375f +
5269       //       (1.5327582f +
5270       //         (-1.0688956f +
5271       //           (0.49102474f +
5272       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5273       //
5274       // error 0.0000037995730, which is better than 18 bits
5275       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5276                                getF32Constant(DAG, 0x3c5d51ce, dl));
5277       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5278                                getF32Constant(DAG, 0x3e00685a, dl));
5279       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5280       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5281                                getF32Constant(DAG, 0x3efb6798, dl));
5282       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5283       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5284                                getF32Constant(DAG, 0x3f88d192, dl));
5285       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5286       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5287                                getF32Constant(DAG, 0x3fc4316c, dl));
5288       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5289       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5290                                     getF32Constant(DAG, 0x3f57ce70, dl));
5291     }
5292 
5293     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5294   }
5295 
5296   // No special expansion.
5297   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5298 }
5299 
5300 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5301 /// limited-precision mode.
5302 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5303                           const TargetLowering &TLI, SDNodeFlags Flags) {
5304   if (Op.getValueType() == MVT::f32 &&
5305       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5306     return getLimitedPrecisionExp2(Op, dl, DAG);
5307 
5308   // No special expansion.
5309   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5310 }
5311 
5312 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
5313 /// limited-precision mode with x == 10.0f.
5314 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5315                          SelectionDAG &DAG, const TargetLowering &TLI,
5316                          SDNodeFlags Flags) {
5317   bool IsExp10 = false;
5318   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5319       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5320     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5321       APFloat Ten(10.0f);
5322       IsExp10 = LHSC->isExactlyValue(Ten);
5323     }
5324   }
5325 
5326   // TODO: What fast-math-flags should be set on the FMUL node?
5327   if (IsExp10) {
5328     // Put the exponent in the right bit position for later addition to the
5329     // final result:
5330     //
5331     //   #define LOG2OF10 3.3219281f
5332     //   t0 = Op * LOG2OF10;
5333     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5334                              getF32Constant(DAG, 0x40549a78, dl));
5335     return getLimitedPrecisionExp2(t0, dl, DAG);
5336   }
5337 
5338   // No special expansion.
5339   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5340 }
5341 
5342 /// ExpandPowI - Expand a llvm.powi intrinsic.
5343 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5344                           SelectionDAG &DAG) {
5345   // If RHS is a constant, we can expand this out to a multiplication tree,
5346   // otherwise we end up lowering to a call to __powidf2 (for example).  When
5347   // optimizing for size, we only want to do this if the expansion would produce
5348   // a small number of multiplies, otherwise we do the full expansion.
5349   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5350     // Get the exponent as a positive value.
5351     unsigned Val = RHSC->getSExtValue();
5352     if ((int)Val < 0) Val = -Val;
5353 
5354     // powi(x, 0) -> 1.0
5355     if (Val == 0)
5356       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5357 
5358     bool OptForSize = DAG.shouldOptForSize();
5359     if (!OptForSize ||
5360         // If optimizing for size, don't insert too many multiplies.
5361         // This inserts up to 5 multiplies.
5362         countPopulation(Val) + Log2_32(Val) < 7) {
5363       // We use the simple binary decomposition method to generate the multiply
5364       // sequence.  There are more optimal ways to do this (for example,
5365       // powi(x,15) generates one more multiply than it should), but this has
5366       // the benefit of being both really simple and much better than a libcall.
5367       SDValue Res;  // Logically starts equal to 1.0
5368       SDValue CurSquare = LHS;
5369       // TODO: Intrinsics should have fast-math-flags that propagate to these
5370       // nodes.
5371       while (Val) {
5372         if (Val & 1) {
5373           if (Res.getNode())
5374             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
5375           else
5376             Res = CurSquare;  // 1.0*CurSquare.
5377         }
5378 
5379         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5380                                 CurSquare, CurSquare);
5381         Val >>= 1;
5382       }
5383 
5384       // If the original was negative, invert the result, producing 1/(x*x*x).
5385       if (RHSC->getSExtValue() < 0)
5386         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5387                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5388       return Res;
5389     }
5390   }
5391 
5392   // Otherwise, expand to a libcall.
5393   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5394 }
5395 
5396 static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5397                             SDValue LHS, SDValue RHS, SDValue Scale,
5398                             SelectionDAG &DAG, const TargetLowering &TLI) {
5399   EVT VT = LHS.getValueType();
5400   bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5401   bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5402   LLVMContext &Ctx = *DAG.getContext();
5403 
5404   // If the type is legal but the operation isn't, this node might survive all
5405   // the way to operation legalization. If we end up there and we do not have
5406   // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5407   // node.
5408 
5409   // Coax the legalizer into expanding the node during type legalization instead
5410   // by bumping the size by one bit. This will force it to Promote, enabling the
5411   // early expansion and avoiding the need to expand later.
5412 
5413   // We don't have to do this if Scale is 0; that can always be expanded, unless
5414   // it's a saturating signed operation. Those can experience true integer
5415   // division overflow, a case which we must avoid.
5416 
5417   // FIXME: We wouldn't have to do this (or any of the early
5418   // expansion/promotion) if it was possible to expand a libcall of an
5419   // illegal type during operation legalization. But it's not, so things
5420   // get a bit hacky.
5421   unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
5422   if ((ScaleInt > 0 || (Saturating && Signed)) &&
5423       (TLI.isTypeLegal(VT) ||
5424        (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5425     TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5426         Opcode, VT, ScaleInt);
5427     if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5428       EVT PromVT;
5429       if (VT.isScalarInteger())
5430         PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5431       else if (VT.isVector()) {
5432         PromVT = VT.getVectorElementType();
5433         PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5434         PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5435       } else
5436         llvm_unreachable("Wrong VT for DIVFIX?");
5437       if (Signed) {
5438         LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
5439         RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
5440       } else {
5441         LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
5442         RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
5443       }
5444       EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5445       // For saturating operations, we need to shift up the LHS to get the
5446       // proper saturation width, and then shift down again afterwards.
5447       if (Saturating)
5448         LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5449                           DAG.getConstant(1, DL, ShiftTy));
5450       SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5451       if (Saturating)
5452         Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5453                           DAG.getConstant(1, DL, ShiftTy));
5454       return DAG.getZExtOrTrunc(Res, DL, VT);
5455     }
5456   }
5457 
5458   return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5459 }
5460 
5461 // getUnderlyingArgRegs - Find underlying registers used for a truncated,
5462 // bitcasted, or split argument. Returns a list of <Register, size in bits>
5463 static void
5464 getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5465                      const SDValue &N) {
5466   switch (N.getOpcode()) {
5467   case ISD::CopyFromReg: {
5468     SDValue Op = N.getOperand(1);
5469     Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5470                       Op.getValueType().getSizeInBits());
5471     return;
5472   }
5473   case ISD::BITCAST:
5474   case ISD::AssertZext:
5475   case ISD::AssertSext:
5476   case ISD::TRUNCATE:
5477     getUnderlyingArgRegs(Regs, N.getOperand(0));
5478     return;
5479   case ISD::BUILD_PAIR:
5480   case ISD::BUILD_VECTOR:
5481   case ISD::CONCAT_VECTORS:
5482     for (SDValue Op : N->op_values())
5483       getUnderlyingArgRegs(Regs, Op);
5484     return;
5485   default:
5486     return;
5487   }
5488 }
5489 
5490 /// If the DbgValueInst is a dbg_value of a function argument, create the
5491 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
5492 /// instruction selection, they will be inserted to the entry BB.
5493 /// We don't currently support this for variadic dbg_values, as they shouldn't
5494 /// appear for function arguments or in the prologue.
5495 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5496     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5497     DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5498   const Argument *Arg = dyn_cast<Argument>(V);
5499   if (!Arg)
5500     return false;
5501 
5502   MachineFunction &MF = DAG.getMachineFunction();
5503   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5504 
5505   // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5506   // we've been asked to pursue.
5507   auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5508                               bool Indirect) {
5509     if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5510       // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5511       // pointing at the VReg, which will be patched up later.
5512       auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5513       auto MIB = BuildMI(MF, DL, Inst);
5514       MIB.addReg(Reg);
5515       MIB.addImm(0);
5516       MIB.addMetadata(Variable);
5517       auto *NewDIExpr = FragExpr;
5518       // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5519       // the DIExpression.
5520       if (Indirect)
5521         NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5522       MIB.addMetadata(NewDIExpr);
5523       return MIB;
5524     } else {
5525       // Create a completely standard DBG_VALUE.
5526       auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5527       return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5528     }
5529   };
5530 
5531   if (Kind == FuncArgumentDbgValueKind::Value) {
5532     // ArgDbgValues are hoisted to the beginning of the entry block. So we
5533     // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5534     // the entry block.
5535     bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5536     if (!IsInEntryBlock)
5537       return false;
5538 
5539     // ArgDbgValues are hoisted to the beginning of the entry block.  So we
5540     // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5541     // variable that also is a param.
5542     //
5543     // Although, if we are at the top of the entry block already, we can still
5544     // emit using ArgDbgValue. This might catch some situations when the
5545     // dbg.value refers to an argument that isn't used in the entry block, so
5546     // any CopyToReg node would be optimized out and the only way to express
5547     // this DBG_VALUE is by using the physical reg (or FI) as done in this
5548     // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
5549     // we should only emit as ArgDbgValue if the Variable is an argument to the
5550     // current function, and the dbg.value intrinsic is found in the entry
5551     // block.
5552     bool VariableIsFunctionInputArg = Variable->isParameter() &&
5553         !DL->getInlinedAt();
5554     bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5555     if (!IsInPrologue && !VariableIsFunctionInputArg)
5556       return false;
5557 
5558     // Here we assume that a function argument on IR level only can be used to
5559     // describe one input parameter on source level. If we for example have
5560     // source code like this
5561     //
5562     //    struct A { long x, y; };
5563     //    void foo(struct A a, long b) {
5564     //      ...
5565     //      b = a.x;
5566     //      ...
5567     //    }
5568     //
5569     // and IR like this
5570     //
5571     //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
5572     //  entry:
5573     //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5574     //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5575     //    call void @llvm.dbg.value(metadata i32 %b, "b",
5576     //    ...
5577     //    call void @llvm.dbg.value(metadata i32 %a1, "b"
5578     //    ...
5579     //
5580     // then the last dbg.value is describing a parameter "b" using a value that
5581     // is an argument. But since we already has used %a1 to describe a parameter
5582     // we should not handle that last dbg.value here (that would result in an
5583     // incorrect hoisting of the DBG_VALUE to the function entry).
5584     // Notice that we allow one dbg.value per IR level argument, to accommodate
5585     // for the situation with fragments above.
5586     if (VariableIsFunctionInputArg) {
5587       unsigned ArgNo = Arg->getArgNo();
5588       if (ArgNo >= FuncInfo.DescribedArgs.size())
5589         FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
5590       else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
5591         return false;
5592       FuncInfo.DescribedArgs.set(ArgNo);
5593     }
5594   }
5595 
5596   bool IsIndirect = false;
5597   Optional<MachineOperand> Op;
5598   // Some arguments' frame index is recorded during argument lowering.
5599   int FI = FuncInfo.getArgumentFrameIndex(Arg);
5600   if (FI != std::numeric_limits<int>::max())
5601     Op = MachineOperand::CreateFI(FI);
5602 
5603   SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
5604   if (!Op && N.getNode()) {
5605     getUnderlyingArgRegs(ArgRegsAndSizes, N);
5606     Register Reg;
5607     if (ArgRegsAndSizes.size() == 1)
5608       Reg = ArgRegsAndSizes.front().first;
5609 
5610     if (Reg && Reg.isVirtual()) {
5611       MachineRegisterInfo &RegInfo = MF.getRegInfo();
5612       Register PR = RegInfo.getLiveInPhysReg(Reg);
5613       if (PR)
5614         Reg = PR;
5615     }
5616     if (Reg) {
5617       Op = MachineOperand::CreateReg(Reg, false);
5618       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5619     }
5620   }
5621 
5622   if (!Op && N.getNode()) {
5623     // Check if frame index is available.
5624     SDValue LCandidate = peekThroughBitcasts(N);
5625     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
5626       if (FrameIndexSDNode *FINode =
5627           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
5628         Op = MachineOperand::CreateFI(FINode->getIndex());
5629   }
5630 
5631   if (!Op) {
5632     // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
5633     auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
5634                                          SplitRegs) {
5635       unsigned Offset = 0;
5636       for (const auto &RegAndSize : SplitRegs) {
5637         // If the expression is already a fragment, the current register
5638         // offset+size might extend beyond the fragment. In this case, only
5639         // the register bits that are inside the fragment are relevant.
5640         int RegFragmentSizeInBits = RegAndSize.second;
5641         if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
5642           uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
5643           // The register is entirely outside the expression fragment,
5644           // so is irrelevant for debug info.
5645           if (Offset >= ExprFragmentSizeInBits)
5646             break;
5647           // The register is partially outside the expression fragment, only
5648           // the low bits within the fragment are relevant for debug info.
5649           if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
5650             RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
5651           }
5652         }
5653 
5654         auto FragmentExpr = DIExpression::createFragmentExpression(
5655             Expr, Offset, RegFragmentSizeInBits);
5656         Offset += RegAndSize.second;
5657         // If a valid fragment expression cannot be created, the variable's
5658         // correct value cannot be determined and so it is set as Undef.
5659         if (!FragmentExpr) {
5660           SDDbgValue *SDV = DAG.getConstantDbgValue(
5661               Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
5662           DAG.AddDbgValue(SDV, false);
5663           continue;
5664         }
5665         MachineInstr *NewMI =
5666             MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
5667                              Kind != FuncArgumentDbgValueKind::Value);
5668         FuncInfo.ArgDbgValues.push_back(NewMI);
5669       }
5670     };
5671 
5672     // Check if ValueMap has reg number.
5673     DenseMap<const Value *, Register>::const_iterator
5674       VMI = FuncInfo.ValueMap.find(V);
5675     if (VMI != FuncInfo.ValueMap.end()) {
5676       const auto &TLI = DAG.getTargetLoweringInfo();
5677       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
5678                        V->getType(), None);
5679       if (RFV.occupiesMultipleRegs()) {
5680         splitMultiRegDbgValue(RFV.getRegsAndSizes());
5681         return true;
5682       }
5683 
5684       Op = MachineOperand::CreateReg(VMI->second, false);
5685       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5686     } else if (ArgRegsAndSizes.size() > 1) {
5687       // This was split due to the calling convention, and no virtual register
5688       // mapping exists for the value.
5689       splitMultiRegDbgValue(ArgRegsAndSizes);
5690       return true;
5691     }
5692   }
5693 
5694   if (!Op)
5695     return false;
5696 
5697   assert(Variable->isValidLocationForIntrinsic(DL) &&
5698          "Expected inlined-at fields to agree");
5699   MachineInstr *NewMI = nullptr;
5700 
5701   if (Op->isReg())
5702     NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
5703   else
5704     NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
5705                     Variable, Expr);
5706 
5707   // Otherwise, use ArgDbgValues.
5708   FuncInfo.ArgDbgValues.push_back(NewMI);
5709   return true;
5710 }
5711 
5712 /// Return the appropriate SDDbgValue based on N.
5713 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5714                                              DILocalVariable *Variable,
5715                                              DIExpression *Expr,
5716                                              const DebugLoc &dl,
5717                                              unsigned DbgSDNodeOrder) {
5718   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5719     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5720     // stack slot locations.
5721     //
5722     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5723     // debug values here after optimization:
5724     //
5725     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5726     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5727     //
5728     // Both describe the direct values of their associated variables.
5729     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5730                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5731   }
5732   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5733                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5734 }
5735 
5736 static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
5737   switch (Intrinsic) {
5738   case Intrinsic::smul_fix:
5739     return ISD::SMULFIX;
5740   case Intrinsic::umul_fix:
5741     return ISD::UMULFIX;
5742   case Intrinsic::smul_fix_sat:
5743     return ISD::SMULFIXSAT;
5744   case Intrinsic::umul_fix_sat:
5745     return ISD::UMULFIXSAT;
5746   case Intrinsic::sdiv_fix:
5747     return ISD::SDIVFIX;
5748   case Intrinsic::udiv_fix:
5749     return ISD::UDIVFIX;
5750   case Intrinsic::sdiv_fix_sat:
5751     return ISD::SDIVFIXSAT;
5752   case Intrinsic::udiv_fix_sat:
5753     return ISD::UDIVFIXSAT;
5754   default:
5755     llvm_unreachable("Unhandled fixed point intrinsic");
5756   }
5757 }
5758 
5759 void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
5760                                            const char *FunctionName) {
5761   assert(FunctionName && "FunctionName must not be nullptr");
5762   SDValue Callee = DAG.getExternalSymbol(
5763       FunctionName,
5764       DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
5765   LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5766 }
5767 
5768 /// Given a @llvm.call.preallocated.setup, return the corresponding
5769 /// preallocated call.
5770 static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
5771   assert(cast<CallBase>(PreallocatedSetup)
5772                  ->getCalledFunction()
5773                  ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
5774          "expected call_preallocated_setup Value");
5775   for (auto *U : PreallocatedSetup->users()) {
5776     auto *UseCall = cast<CallBase>(U);
5777     const Function *Fn = UseCall->getCalledFunction();
5778     if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
5779       return UseCall;
5780     }
5781   }
5782   llvm_unreachable("expected corresponding call to preallocated setup/arg");
5783 }
5784 
5785 /// Lower the call to the specified intrinsic function.
5786 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
5787                                              unsigned Intrinsic) {
5788   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5789   SDLoc sdl = getCurSDLoc();
5790   DebugLoc dl = getCurDebugLoc();
5791   SDValue Res;
5792 
5793   SDNodeFlags Flags;
5794   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
5795     Flags.copyFMF(*FPOp);
5796 
5797   switch (Intrinsic) {
5798   default:
5799     // By default, turn this into a target intrinsic node.
5800     visitTargetIntrinsic(I, Intrinsic);
5801     return;
5802   case Intrinsic::vscale: {
5803     match(&I, m_VScale(DAG.getDataLayout()));
5804     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5805     setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
5806     return;
5807   }
5808   case Intrinsic::vastart:  visitVAStart(I); return;
5809   case Intrinsic::vaend:    visitVAEnd(I); return;
5810   case Intrinsic::vacopy:   visitVACopy(I); return;
5811   case Intrinsic::returnaddress:
5812     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5813                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5814                              getValue(I.getArgOperand(0))));
5815     return;
5816   case Intrinsic::addressofreturnaddress:
5817     setValue(&I,
5818              DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5819                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5820     return;
5821   case Intrinsic::sponentry:
5822     setValue(&I,
5823              DAG.getNode(ISD::SPONENTRY, sdl,
5824                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5825     return;
5826   case Intrinsic::frameaddress:
5827     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5828                              TLI.getFrameIndexTy(DAG.getDataLayout()),
5829                              getValue(I.getArgOperand(0))));
5830     return;
5831   case Intrinsic::read_volatile_register:
5832   case Intrinsic::read_register: {
5833     Value *Reg = I.getArgOperand(0);
5834     SDValue Chain = getRoot();
5835     SDValue RegName =
5836         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5837     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5838     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5839       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5840     setValue(&I, Res);
5841     DAG.setRoot(Res.getValue(1));
5842     return;
5843   }
5844   case Intrinsic::write_register: {
5845     Value *Reg = I.getArgOperand(0);
5846     Value *RegValue = I.getArgOperand(1);
5847     SDValue Chain = getRoot();
5848     SDValue RegName =
5849         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5850     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5851                             RegName, getValue(RegValue)));
5852     return;
5853   }
5854   case Intrinsic::memcpy: {
5855     const auto &MCI = cast<MemCpyInst>(I);
5856     SDValue Op1 = getValue(I.getArgOperand(0));
5857     SDValue Op2 = getValue(I.getArgOperand(1));
5858     SDValue Op3 = getValue(I.getArgOperand(2));
5859     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5860     Align DstAlign = MCI.getDestAlign().valueOrOne();
5861     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5862     Align Alignment = commonAlignment(DstAlign, SrcAlign);
5863     bool isVol = MCI.isVolatile();
5864     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5865     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5866     // node.
5867     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5868     SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5869                                /* AlwaysInline */ false, isTC,
5870                                MachinePointerInfo(I.getArgOperand(0)),
5871                                MachinePointerInfo(I.getArgOperand(1)),
5872                                I.getAAMetadata());
5873     updateDAGForMaybeTailCall(MC);
5874     return;
5875   }
5876   case Intrinsic::memcpy_inline: {
5877     const auto &MCI = cast<MemCpyInlineInst>(I);
5878     SDValue Dst = getValue(I.getArgOperand(0));
5879     SDValue Src = getValue(I.getArgOperand(1));
5880     SDValue Size = getValue(I.getArgOperand(2));
5881     assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
5882     // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
5883     Align DstAlign = MCI.getDestAlign().valueOrOne();
5884     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5885     Align Alignment = commonAlignment(DstAlign, SrcAlign);
5886     bool isVol = MCI.isVolatile();
5887     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5888     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5889     // node.
5890     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
5891                                /* AlwaysInline */ true, isTC,
5892                                MachinePointerInfo(I.getArgOperand(0)),
5893                                MachinePointerInfo(I.getArgOperand(1)),
5894                                I.getAAMetadata());
5895     updateDAGForMaybeTailCall(MC);
5896     return;
5897   }
5898   case Intrinsic::memset: {
5899     const auto &MSI = cast<MemSetInst>(I);
5900     SDValue Op1 = getValue(I.getArgOperand(0));
5901     SDValue Op2 = getValue(I.getArgOperand(1));
5902     SDValue Op3 = getValue(I.getArgOperand(2));
5903     // @llvm.memset defines 0 and 1 to both mean no alignment.
5904     Align Alignment = MSI.getDestAlign().valueOrOne();
5905     bool isVol = MSI.isVolatile();
5906     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5907     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5908     SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC,
5909                                MachinePointerInfo(I.getArgOperand(0)),
5910                                I.getAAMetadata());
5911     updateDAGForMaybeTailCall(MS);
5912     return;
5913   }
5914   case Intrinsic::memmove: {
5915     const auto &MMI = cast<MemMoveInst>(I);
5916     SDValue Op1 = getValue(I.getArgOperand(0));
5917     SDValue Op2 = getValue(I.getArgOperand(1));
5918     SDValue Op3 = getValue(I.getArgOperand(2));
5919     // @llvm.memmove defines 0 and 1 to both mean no alignment.
5920     Align DstAlign = MMI.getDestAlign().valueOrOne();
5921     Align SrcAlign = MMI.getSourceAlign().valueOrOne();
5922     Align Alignment = commonAlignment(DstAlign, SrcAlign);
5923     bool isVol = MMI.isVolatile();
5924     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5925     // FIXME: Support passing different dest/src alignments to the memmove DAG
5926     // node.
5927     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5928     SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5929                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
5930                                 MachinePointerInfo(I.getArgOperand(1)),
5931                                 I.getAAMetadata());
5932     updateDAGForMaybeTailCall(MM);
5933     return;
5934   }
5935   case Intrinsic::memcpy_element_unordered_atomic: {
5936     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5937     SDValue Dst = getValue(MI.getRawDest());
5938     SDValue Src = getValue(MI.getRawSource());
5939     SDValue Length = getValue(MI.getLength());
5940 
5941     unsigned DstAlign = MI.getDestAlignment();
5942     unsigned SrcAlign = MI.getSourceAlignment();
5943     Type *LengthTy = MI.getLength()->getType();
5944     unsigned ElemSz = MI.getElementSizeInBytes();
5945     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5946     SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5947                                      SrcAlign, Length, LengthTy, ElemSz, isTC,
5948                                      MachinePointerInfo(MI.getRawDest()),
5949                                      MachinePointerInfo(MI.getRawSource()));
5950     updateDAGForMaybeTailCall(MC);
5951     return;
5952   }
5953   case Intrinsic::memmove_element_unordered_atomic: {
5954     auto &MI = cast<AtomicMemMoveInst>(I);
5955     SDValue Dst = getValue(MI.getRawDest());
5956     SDValue Src = getValue(MI.getRawSource());
5957     SDValue Length = getValue(MI.getLength());
5958 
5959     unsigned DstAlign = MI.getDestAlignment();
5960     unsigned SrcAlign = MI.getSourceAlignment();
5961     Type *LengthTy = MI.getLength()->getType();
5962     unsigned ElemSz = MI.getElementSizeInBytes();
5963     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5964     SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5965                                       SrcAlign, Length, LengthTy, ElemSz, isTC,
5966                                       MachinePointerInfo(MI.getRawDest()),
5967                                       MachinePointerInfo(MI.getRawSource()));
5968     updateDAGForMaybeTailCall(MC);
5969     return;
5970   }
5971   case Intrinsic::memset_element_unordered_atomic: {
5972     auto &MI = cast<AtomicMemSetInst>(I);
5973     SDValue Dst = getValue(MI.getRawDest());
5974     SDValue Val = getValue(MI.getValue());
5975     SDValue Length = getValue(MI.getLength());
5976 
5977     unsigned DstAlign = MI.getDestAlignment();
5978     Type *LengthTy = MI.getLength()->getType();
5979     unsigned ElemSz = MI.getElementSizeInBytes();
5980     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5981     SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5982                                      LengthTy, ElemSz, isTC,
5983                                      MachinePointerInfo(MI.getRawDest()));
5984     updateDAGForMaybeTailCall(MC);
5985     return;
5986   }
5987   case Intrinsic::call_preallocated_setup: {
5988     const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
5989     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
5990     SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
5991                               getRoot(), SrcValue);
5992     setValue(&I, Res);
5993     DAG.setRoot(Res);
5994     return;
5995   }
5996   case Intrinsic::call_preallocated_arg: {
5997     const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
5998     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
5999     SDValue Ops[3];
6000     Ops[0] = getRoot();
6001     Ops[1] = SrcValue;
6002     Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6003                                    MVT::i32); // arg index
6004     SDValue Res = DAG.getNode(
6005         ISD::PREALLOCATED_ARG, sdl,
6006         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6007     setValue(&I, Res);
6008     DAG.setRoot(Res.getValue(1));
6009     return;
6010   }
6011   case Intrinsic::dbg_addr:
6012   case Intrinsic::dbg_declare: {
6013     // Assume dbg.addr and dbg.declare can not currently use DIArgList, i.e.
6014     // they are non-variadic.
6015     const auto &DI = cast<DbgVariableIntrinsic>(I);
6016     assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6017     DILocalVariable *Variable = DI.getVariable();
6018     DIExpression *Expression = DI.getExpression();
6019     dropDanglingDebugInfo(Variable, Expression);
6020     assert(Variable && "Missing variable");
6021     LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DI
6022                       << "\n");
6023     // Check if address has undef value.
6024     const Value *Address = DI.getVariableLocationOp(0);
6025     if (!Address || isa<UndefValue>(Address) ||
6026         (Address->use_empty() && !isa<Argument>(Address))) {
6027       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6028                         << " (bad/undef/unused-arg address)\n");
6029       return;
6030     }
6031 
6032     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
6033 
6034     // Check if this variable can be described by a frame index, typically
6035     // either as a static alloca or a byval parameter.
6036     int FI = std::numeric_limits<int>::max();
6037     if (const auto *AI =
6038             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
6039       if (AI->isStaticAlloca()) {
6040         auto I = FuncInfo.StaticAllocaMap.find(AI);
6041         if (I != FuncInfo.StaticAllocaMap.end())
6042           FI = I->second;
6043       }
6044     } else if (const auto *Arg = dyn_cast<Argument>(
6045                    Address->stripInBoundsConstantOffsets())) {
6046       FI = FuncInfo.getArgumentFrameIndex(Arg);
6047     }
6048 
6049     // llvm.dbg.addr is control dependent and always generates indirect
6050     // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
6051     // the MachineFunction variable table.
6052     if (FI != std::numeric_limits<int>::max()) {
6053       if (Intrinsic == Intrinsic::dbg_addr) {
6054         SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
6055             Variable, Expression, FI, getRoot().getNode(), /*IsIndirect*/ true,
6056             dl, SDNodeOrder);
6057         DAG.AddDbgValue(SDV, isParameter);
6058       } else {
6059         LLVM_DEBUG(dbgs() << "Skipping " << DI
6060                           << " (variable info stashed in MF side table)\n");
6061       }
6062       return;
6063     }
6064 
6065     SDValue &N = NodeMap[Address];
6066     if (!N.getNode() && isa<Argument>(Address))
6067       // Check unused arguments map.
6068       N = UnusedArgNodeMap[Address];
6069     SDDbgValue *SDV;
6070     if (N.getNode()) {
6071       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
6072         Address = BCI->getOperand(0);
6073       // Parameters are handled specially.
6074       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
6075       if (isParameter && FINode) {
6076         // Byval parameter. We have a frame index at this point.
6077         SDV =
6078             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
6079                                       /*IsIndirect*/ true, dl, SDNodeOrder);
6080       } else if (isa<Argument>(Address)) {
6081         // Address is an argument, so try to emit its dbg value using
6082         // virtual register info from the FuncInfo.ValueMap.
6083         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6084                                  FuncArgumentDbgValueKind::Declare, N);
6085         return;
6086       } else {
6087         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
6088                               true, dl, SDNodeOrder);
6089       }
6090       DAG.AddDbgValue(SDV, isParameter);
6091     } else {
6092       // If Address is an argument then try to emit its dbg value using
6093       // virtual register info from the FuncInfo.ValueMap.
6094       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6095                                     FuncArgumentDbgValueKind::Declare, N)) {
6096         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6097                           << " (could not emit func-arg dbg_value)\n");
6098       }
6099     }
6100     return;
6101   }
6102   case Intrinsic::dbg_label: {
6103     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6104     DILabel *Label = DI.getLabel();
6105     assert(Label && "Missing label");
6106 
6107     SDDbgLabel *SDV;
6108     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6109     DAG.AddDbgLabel(SDV);
6110     return;
6111   }
6112   case Intrinsic::dbg_value: {
6113     const DbgValueInst &DI = cast<DbgValueInst>(I);
6114     assert(DI.getVariable() && "Missing variable");
6115 
6116     DILocalVariable *Variable = DI.getVariable();
6117     DIExpression *Expression = DI.getExpression();
6118     dropDanglingDebugInfo(Variable, Expression);
6119     SmallVector<Value *, 4> Values(DI.getValues());
6120     if (Values.empty())
6121       return;
6122 
6123     if (llvm::is_contained(Values, nullptr))
6124       return;
6125 
6126     bool IsVariadic = DI.hasArgList();
6127     if (!handleDebugValue(Values, Variable, Expression, dl, DI.getDebugLoc(),
6128                           SDNodeOrder, IsVariadic))
6129       addDanglingDebugInfo(&DI, dl, SDNodeOrder);
6130     return;
6131   }
6132 
6133   case Intrinsic::eh_typeid_for: {
6134     // Find the type id for the given typeinfo.
6135     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6136     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6137     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6138     setValue(&I, Res);
6139     return;
6140   }
6141 
6142   case Intrinsic::eh_return_i32:
6143   case Intrinsic::eh_return_i64:
6144     DAG.getMachineFunction().setCallsEHReturn(true);
6145     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6146                             MVT::Other,
6147                             getControlRoot(),
6148                             getValue(I.getArgOperand(0)),
6149                             getValue(I.getArgOperand(1))));
6150     return;
6151   case Intrinsic::eh_unwind_init:
6152     DAG.getMachineFunction().setCallsUnwindInit(true);
6153     return;
6154   case Intrinsic::eh_dwarf_cfa:
6155     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
6156                              TLI.getPointerTy(DAG.getDataLayout()),
6157                              getValue(I.getArgOperand(0))));
6158     return;
6159   case Intrinsic::eh_sjlj_callsite: {
6160     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6161     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6162     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6163 
6164     MMI.setCurrentCallSite(CI->getZExtValue());
6165     return;
6166   }
6167   case Intrinsic::eh_sjlj_functioncontext: {
6168     // Get and store the index of the function context.
6169     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6170     AllocaInst *FnCtx =
6171       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6172     int FI = FuncInfo.StaticAllocaMap[FnCtx];
6173     MFI.setFunctionContextIndex(FI);
6174     return;
6175   }
6176   case Intrinsic::eh_sjlj_setjmp: {
6177     SDValue Ops[2];
6178     Ops[0] = getRoot();
6179     Ops[1] = getValue(I.getArgOperand(0));
6180     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6181                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
6182     setValue(&I, Op.getValue(0));
6183     DAG.setRoot(Op.getValue(1));
6184     return;
6185   }
6186   case Intrinsic::eh_sjlj_longjmp:
6187     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6188                             getRoot(), getValue(I.getArgOperand(0))));
6189     return;
6190   case Intrinsic::eh_sjlj_setup_dispatch:
6191     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6192                             getRoot()));
6193     return;
6194   case Intrinsic::masked_gather:
6195     visitMaskedGather(I);
6196     return;
6197   case Intrinsic::masked_load:
6198     visitMaskedLoad(I);
6199     return;
6200   case Intrinsic::masked_scatter:
6201     visitMaskedScatter(I);
6202     return;
6203   case Intrinsic::masked_store:
6204     visitMaskedStore(I);
6205     return;
6206   case Intrinsic::masked_expandload:
6207     visitMaskedLoad(I, true /* IsExpanding */);
6208     return;
6209   case Intrinsic::masked_compressstore:
6210     visitMaskedStore(I, true /* IsCompressing */);
6211     return;
6212   case Intrinsic::powi:
6213     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6214                             getValue(I.getArgOperand(1)), DAG));
6215     return;
6216   case Intrinsic::log:
6217     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6218     return;
6219   case Intrinsic::log2:
6220     setValue(&I,
6221              expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6222     return;
6223   case Intrinsic::log10:
6224     setValue(&I,
6225              expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6226     return;
6227   case Intrinsic::exp:
6228     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6229     return;
6230   case Intrinsic::exp2:
6231     setValue(&I,
6232              expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6233     return;
6234   case Intrinsic::pow:
6235     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6236                            getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6237     return;
6238   case Intrinsic::sqrt:
6239   case Intrinsic::fabs:
6240   case Intrinsic::sin:
6241   case Intrinsic::cos:
6242   case Intrinsic::floor:
6243   case Intrinsic::ceil:
6244   case Intrinsic::trunc:
6245   case Intrinsic::rint:
6246   case Intrinsic::nearbyint:
6247   case Intrinsic::round:
6248   case Intrinsic::roundeven:
6249   case Intrinsic::canonicalize: {
6250     unsigned Opcode;
6251     switch (Intrinsic) {
6252     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6253     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
6254     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
6255     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
6256     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
6257     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
6258     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
6259     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
6260     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
6261     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6262     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
6263     case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6264     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6265     }
6266 
6267     setValue(&I, DAG.getNode(Opcode, sdl,
6268                              getValue(I.getArgOperand(0)).getValueType(),
6269                              getValue(I.getArgOperand(0)), Flags));
6270     return;
6271   }
6272   case Intrinsic::lround:
6273   case Intrinsic::llround:
6274   case Intrinsic::lrint:
6275   case Intrinsic::llrint: {
6276     unsigned Opcode;
6277     switch (Intrinsic) {
6278     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6279     case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
6280     case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6281     case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
6282     case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
6283     }
6284 
6285     EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6286     setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6287                              getValue(I.getArgOperand(0))));
6288     return;
6289   }
6290   case Intrinsic::minnum:
6291     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
6292                              getValue(I.getArgOperand(0)).getValueType(),
6293                              getValue(I.getArgOperand(0)),
6294                              getValue(I.getArgOperand(1)), Flags));
6295     return;
6296   case Intrinsic::maxnum:
6297     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
6298                              getValue(I.getArgOperand(0)).getValueType(),
6299                              getValue(I.getArgOperand(0)),
6300                              getValue(I.getArgOperand(1)), Flags));
6301     return;
6302   case Intrinsic::minimum:
6303     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
6304                              getValue(I.getArgOperand(0)).getValueType(),
6305                              getValue(I.getArgOperand(0)),
6306                              getValue(I.getArgOperand(1)), Flags));
6307     return;
6308   case Intrinsic::maximum:
6309     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
6310                              getValue(I.getArgOperand(0)).getValueType(),
6311                              getValue(I.getArgOperand(0)),
6312                              getValue(I.getArgOperand(1)), Flags));
6313     return;
6314   case Intrinsic::copysign:
6315     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
6316                              getValue(I.getArgOperand(0)).getValueType(),
6317                              getValue(I.getArgOperand(0)),
6318                              getValue(I.getArgOperand(1)), Flags));
6319     return;
6320   case Intrinsic::arithmetic_fence: {
6321     setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
6322                              getValue(I.getArgOperand(0)).getValueType(),
6323                              getValue(I.getArgOperand(0)), Flags));
6324     return;
6325   }
6326   case Intrinsic::fma:
6327     setValue(&I, DAG.getNode(
6328                      ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6329                      getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6330                      getValue(I.getArgOperand(2)), Flags));
6331     return;
6332 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
6333   case Intrinsic::INTRINSIC:
6334 #include "llvm/IR/ConstrainedOps.def"
6335     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6336     return;
6337 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6338 #include "llvm/IR/VPIntrinsics.def"
6339     visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6340     return;
6341   case Intrinsic::fptrunc_round: {
6342     // Get the last argument, the metadata and convert it to an integer in the
6343     // call
6344     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6345     Optional<RoundingMode> RoundMode =
6346         convertStrToRoundingMode(cast<MDString>(MD)->getString());
6347 
6348     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6349 
6350     // Propagate fast-math-flags from IR to node(s).
6351     SDNodeFlags Flags;
6352     Flags.copyFMF(*cast<FPMathOperator>(&I));
6353     SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6354 
6355     SDValue Result;
6356     Result = DAG.getNode(
6357         ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6358         DAG.getTargetConstant((int)RoundMode.getValue(), sdl,
6359                               TLI.getPointerTy(DAG.getDataLayout())));
6360     setValue(&I, Result);
6361 
6362     return;
6363   }
6364   case Intrinsic::fmuladd: {
6365     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6366     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6367         TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
6368       setValue(&I, DAG.getNode(ISD::FMA, sdl,
6369                                getValue(I.getArgOperand(0)).getValueType(),
6370                                getValue(I.getArgOperand(0)),
6371                                getValue(I.getArgOperand(1)),
6372                                getValue(I.getArgOperand(2)), Flags));
6373     } else {
6374       // TODO: Intrinsic calls should have fast-math-flags.
6375       SDValue Mul = DAG.getNode(
6376           ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6377           getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6378       SDValue Add = DAG.getNode(ISD::FADD, sdl,
6379                                 getValue(I.getArgOperand(0)).getValueType(),
6380                                 Mul, getValue(I.getArgOperand(2)), Flags);
6381       setValue(&I, Add);
6382     }
6383     return;
6384   }
6385   case Intrinsic::convert_to_fp16:
6386     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6387                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6388                                          getValue(I.getArgOperand(0)),
6389                                          DAG.getTargetConstant(0, sdl,
6390                                                                MVT::i32))));
6391     return;
6392   case Intrinsic::convert_from_fp16:
6393     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6394                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
6395                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6396                                          getValue(I.getArgOperand(0)))));
6397     return;
6398   case Intrinsic::fptosi_sat: {
6399     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6400     setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
6401                              getValue(I.getArgOperand(0)),
6402                              DAG.getValueType(VT.getScalarType())));
6403     return;
6404   }
6405   case Intrinsic::fptoui_sat: {
6406     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6407     setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
6408                              getValue(I.getArgOperand(0)),
6409                              DAG.getValueType(VT.getScalarType())));
6410     return;
6411   }
6412   case Intrinsic::set_rounding:
6413     Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6414                       {getRoot(), getValue(I.getArgOperand(0))});
6415     setValue(&I, Res);
6416     DAG.setRoot(Res.getValue(0));
6417     return;
6418   case Intrinsic::pcmarker: {
6419     SDValue Tmp = getValue(I.getArgOperand(0));
6420     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6421     return;
6422   }
6423   case Intrinsic::readcyclecounter: {
6424     SDValue Op = getRoot();
6425     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
6426                       DAG.getVTList(MVT::i64, MVT::Other), Op);
6427     setValue(&I, Res);
6428     DAG.setRoot(Res.getValue(1));
6429     return;
6430   }
6431   case Intrinsic::bitreverse:
6432     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
6433                              getValue(I.getArgOperand(0)).getValueType(),
6434                              getValue(I.getArgOperand(0))));
6435     return;
6436   case Intrinsic::bswap:
6437     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
6438                              getValue(I.getArgOperand(0)).getValueType(),
6439                              getValue(I.getArgOperand(0))));
6440     return;
6441   case Intrinsic::cttz: {
6442     SDValue Arg = getValue(I.getArgOperand(0));
6443     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6444     EVT Ty = Arg.getValueType();
6445     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
6446                              sdl, Ty, Arg));
6447     return;
6448   }
6449   case Intrinsic::ctlz: {
6450     SDValue Arg = getValue(I.getArgOperand(0));
6451     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6452     EVT Ty = Arg.getValueType();
6453     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
6454                              sdl, Ty, Arg));
6455     return;
6456   }
6457   case Intrinsic::ctpop: {
6458     SDValue Arg = getValue(I.getArgOperand(0));
6459     EVT Ty = Arg.getValueType();
6460     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
6461     return;
6462   }
6463   case Intrinsic::fshl:
6464   case Intrinsic::fshr: {
6465     bool IsFSHL = Intrinsic == Intrinsic::fshl;
6466     SDValue X = getValue(I.getArgOperand(0));
6467     SDValue Y = getValue(I.getArgOperand(1));
6468     SDValue Z = getValue(I.getArgOperand(2));
6469     EVT VT = X.getValueType();
6470 
6471     if (X == Y) {
6472       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
6473       setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
6474     } else {
6475       auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
6476       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
6477     }
6478     return;
6479   }
6480   case Intrinsic::sadd_sat: {
6481     SDValue Op1 = getValue(I.getArgOperand(0));
6482     SDValue Op2 = getValue(I.getArgOperand(1));
6483     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6484     return;
6485   }
6486   case Intrinsic::uadd_sat: {
6487     SDValue Op1 = getValue(I.getArgOperand(0));
6488     SDValue Op2 = getValue(I.getArgOperand(1));
6489     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6490     return;
6491   }
6492   case Intrinsic::ssub_sat: {
6493     SDValue Op1 = getValue(I.getArgOperand(0));
6494     SDValue Op2 = getValue(I.getArgOperand(1));
6495     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6496     return;
6497   }
6498   case Intrinsic::usub_sat: {
6499     SDValue Op1 = getValue(I.getArgOperand(0));
6500     SDValue Op2 = getValue(I.getArgOperand(1));
6501     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6502     return;
6503   }
6504   case Intrinsic::sshl_sat: {
6505     SDValue Op1 = getValue(I.getArgOperand(0));
6506     SDValue Op2 = getValue(I.getArgOperand(1));
6507     setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6508     return;
6509   }
6510   case Intrinsic::ushl_sat: {
6511     SDValue Op1 = getValue(I.getArgOperand(0));
6512     SDValue Op2 = getValue(I.getArgOperand(1));
6513     setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6514     return;
6515   }
6516   case Intrinsic::smul_fix:
6517   case Intrinsic::umul_fix:
6518   case Intrinsic::smul_fix_sat:
6519   case Intrinsic::umul_fix_sat: {
6520     SDValue Op1 = getValue(I.getArgOperand(0));
6521     SDValue Op2 = getValue(I.getArgOperand(1));
6522     SDValue Op3 = getValue(I.getArgOperand(2));
6523     setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6524                              Op1.getValueType(), Op1, Op2, Op3));
6525     return;
6526   }
6527   case Intrinsic::sdiv_fix:
6528   case Intrinsic::udiv_fix:
6529   case Intrinsic::sdiv_fix_sat:
6530   case Intrinsic::udiv_fix_sat: {
6531     SDValue Op1 = getValue(I.getArgOperand(0));
6532     SDValue Op2 = getValue(I.getArgOperand(1));
6533     SDValue Op3 = getValue(I.getArgOperand(2));
6534     setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6535                               Op1, Op2, Op3, DAG, TLI));
6536     return;
6537   }
6538   case Intrinsic::smax: {
6539     SDValue Op1 = getValue(I.getArgOperand(0));
6540     SDValue Op2 = getValue(I.getArgOperand(1));
6541     setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
6542     return;
6543   }
6544   case Intrinsic::smin: {
6545     SDValue Op1 = getValue(I.getArgOperand(0));
6546     SDValue Op2 = getValue(I.getArgOperand(1));
6547     setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
6548     return;
6549   }
6550   case Intrinsic::umax: {
6551     SDValue Op1 = getValue(I.getArgOperand(0));
6552     SDValue Op2 = getValue(I.getArgOperand(1));
6553     setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
6554     return;
6555   }
6556   case Intrinsic::umin: {
6557     SDValue Op1 = getValue(I.getArgOperand(0));
6558     SDValue Op2 = getValue(I.getArgOperand(1));
6559     setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
6560     return;
6561   }
6562   case Intrinsic::abs: {
6563     // TODO: Preserve "int min is poison" arg in SDAG?
6564     SDValue Op1 = getValue(I.getArgOperand(0));
6565     setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
6566     return;
6567   }
6568   case Intrinsic::stacksave: {
6569     SDValue Op = getRoot();
6570     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6571     Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
6572     setValue(&I, Res);
6573     DAG.setRoot(Res.getValue(1));
6574     return;
6575   }
6576   case Intrinsic::stackrestore:
6577     Res = getValue(I.getArgOperand(0));
6578     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
6579     return;
6580   case Intrinsic::get_dynamic_area_offset: {
6581     SDValue Op = getRoot();
6582     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6583     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6584     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
6585     // target.
6586     if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
6587       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
6588                          " intrinsic!");
6589     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
6590                       Op);
6591     DAG.setRoot(Op);
6592     setValue(&I, Res);
6593     return;
6594   }
6595   case Intrinsic::stackguard: {
6596     MachineFunction &MF = DAG.getMachineFunction();
6597     const Module &M = *MF.getFunction().getParent();
6598     SDValue Chain = getRoot();
6599     if (TLI.useLoadStackGuardNode()) {
6600       Res = getLoadStackGuard(DAG, sdl, Chain);
6601     } else {
6602       EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6603       const Value *Global = TLI.getSDagStackGuard(M);
6604       Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
6605       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
6606                         MachinePointerInfo(Global, 0), Align,
6607                         MachineMemOperand::MOVolatile);
6608     }
6609     if (TLI.useStackGuardXorFP())
6610       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
6611     DAG.setRoot(Chain);
6612     setValue(&I, Res);
6613     return;
6614   }
6615   case Intrinsic::stackprotector: {
6616     // Emit code into the DAG to store the stack guard onto the stack.
6617     MachineFunction &MF = DAG.getMachineFunction();
6618     MachineFrameInfo &MFI = MF.getFrameInfo();
6619     SDValue Src, Chain = getRoot();
6620 
6621     if (TLI.useLoadStackGuardNode())
6622       Src = getLoadStackGuard(DAG, sdl, Chain);
6623     else
6624       Src = getValue(I.getArgOperand(0));   // The guard's value.
6625 
6626     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
6627 
6628     int FI = FuncInfo.StaticAllocaMap[Slot];
6629     MFI.setStackProtectorIndex(FI);
6630     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6631 
6632     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
6633 
6634     // Store the stack protector onto the stack.
6635     Res = DAG.getStore(
6636         Chain, sdl, Src, FIN,
6637         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
6638         MaybeAlign(), MachineMemOperand::MOVolatile);
6639     setValue(&I, Res);
6640     DAG.setRoot(Res);
6641     return;
6642   }
6643   case Intrinsic::objectsize:
6644     llvm_unreachable("llvm.objectsize.* should have been lowered already");
6645 
6646   case Intrinsic::is_constant:
6647     llvm_unreachable("llvm.is.constant.* should have been lowered already");
6648 
6649   case Intrinsic::annotation:
6650   case Intrinsic::ptr_annotation:
6651   case Intrinsic::launder_invariant_group:
6652   case Intrinsic::strip_invariant_group:
6653     // Drop the intrinsic, but forward the value
6654     setValue(&I, getValue(I.getOperand(0)));
6655     return;
6656 
6657   case Intrinsic::assume:
6658   case Intrinsic::experimental_noalias_scope_decl:
6659   case Intrinsic::var_annotation:
6660   case Intrinsic::sideeffect:
6661     // Discard annotate attributes, noalias scope declarations, assumptions, and
6662     // artificial side-effects.
6663     return;
6664 
6665   case Intrinsic::codeview_annotation: {
6666     // Emit a label associated with this metadata.
6667     MachineFunction &MF = DAG.getMachineFunction();
6668     MCSymbol *Label =
6669         MF.getMMI().getContext().createTempSymbol("annotation", true);
6670     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
6671     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
6672     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
6673     DAG.setRoot(Res);
6674     return;
6675   }
6676 
6677   case Intrinsic::init_trampoline: {
6678     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
6679 
6680     SDValue Ops[6];
6681     Ops[0] = getRoot();
6682     Ops[1] = getValue(I.getArgOperand(0));
6683     Ops[2] = getValue(I.getArgOperand(1));
6684     Ops[3] = getValue(I.getArgOperand(2));
6685     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
6686     Ops[5] = DAG.getSrcValue(F);
6687 
6688     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
6689 
6690     DAG.setRoot(Res);
6691     return;
6692   }
6693   case Intrinsic::adjust_trampoline:
6694     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
6695                              TLI.getPointerTy(DAG.getDataLayout()),
6696                              getValue(I.getArgOperand(0))));
6697     return;
6698   case Intrinsic::gcroot: {
6699     assert(DAG.getMachineFunction().getFunction().hasGC() &&
6700            "only valid in functions with gc specified, enforced by Verifier");
6701     assert(GFI && "implied by previous");
6702     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
6703     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
6704 
6705     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
6706     GFI->addStackRoot(FI->getIndex(), TypeMap);
6707     return;
6708   }
6709   case Intrinsic::gcread:
6710   case Intrinsic::gcwrite:
6711     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
6712   case Intrinsic::flt_rounds:
6713     Res = DAG.getNode(ISD::FLT_ROUNDS_, sdl, {MVT::i32, MVT::Other}, getRoot());
6714     setValue(&I, Res);
6715     DAG.setRoot(Res.getValue(1));
6716     return;
6717 
6718   case Intrinsic::expect:
6719     // Just replace __builtin_expect(exp, c) with EXP.
6720     setValue(&I, getValue(I.getArgOperand(0)));
6721     return;
6722 
6723   case Intrinsic::ubsantrap:
6724   case Intrinsic::debugtrap:
6725   case Intrinsic::trap: {
6726     StringRef TrapFuncName =
6727         I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
6728     if (TrapFuncName.empty()) {
6729       switch (Intrinsic) {
6730       case Intrinsic::trap:
6731         DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
6732         break;
6733       case Intrinsic::debugtrap:
6734         DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
6735         break;
6736       case Intrinsic::ubsantrap:
6737         DAG.setRoot(DAG.getNode(
6738             ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
6739             DAG.getTargetConstant(
6740                 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
6741                 MVT::i32)));
6742         break;
6743       default: llvm_unreachable("unknown trap intrinsic");
6744       }
6745       return;
6746     }
6747     TargetLowering::ArgListTy Args;
6748     if (Intrinsic == Intrinsic::ubsantrap) {
6749       Args.push_back(TargetLoweringBase::ArgListEntry());
6750       Args[0].Val = I.getArgOperand(0);
6751       Args[0].Node = getValue(Args[0].Val);
6752       Args[0].Ty = Args[0].Val->getType();
6753     }
6754 
6755     TargetLowering::CallLoweringInfo CLI(DAG);
6756     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6757         CallingConv::C, I.getType(),
6758         DAG.getExternalSymbol(TrapFuncName.data(),
6759                               TLI.getPointerTy(DAG.getDataLayout())),
6760         std::move(Args));
6761 
6762     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6763     DAG.setRoot(Result.second);
6764     return;
6765   }
6766 
6767   case Intrinsic::uadd_with_overflow:
6768   case Intrinsic::sadd_with_overflow:
6769   case Intrinsic::usub_with_overflow:
6770   case Intrinsic::ssub_with_overflow:
6771   case Intrinsic::umul_with_overflow:
6772   case Intrinsic::smul_with_overflow: {
6773     ISD::NodeType Op;
6774     switch (Intrinsic) {
6775     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6776     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6777     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6778     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6779     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6780     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6781     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6782     }
6783     SDValue Op1 = getValue(I.getArgOperand(0));
6784     SDValue Op2 = getValue(I.getArgOperand(1));
6785 
6786     EVT ResultVT = Op1.getValueType();
6787     EVT OverflowVT = MVT::i1;
6788     if (ResultVT.isVector())
6789       OverflowVT = EVT::getVectorVT(
6790           *Context, OverflowVT, ResultVT.getVectorElementCount());
6791 
6792     SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
6793     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6794     return;
6795   }
6796   case Intrinsic::prefetch: {
6797     SDValue Ops[5];
6798     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6799     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6800     Ops[0] = DAG.getRoot();
6801     Ops[1] = getValue(I.getArgOperand(0));
6802     Ops[2] = getValue(I.getArgOperand(1));
6803     Ops[3] = getValue(I.getArgOperand(2));
6804     Ops[4] = getValue(I.getArgOperand(3));
6805     SDValue Result = DAG.getMemIntrinsicNode(
6806         ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
6807         EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
6808         /* align */ None, Flags);
6809 
6810     // Chain the prefetch in parallell with any pending loads, to stay out of
6811     // the way of later optimizations.
6812     PendingLoads.push_back(Result);
6813     Result = getRoot();
6814     DAG.setRoot(Result);
6815     return;
6816   }
6817   case Intrinsic::lifetime_start:
6818   case Intrinsic::lifetime_end: {
6819     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6820     // Stack coloring is not enabled in O0, discard region information.
6821     if (TM.getOptLevel() == CodeGenOpt::None)
6822       return;
6823 
6824     const int64_t ObjectSize =
6825         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
6826     Value *const ObjectPtr = I.getArgOperand(1);
6827     SmallVector<const Value *, 4> Allocas;
6828     getUnderlyingObjects(ObjectPtr, Allocas);
6829 
6830     for (const Value *Alloca : Allocas) {
6831       const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
6832 
6833       // Could not find an Alloca.
6834       if (!LifetimeObject)
6835         continue;
6836 
6837       // First check that the Alloca is static, otherwise it won't have a
6838       // valid frame index.
6839       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6840       if (SI == FuncInfo.StaticAllocaMap.end())
6841         return;
6842 
6843       const int FrameIndex = SI->second;
6844       int64_t Offset;
6845       if (GetPointerBaseWithConstantOffset(
6846               ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
6847         Offset = -1; // Cannot determine offset from alloca to lifetime object.
6848       Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
6849                                 Offset);
6850       DAG.setRoot(Res);
6851     }
6852     return;
6853   }
6854   case Intrinsic::pseudoprobe: {
6855     auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
6856     auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6857     auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
6858     Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
6859     DAG.setRoot(Res);
6860     return;
6861   }
6862   case Intrinsic::invariant_start:
6863     // Discard region information.
6864     setValue(&I,
6865              DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
6866     return;
6867   case Intrinsic::invariant_end:
6868     // Discard region information.
6869     return;
6870   case Intrinsic::clear_cache:
6871     /// FunctionName may be null.
6872     if (const char *FunctionName = TLI.getClearCacheBuiltinName())
6873       lowerCallToExternalSymbol(I, FunctionName);
6874     return;
6875   case Intrinsic::donothing:
6876   case Intrinsic::seh_try_begin:
6877   case Intrinsic::seh_scope_begin:
6878   case Intrinsic::seh_try_end:
6879   case Intrinsic::seh_scope_end:
6880     // ignore
6881     return;
6882   case Intrinsic::experimental_stackmap:
6883     visitStackmap(I);
6884     return;
6885   case Intrinsic::experimental_patchpoint_void:
6886   case Intrinsic::experimental_patchpoint_i64:
6887     visitPatchpoint(I);
6888     return;
6889   case Intrinsic::experimental_gc_statepoint:
6890     LowerStatepoint(cast<GCStatepointInst>(I));
6891     return;
6892   case Intrinsic::experimental_gc_result:
6893     visitGCResult(cast<GCResultInst>(I));
6894     return;
6895   case Intrinsic::experimental_gc_relocate:
6896     visitGCRelocate(cast<GCRelocateInst>(I));
6897     return;
6898   case Intrinsic::instrprof_cover:
6899     llvm_unreachable("instrprof failed to lower a cover");
6900   case Intrinsic::instrprof_increment:
6901     llvm_unreachable("instrprof failed to lower an increment");
6902   case Intrinsic::instrprof_value_profile:
6903     llvm_unreachable("instrprof failed to lower a value profiling call");
6904   case Intrinsic::localescape: {
6905     MachineFunction &MF = DAG.getMachineFunction();
6906     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6907 
6908     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6909     // is the same on all targets.
6910     for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
6911       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6912       if (isa<ConstantPointerNull>(Arg))
6913         continue; // Skip null pointers. They represent a hole in index space.
6914       AllocaInst *Slot = cast<AllocaInst>(Arg);
6915       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
6916              "can only escape static allocas");
6917       int FI = FuncInfo.StaticAllocaMap[Slot];
6918       MCSymbol *FrameAllocSym =
6919           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6920               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6921       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6922               TII->get(TargetOpcode::LOCAL_ESCAPE))
6923           .addSym(FrameAllocSym)
6924           .addFrameIndex(FI);
6925     }
6926 
6927     return;
6928   }
6929 
6930   case Intrinsic::localrecover: {
6931     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6932     MachineFunction &MF = DAG.getMachineFunction();
6933 
6934     // Get the symbol that defines the frame offset.
6935     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6936     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6937     unsigned IdxVal =
6938         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6939     MCSymbol *FrameAllocSym =
6940         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6941             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6942 
6943     Value *FP = I.getArgOperand(1);
6944     SDValue FPVal = getValue(FP);
6945     EVT PtrVT = FPVal.getValueType();
6946 
6947     // Create a MCSymbol for the label to avoid any target lowering
6948     // that would make this PC relative.
6949     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6950     SDValue OffsetVal =
6951         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6952 
6953     // Add the offset to the FP.
6954     SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
6955     setValue(&I, Add);
6956 
6957     return;
6958   }
6959 
6960   case Intrinsic::eh_exceptionpointer:
6961   case Intrinsic::eh_exceptioncode: {
6962     // Get the exception pointer vreg, copy from it, and resize it to fit.
6963     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
6964     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
6965     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
6966     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
6967     SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
6968     if (Intrinsic == Intrinsic::eh_exceptioncode)
6969       N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
6970     setValue(&I, N);
6971     return;
6972   }
6973   case Intrinsic::xray_customevent: {
6974     // Here we want to make sure that the intrinsic behaves as if it has a
6975     // specific calling convention, and only for x86_64.
6976     // FIXME: Support other platforms later.
6977     const auto &Triple = DAG.getTarget().getTargetTriple();
6978     if (Triple.getArch() != Triple::x86_64)
6979       return;
6980 
6981     SmallVector<SDValue, 8> Ops;
6982 
6983     // We want to say that we always want the arguments in registers.
6984     SDValue LogEntryVal = getValue(I.getArgOperand(0));
6985     SDValue StrSizeVal = getValue(I.getArgOperand(1));
6986     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6987     SDValue Chain = getRoot();
6988     Ops.push_back(LogEntryVal);
6989     Ops.push_back(StrSizeVal);
6990     Ops.push_back(Chain);
6991 
6992     // We need to enforce the calling convention for the callsite, so that
6993     // argument ordering is enforced correctly, and that register allocation can
6994     // see that some registers may be assumed clobbered and have to preserve
6995     // them across calls to the intrinsic.
6996     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
6997                                            sdl, NodeTys, Ops);
6998     SDValue patchableNode = SDValue(MN, 0);
6999     DAG.setRoot(patchableNode);
7000     setValue(&I, patchableNode);
7001     return;
7002   }
7003   case Intrinsic::xray_typedevent: {
7004     // Here we want to make sure that the intrinsic behaves as if it has a
7005     // specific calling convention, and only for x86_64.
7006     // FIXME: Support other platforms later.
7007     const auto &Triple = DAG.getTarget().getTargetTriple();
7008     if (Triple.getArch() != Triple::x86_64)
7009       return;
7010 
7011     SmallVector<SDValue, 8> Ops;
7012 
7013     // We want to say that we always want the arguments in registers.
7014     // It's unclear to me how manipulating the selection DAG here forces callers
7015     // to provide arguments in registers instead of on the stack.
7016     SDValue LogTypeId = getValue(I.getArgOperand(0));
7017     SDValue LogEntryVal = getValue(I.getArgOperand(1));
7018     SDValue StrSizeVal = getValue(I.getArgOperand(2));
7019     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7020     SDValue Chain = getRoot();
7021     Ops.push_back(LogTypeId);
7022     Ops.push_back(LogEntryVal);
7023     Ops.push_back(StrSizeVal);
7024     Ops.push_back(Chain);
7025 
7026     // We need to enforce the calling convention for the callsite, so that
7027     // argument ordering is enforced correctly, and that register allocation can
7028     // see that some registers may be assumed clobbered and have to preserve
7029     // them across calls to the intrinsic.
7030     MachineSDNode *MN = DAG.getMachineNode(
7031         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7032     SDValue patchableNode = SDValue(MN, 0);
7033     DAG.setRoot(patchableNode);
7034     setValue(&I, patchableNode);
7035     return;
7036   }
7037   case Intrinsic::experimental_deoptimize:
7038     LowerDeoptimizeCall(&I);
7039     return;
7040   case Intrinsic::experimental_stepvector:
7041     visitStepVector(I);
7042     return;
7043   case Intrinsic::vector_reduce_fadd:
7044   case Intrinsic::vector_reduce_fmul:
7045   case Intrinsic::vector_reduce_add:
7046   case Intrinsic::vector_reduce_mul:
7047   case Intrinsic::vector_reduce_and:
7048   case Intrinsic::vector_reduce_or:
7049   case Intrinsic::vector_reduce_xor:
7050   case Intrinsic::vector_reduce_smax:
7051   case Intrinsic::vector_reduce_smin:
7052   case Intrinsic::vector_reduce_umax:
7053   case Intrinsic::vector_reduce_umin:
7054   case Intrinsic::vector_reduce_fmax:
7055   case Intrinsic::vector_reduce_fmin:
7056     visitVectorReduce(I, Intrinsic);
7057     return;
7058 
7059   case Intrinsic::icall_branch_funnel: {
7060     SmallVector<SDValue, 16> Ops;
7061     Ops.push_back(getValue(I.getArgOperand(0)));
7062 
7063     int64_t Offset;
7064     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7065         I.getArgOperand(1), Offset, DAG.getDataLayout()));
7066     if (!Base)
7067       report_fatal_error(
7068           "llvm.icall.branch.funnel operand must be a GlobalValue");
7069     Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7070 
7071     struct BranchFunnelTarget {
7072       int64_t Offset;
7073       SDValue Target;
7074     };
7075     SmallVector<BranchFunnelTarget, 8> Targets;
7076 
7077     for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7078       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7079           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7080       if (ElemBase != Base)
7081         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7082                            "to the same GlobalValue");
7083 
7084       SDValue Val = getValue(I.getArgOperand(Op + 1));
7085       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7086       if (!GA)
7087         report_fatal_error(
7088             "llvm.icall.branch.funnel operand must be a GlobalValue");
7089       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
7090                                      GA->getGlobal(), sdl, Val.getValueType(),
7091                                      GA->getOffset())});
7092     }
7093     llvm::sort(Targets,
7094                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7095                  return T1.Offset < T2.Offset;
7096                });
7097 
7098     for (auto &T : Targets) {
7099       Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7100       Ops.push_back(T.Target);
7101     }
7102 
7103     Ops.push_back(DAG.getRoot()); // Chain
7104     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7105                                  MVT::Other, Ops),
7106               0);
7107     DAG.setRoot(N);
7108     setValue(&I, N);
7109     HasTailCall = true;
7110     return;
7111   }
7112 
7113   case Intrinsic::wasm_landingpad_index:
7114     // Information this intrinsic contained has been transferred to
7115     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7116     // delete it now.
7117     return;
7118 
7119   case Intrinsic::aarch64_settag:
7120   case Intrinsic::aarch64_settag_zero: {
7121     const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7122     bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7123     SDValue Val = TSI.EmitTargetCodeForSetTag(
7124         DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7125         getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7126         ZeroMemory);
7127     DAG.setRoot(Val);
7128     setValue(&I, Val);
7129     return;
7130   }
7131   case Intrinsic::ptrmask: {
7132     SDValue Ptr = getValue(I.getOperand(0));
7133     SDValue Const = getValue(I.getOperand(1));
7134 
7135     EVT PtrVT = Ptr.getValueType();
7136     setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr,
7137                              DAG.getZExtOrTrunc(Const, sdl, PtrVT)));
7138     return;
7139   }
7140   case Intrinsic::get_active_lane_mask: {
7141     EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7142     SDValue Index = getValue(I.getOperand(0));
7143     EVT ElementVT = Index.getValueType();
7144 
7145     if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7146       visitTargetIntrinsic(I, Intrinsic);
7147       return;
7148     }
7149 
7150     SDValue TripCount = getValue(I.getOperand(1));
7151     auto VecTy = CCVT.changeVectorElementType(ElementVT);
7152 
7153     SDValue VectorIndex, VectorTripCount;
7154     if (VecTy.isScalableVector()) {
7155       VectorIndex = DAG.getSplatVector(VecTy, sdl, Index);
7156       VectorTripCount = DAG.getSplatVector(VecTy, sdl, TripCount);
7157     } else {
7158       VectorIndex = DAG.getSplatBuildVector(VecTy, sdl, Index);
7159       VectorTripCount = DAG.getSplatBuildVector(VecTy, sdl, TripCount);
7160     }
7161     SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7162     SDValue VectorInduction = DAG.getNode(
7163         ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7164     SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7165                                  VectorTripCount, ISD::CondCode::SETULT);
7166     setValue(&I, SetCC);
7167     return;
7168   }
7169   case Intrinsic::experimental_vector_insert: {
7170     SDValue Vec = getValue(I.getOperand(0));
7171     SDValue SubVec = getValue(I.getOperand(1));
7172     SDValue Index = getValue(I.getOperand(2));
7173 
7174     // The intrinsic's index type is i64, but the SDNode requires an index type
7175     // suitable for the target. Convert the index as required.
7176     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7177     if (Index.getValueType() != VectorIdxTy)
7178       Index = DAG.getVectorIdxConstant(
7179           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7180 
7181     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7182     setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7183                              Index));
7184     return;
7185   }
7186   case Intrinsic::experimental_vector_extract: {
7187     SDValue Vec = getValue(I.getOperand(0));
7188     SDValue Index = getValue(I.getOperand(1));
7189     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7190 
7191     // The intrinsic's index type is i64, but the SDNode requires an index type
7192     // suitable for the target. Convert the index as required.
7193     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7194     if (Index.getValueType() != VectorIdxTy)
7195       Index = DAG.getVectorIdxConstant(
7196           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7197 
7198     setValue(&I,
7199              DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7200     return;
7201   }
7202   case Intrinsic::experimental_vector_reverse:
7203     visitVectorReverse(I);
7204     return;
7205   case Intrinsic::experimental_vector_splice:
7206     visitVectorSplice(I);
7207     return;
7208   }
7209 }
7210 
7211 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7212     const ConstrainedFPIntrinsic &FPI) {
7213   SDLoc sdl = getCurSDLoc();
7214 
7215   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7216   SmallVector<EVT, 4> ValueVTs;
7217   ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
7218   ValueVTs.push_back(MVT::Other); // Out chain
7219 
7220   // We do not need to serialize constrained FP intrinsics against
7221   // each other or against (nonvolatile) loads, so they can be
7222   // chained like loads.
7223   SDValue Chain = DAG.getRoot();
7224   SmallVector<SDValue, 4> Opers;
7225   Opers.push_back(Chain);
7226   if (FPI.isUnaryOp()) {
7227     Opers.push_back(getValue(FPI.getArgOperand(0)));
7228   } else if (FPI.isTernaryOp()) {
7229     Opers.push_back(getValue(FPI.getArgOperand(0)));
7230     Opers.push_back(getValue(FPI.getArgOperand(1)));
7231     Opers.push_back(getValue(FPI.getArgOperand(2)));
7232   } else {
7233     Opers.push_back(getValue(FPI.getArgOperand(0)));
7234     Opers.push_back(getValue(FPI.getArgOperand(1)));
7235   }
7236 
7237   auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7238     assert(Result.getNode()->getNumValues() == 2);
7239 
7240     // Push node to the appropriate list so that future instructions can be
7241     // chained up correctly.
7242     SDValue OutChain = Result.getValue(1);
7243     switch (EB) {
7244     case fp::ExceptionBehavior::ebIgnore:
7245       // The only reason why ebIgnore nodes still need to be chained is that
7246       // they might depend on the current rounding mode, and therefore must
7247       // not be moved across instruction that may change that mode.
7248       LLVM_FALLTHROUGH;
7249     case fp::ExceptionBehavior::ebMayTrap:
7250       // These must not be moved across calls or instructions that may change
7251       // floating-point exception masks.
7252       PendingConstrainedFP.push_back(OutChain);
7253       break;
7254     case fp::ExceptionBehavior::ebStrict:
7255       // These must not be moved across calls or instructions that may change
7256       // floating-point exception masks or read floating-point exception flags.
7257       // In addition, they cannot be optimized out even if unused.
7258       PendingConstrainedFPStrict.push_back(OutChain);
7259       break;
7260     }
7261   };
7262 
7263   SDVTList VTs = DAG.getVTList(ValueVTs);
7264   fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue();
7265 
7266   SDNodeFlags Flags;
7267   if (EB == fp::ExceptionBehavior::ebIgnore)
7268     Flags.setNoFPExcept(true);
7269 
7270   if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7271     Flags.copyFMF(*FPOp);
7272 
7273   unsigned Opcode;
7274   switch (FPI.getIntrinsicID()) {
7275   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
7276 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
7277   case Intrinsic::INTRINSIC:                                                   \
7278     Opcode = ISD::STRICT_##DAGN;                                               \
7279     break;
7280 #include "llvm/IR/ConstrainedOps.def"
7281   case Intrinsic::experimental_constrained_fmuladd: {
7282     Opcode = ISD::STRICT_FMA;
7283     // Break fmuladd into fmul and fadd.
7284     if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
7285         !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(),
7286                                         ValueVTs[0])) {
7287       Opers.pop_back();
7288       SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
7289       pushOutChain(Mul, EB);
7290       Opcode = ISD::STRICT_FADD;
7291       Opers.clear();
7292       Opers.push_back(Mul.getValue(1));
7293       Opers.push_back(Mul.getValue(0));
7294       Opers.push_back(getValue(FPI.getArgOperand(2)));
7295     }
7296     break;
7297   }
7298   }
7299 
7300   // A few strict DAG nodes carry additional operands that are not
7301   // set up by the default code above.
7302   switch (Opcode) {
7303   default: break;
7304   case ISD::STRICT_FP_ROUND:
7305     Opers.push_back(
7306         DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
7307     break;
7308   case ISD::STRICT_FSETCC:
7309   case ISD::STRICT_FSETCCS: {
7310     auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
7311     ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
7312     if (TM.Options.NoNaNsFPMath)
7313       Condition = getFCmpCodeWithoutNaN(Condition);
7314     Opers.push_back(DAG.getCondCode(Condition));
7315     break;
7316   }
7317   }
7318 
7319   SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
7320   pushOutChain(Result, EB);
7321 
7322   SDValue FPResult = Result.getValue(0);
7323   setValue(&FPI, FPResult);
7324 }
7325 
7326 static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
7327   Optional<unsigned> ResOPC;
7328   switch (VPIntrin.getIntrinsicID()) {
7329 #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)                                    \
7330   case Intrinsic::VPID:                                                        \
7331     ResOPC = ISD::VPSD;                                                        \
7332     break;
7333 #include "llvm/IR/VPIntrinsics.def"
7334   }
7335 
7336   if (!ResOPC.hasValue())
7337     llvm_unreachable(
7338         "Inconsistency: no SDNode available for this VPIntrinsic!");
7339 
7340   if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
7341       *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
7342     if (VPIntrin.getFastMathFlags().allowReassoc())
7343       return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
7344                                                 : ISD::VP_REDUCE_FMUL;
7345   }
7346 
7347   return ResOPC.getValue();
7348 }
7349 
7350 void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
7351                                             SmallVector<SDValue, 7> &OpValues,
7352                                             bool IsGather) {
7353   SDLoc DL = getCurSDLoc();
7354   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7355   Value *PtrOperand = VPIntrin.getArgOperand(0);
7356   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7357   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7358   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7359   SDValue LD;
7360   bool AddToChain = true;
7361   if (!IsGather) {
7362     // Do not serialize variable-length loads of constant memory with
7363     // anything.
7364     if (!Alignment)
7365       Alignment = DAG.getEVTAlign(VT);
7366     MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7367     AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7368     SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7369     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7370         MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7371         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7372     LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
7373                        MMO, false /*IsExpanding */);
7374   } else {
7375     if (!Alignment)
7376       Alignment = DAG.getEVTAlign(VT.getScalarType());
7377     unsigned AS =
7378         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7379     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7380         MachinePointerInfo(AS), MachineMemOperand::MOLoad,
7381         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7382     SDValue Base, Index, Scale;
7383     ISD::MemIndexType IndexType;
7384     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7385                                       this, VPIntrin.getParent());
7386     if (!UniformBase) {
7387       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7388       Index = getValue(PtrOperand);
7389       IndexType = ISD::SIGNED_UNSCALED;
7390       Scale =
7391           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7392     }
7393     EVT IdxVT = Index.getValueType();
7394     EVT EltTy = IdxVT.getVectorElementType();
7395     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7396       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7397       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7398     }
7399     LD = DAG.getGatherVP(
7400         DAG.getVTList(VT, MVT::Other), VT, DL,
7401         {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
7402         IndexType);
7403   }
7404   if (AddToChain)
7405     PendingLoads.push_back(LD.getValue(1));
7406   setValue(&VPIntrin, LD);
7407 }
7408 
7409 void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
7410                                               SmallVector<SDValue, 7> &OpValues,
7411                                               bool IsScatter) {
7412   SDLoc DL = getCurSDLoc();
7413   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7414   Value *PtrOperand = VPIntrin.getArgOperand(1);
7415   EVT VT = OpValues[0].getValueType();
7416   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7417   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7418   SDValue ST;
7419   if (!IsScatter) {
7420     if (!Alignment)
7421       Alignment = DAG.getEVTAlign(VT);
7422     SDValue Ptr = OpValues[1];
7423     SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
7424     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7425         MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7426         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7427     ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
7428                         OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
7429                         /* IsTruncating */ false, /*IsCompressing*/ false);
7430   } else {
7431     if (!Alignment)
7432       Alignment = DAG.getEVTAlign(VT.getScalarType());
7433     unsigned AS =
7434         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7435     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7436         MachinePointerInfo(AS), MachineMemOperand::MOStore,
7437         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7438     SDValue Base, Index, Scale;
7439     ISD::MemIndexType IndexType;
7440     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7441                                       this, VPIntrin.getParent());
7442     if (!UniformBase) {
7443       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7444       Index = getValue(PtrOperand);
7445       IndexType = ISD::SIGNED_UNSCALED;
7446       Scale =
7447           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7448     }
7449     EVT IdxVT = Index.getValueType();
7450     EVT EltTy = IdxVT.getVectorElementType();
7451     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7452       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7453       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7454     }
7455     ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
7456                           {getMemoryRoot(), OpValues[0], Base, Index, Scale,
7457                            OpValues[2], OpValues[3]},
7458                           MMO, IndexType);
7459   }
7460   DAG.setRoot(ST);
7461   setValue(&VPIntrin, ST);
7462 }
7463 
7464 void SelectionDAGBuilder::visitVPStridedLoad(
7465     const VPIntrinsic &VPIntrin, EVT VT, SmallVectorImpl<SDValue> &OpValues) {
7466   SDLoc DL = getCurSDLoc();
7467   Value *PtrOperand = VPIntrin.getArgOperand(0);
7468   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7469   if (!Alignment)
7470     Alignment = DAG.getEVTAlign(VT.getScalarType());
7471   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7472   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7473   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7474   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7475   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7476   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7477       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7478       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7479 
7480   SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
7481                                     OpValues[2], OpValues[3], MMO,
7482                                     false /*IsExpanding*/);
7483 
7484   if (AddToChain)
7485     PendingLoads.push_back(LD.getValue(1));
7486   setValue(&VPIntrin, LD);
7487 }
7488 
7489 void SelectionDAGBuilder::visitVPStridedStore(
7490     const VPIntrinsic &VPIntrin, SmallVectorImpl<SDValue> &OpValues) {
7491   SDLoc DL = getCurSDLoc();
7492   Value *PtrOperand = VPIntrin.getArgOperand(1);
7493   EVT VT = OpValues[0].getValueType();
7494   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7495   if (!Alignment)
7496     Alignment = DAG.getEVTAlign(VT.getScalarType());
7497   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7498   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7499       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7500       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7501 
7502   SDValue ST = DAG.getStridedStoreVP(
7503       getMemoryRoot(), DL, OpValues[0], OpValues[1],
7504       DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
7505       OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
7506       /*IsCompressing*/ false);
7507 
7508   DAG.setRoot(ST);
7509   setValue(&VPIntrin, ST);
7510 }
7511 
7512 void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
7513   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7514   SDLoc DL = getCurSDLoc();
7515 
7516   ISD::CondCode Condition;
7517   CmpInst::Predicate CondCode = VPIntrin.getPredicate();
7518   bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
7519   if (IsFP) {
7520     // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
7521     // flags, but calls that don't return floating-point types can't be
7522     // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
7523     Condition = getFCmpCondCode(CondCode);
7524     if (TM.Options.NoNaNsFPMath)
7525       Condition = getFCmpCodeWithoutNaN(Condition);
7526   } else {
7527     Condition = getICmpCondCode(CondCode);
7528   }
7529 
7530   SDValue Op1 = getValue(VPIntrin.getOperand(0));
7531   SDValue Op2 = getValue(VPIntrin.getOperand(1));
7532   // #2 is the condition code
7533   SDValue MaskOp = getValue(VPIntrin.getOperand(3));
7534   SDValue EVL = getValue(VPIntrin.getOperand(4));
7535   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7536   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7537          "Unexpected target EVL type");
7538   EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
7539 
7540   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7541                                                         VPIntrin.getType());
7542   setValue(&VPIntrin,
7543            DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
7544 }
7545 
7546 void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
7547     const VPIntrinsic &VPIntrin) {
7548   SDLoc DL = getCurSDLoc();
7549   unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
7550 
7551   auto IID = VPIntrin.getIntrinsicID();
7552 
7553   if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
7554     return visitVPCmp(*CmpI);
7555 
7556   SmallVector<EVT, 4> ValueVTs;
7557   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7558   ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
7559   SDVTList VTs = DAG.getVTList(ValueVTs);
7560 
7561   auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
7562 
7563   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7564   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7565          "Unexpected target EVL type");
7566 
7567   // Request operands.
7568   SmallVector<SDValue, 7> OpValues;
7569   for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
7570     auto Op = getValue(VPIntrin.getArgOperand(I));
7571     if (I == EVLParamPos)
7572       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
7573     OpValues.push_back(Op);
7574   }
7575 
7576   switch (Opcode) {
7577   default: {
7578     SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues);
7579     setValue(&VPIntrin, Result);
7580     break;
7581   }
7582   case ISD::VP_LOAD:
7583   case ISD::VP_GATHER:
7584     visitVPLoadGather(VPIntrin, ValueVTs[0], OpValues,
7585                       Opcode == ISD::VP_GATHER);
7586     break;
7587   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7588     visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
7589     break;
7590   case ISD::VP_STORE:
7591   case ISD::VP_SCATTER:
7592     visitVPStoreScatter(VPIntrin, OpValues, Opcode == ISD::VP_SCATTER);
7593     break;
7594   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7595     visitVPStridedStore(VPIntrin, OpValues);
7596     break;
7597   }
7598 }
7599 
7600 SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
7601                                           const BasicBlock *EHPadBB,
7602                                           MCSymbol *&BeginLabel) {
7603   MachineFunction &MF = DAG.getMachineFunction();
7604   MachineModuleInfo &MMI = MF.getMMI();
7605 
7606   // Insert a label before the invoke call to mark the try range.  This can be
7607   // used to detect deletion of the invoke via the MachineModuleInfo.
7608   BeginLabel = MMI.getContext().createTempSymbol();
7609 
7610   // For SjLj, keep track of which landing pads go with which invokes
7611   // so as to maintain the ordering of pads in the LSDA.
7612   unsigned CallSiteIndex = MMI.getCurrentCallSite();
7613   if (CallSiteIndex) {
7614     MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
7615     LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
7616 
7617     // Now that the call site is handled, stop tracking it.
7618     MMI.setCurrentCallSite(0);
7619   }
7620 
7621   return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
7622 }
7623 
7624 SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
7625                                         const BasicBlock *EHPadBB,
7626                                         MCSymbol *BeginLabel) {
7627   assert(BeginLabel && "BeginLabel should've been set");
7628 
7629   MachineFunction &MF = DAG.getMachineFunction();
7630   MachineModuleInfo &MMI = MF.getMMI();
7631 
7632   // Insert a label at the end of the invoke call to mark the try range.  This
7633   // can be used to detect deletion of the invoke via the MachineModuleInfo.
7634   MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
7635   Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
7636 
7637   // Inform MachineModuleInfo of range.
7638   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
7639   // There is a platform (e.g. wasm) that uses funclet style IR but does not
7640   // actually use outlined funclets and their LSDA info style.
7641   if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
7642     assert(II && "II should've been set");
7643     WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
7644     EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
7645   } else if (!isScopedEHPersonality(Pers)) {
7646     assert(EHPadBB);
7647     MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
7648   }
7649 
7650   return Chain;
7651 }
7652 
7653 std::pair<SDValue, SDValue>
7654 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
7655                                     const BasicBlock *EHPadBB) {
7656   MCSymbol *BeginLabel = nullptr;
7657 
7658   if (EHPadBB) {
7659     // Both PendingLoads and PendingExports must be flushed here;
7660     // this call might not return.
7661     (void)getRoot();
7662     DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
7663     CLI.setChain(getRoot());
7664   }
7665 
7666   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7667   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7668 
7669   assert((CLI.IsTailCall || Result.second.getNode()) &&
7670          "Non-null chain expected with non-tail call!");
7671   assert((Result.second.getNode() || !Result.first.getNode()) &&
7672          "Null value expected with tail call!");
7673 
7674   if (!Result.second.getNode()) {
7675     // As a special case, a null chain means that a tail call has been emitted
7676     // and the DAG root is already updated.
7677     HasTailCall = true;
7678 
7679     // Since there's no actual continuation from this block, nothing can be
7680     // relying on us setting vregs for them.
7681     PendingExports.clear();
7682   } else {
7683     DAG.setRoot(Result.second);
7684   }
7685 
7686   if (EHPadBB) {
7687     DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
7688                            BeginLabel));
7689   }
7690 
7691   return Result;
7692 }
7693 
7694 void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
7695                                       bool isTailCall,
7696                                       bool isMustTailCall,
7697                                       const BasicBlock *EHPadBB) {
7698   auto &DL = DAG.getDataLayout();
7699   FunctionType *FTy = CB.getFunctionType();
7700   Type *RetTy = CB.getType();
7701 
7702   TargetLowering::ArgListTy Args;
7703   Args.reserve(CB.arg_size());
7704 
7705   const Value *SwiftErrorVal = nullptr;
7706   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7707 
7708   if (isTailCall) {
7709     // Avoid emitting tail calls in functions with the disable-tail-calls
7710     // attribute.
7711     auto *Caller = CB.getParent()->getParent();
7712     if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
7713         "true" && !isMustTailCall)
7714       isTailCall = false;
7715 
7716     // We can't tail call inside a function with a swifterror argument. Lowering
7717     // does not support this yet. It would have to move into the swifterror
7718     // register before the call.
7719     if (TLI.supportSwiftError() &&
7720         Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
7721       isTailCall = false;
7722   }
7723 
7724   for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
7725     TargetLowering::ArgListEntry Entry;
7726     const Value *V = *I;
7727 
7728     // Skip empty types
7729     if (V->getType()->isEmptyTy())
7730       continue;
7731 
7732     SDValue ArgNode = getValue(V);
7733     Entry.Node = ArgNode; Entry.Ty = V->getType();
7734 
7735     Entry.setAttributes(&CB, I - CB.arg_begin());
7736 
7737     // Use swifterror virtual register as input to the call.
7738     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
7739       SwiftErrorVal = V;
7740       // We find the virtual register for the actual swifterror argument.
7741       // Instead of using the Value, we use the virtual register instead.
7742       Entry.Node =
7743           DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
7744                           EVT(TLI.getPointerTy(DL)));
7745     }
7746 
7747     Args.push_back(Entry);
7748 
7749     // If we have an explicit sret argument that is an Instruction, (i.e., it
7750     // might point to function-local memory), we can't meaningfully tail-call.
7751     if (Entry.IsSRet && isa<Instruction>(V))
7752       isTailCall = false;
7753   }
7754 
7755   // If call site has a cfguardtarget operand bundle, create and add an
7756   // additional ArgListEntry.
7757   if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
7758     TargetLowering::ArgListEntry Entry;
7759     Value *V = Bundle->Inputs[0];
7760     SDValue ArgNode = getValue(V);
7761     Entry.Node = ArgNode;
7762     Entry.Ty = V->getType();
7763     Entry.IsCFGuardTarget = true;
7764     Args.push_back(Entry);
7765   }
7766 
7767   // Check if target-independent constraints permit a tail call here.
7768   // Target-dependent constraints are checked within TLI->LowerCallTo.
7769   if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
7770     isTailCall = false;
7771 
7772   // Disable tail calls if there is an swifterror argument. Targets have not
7773   // been updated to support tail calls.
7774   if (TLI.supportSwiftError() && SwiftErrorVal)
7775     isTailCall = false;
7776 
7777   TargetLowering::CallLoweringInfo CLI(DAG);
7778   CLI.setDebugLoc(getCurSDLoc())
7779       .setChain(getRoot())
7780       .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
7781       .setTailCall(isTailCall)
7782       .setConvergent(CB.isConvergent())
7783       .setIsPreallocated(
7784           CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
7785   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7786 
7787   if (Result.first.getNode()) {
7788     Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
7789     setValue(&CB, Result.first);
7790   }
7791 
7792   // The last element of CLI.InVals has the SDValue for swifterror return.
7793   // Here we copy it to a virtual register and update SwiftErrorMap for
7794   // book-keeping.
7795   if (SwiftErrorVal && TLI.supportSwiftError()) {
7796     // Get the last element of InVals.
7797     SDValue Src = CLI.InVals.back();
7798     Register VReg =
7799         SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
7800     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
7801     DAG.setRoot(CopyNode);
7802   }
7803 }
7804 
7805 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
7806                              SelectionDAGBuilder &Builder) {
7807   // Check to see if this load can be trivially constant folded, e.g. if the
7808   // input is from a string literal.
7809   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
7810     // Cast pointer to the type we really want to load.
7811     Type *LoadTy =
7812         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
7813     if (LoadVT.isVector())
7814       LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
7815 
7816     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
7817                                          PointerType::getUnqual(LoadTy));
7818 
7819     if (const Constant *LoadCst =
7820             ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
7821                                          LoadTy, Builder.DAG.getDataLayout()))
7822       return Builder.getValue(LoadCst);
7823   }
7824 
7825   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
7826   // still constant memory, the input chain can be the entry node.
7827   SDValue Root;
7828   bool ConstantMemory = false;
7829 
7830   // Do not serialize (non-volatile) loads of constant memory with anything.
7831   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
7832     Root = Builder.DAG.getEntryNode();
7833     ConstantMemory = true;
7834   } else {
7835     // Do not serialize non-volatile loads against each other.
7836     Root = Builder.DAG.getRoot();
7837   }
7838 
7839   SDValue Ptr = Builder.getValue(PtrVal);
7840   SDValue LoadVal =
7841       Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
7842                           MachinePointerInfo(PtrVal), Align(1));
7843 
7844   if (!ConstantMemory)
7845     Builder.PendingLoads.push_back(LoadVal.getValue(1));
7846   return LoadVal;
7847 }
7848 
7849 /// Record the value for an instruction that produces an integer result,
7850 /// converting the type where necessary.
7851 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
7852                                                   SDValue Value,
7853                                                   bool IsSigned) {
7854   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7855                                                     I.getType(), true);
7856   if (IsSigned)
7857     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
7858   else
7859     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
7860   setValue(&I, Value);
7861 }
7862 
7863 /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
7864 /// true and lower it. Otherwise return false, and it will be lowered like a
7865 /// normal call.
7866 /// The caller already checked that \p I calls the appropriate LibFunc with a
7867 /// correct prototype.
7868 bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
7869   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
7870   const Value *Size = I.getArgOperand(2);
7871   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
7872   if (CSize && CSize->getZExtValue() == 0) {
7873     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7874                                                           I.getType(), true);
7875     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
7876     return true;
7877   }
7878 
7879   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7880   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
7881       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
7882       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
7883   if (Res.first.getNode()) {
7884     processIntegerCallValue(I, Res.first, true);
7885     PendingLoads.push_back(Res.second);
7886     return true;
7887   }
7888 
7889   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
7890   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
7891   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
7892     return false;
7893 
7894   // If the target has a fast compare for the given size, it will return a
7895   // preferred load type for that size. Require that the load VT is legal and
7896   // that the target supports unaligned loads of that type. Otherwise, return
7897   // INVALID.
7898   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
7899     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7900     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
7901     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
7902       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
7903       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
7904       // TODO: Check alignment of src and dest ptrs.
7905       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
7906       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
7907       if (!TLI.isTypeLegal(LVT) ||
7908           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
7909           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
7910         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
7911     }
7912 
7913     return LVT;
7914   };
7915 
7916   // This turns into unaligned loads. We only do this if the target natively
7917   // supports the MVT we'll be loading or if it is small enough (<= 4) that
7918   // we'll only produce a small number of byte loads.
7919   MVT LoadVT;
7920   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
7921   switch (NumBitsToCompare) {
7922   default:
7923     return false;
7924   case 16:
7925     LoadVT = MVT::i16;
7926     break;
7927   case 32:
7928     LoadVT = MVT::i32;
7929     break;
7930   case 64:
7931   case 128:
7932   case 256:
7933     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
7934     break;
7935   }
7936 
7937   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
7938     return false;
7939 
7940   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
7941   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
7942 
7943   // Bitcast to a wide integer type if the loads are vectors.
7944   if (LoadVT.isVector()) {
7945     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
7946     LoadL = DAG.getBitcast(CmpVT, LoadL);
7947     LoadR = DAG.getBitcast(CmpVT, LoadR);
7948   }
7949 
7950   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
7951   processIntegerCallValue(I, Cmp, false);
7952   return true;
7953 }
7954 
7955 /// See if we can lower a memchr call into an optimized form. If so, return
7956 /// true and lower it. Otherwise return false, and it will be lowered like a
7957 /// normal call.
7958 /// The caller already checked that \p I calls the appropriate LibFunc with a
7959 /// correct prototype.
7960 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
7961   const Value *Src = I.getArgOperand(0);
7962   const Value *Char = I.getArgOperand(1);
7963   const Value *Length = I.getArgOperand(2);
7964 
7965   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7966   std::pair<SDValue, SDValue> Res =
7967     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
7968                                 getValue(Src), getValue(Char), getValue(Length),
7969                                 MachinePointerInfo(Src));
7970   if (Res.first.getNode()) {
7971     setValue(&I, Res.first);
7972     PendingLoads.push_back(Res.second);
7973     return true;
7974   }
7975 
7976   return false;
7977 }
7978 
7979 /// See if we can lower a mempcpy call into an optimized form. If so, return
7980 /// true and lower it. Otherwise return false, and it will be lowered like a
7981 /// normal call.
7982 /// The caller already checked that \p I calls the appropriate LibFunc with a
7983 /// correct prototype.
7984 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
7985   SDValue Dst = getValue(I.getArgOperand(0));
7986   SDValue Src = getValue(I.getArgOperand(1));
7987   SDValue Size = getValue(I.getArgOperand(2));
7988 
7989   Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
7990   Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
7991   // DAG::getMemcpy needs Alignment to be defined.
7992   Align Alignment = std::min(DstAlign, SrcAlign);
7993 
7994   bool isVol = false;
7995   SDLoc sdl = getCurSDLoc();
7996 
7997   // In the mempcpy context we need to pass in a false value for isTailCall
7998   // because the return pointer needs to be adjusted by the size of
7999   // the copied memory.
8000   SDValue Root = isVol ? getRoot() : getMemoryRoot();
8001   SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false,
8002                              /*isTailCall=*/false,
8003                              MachinePointerInfo(I.getArgOperand(0)),
8004                              MachinePointerInfo(I.getArgOperand(1)),
8005                              I.getAAMetadata());
8006   assert(MC.getNode() != nullptr &&
8007          "** memcpy should not be lowered as TailCall in mempcpy context **");
8008   DAG.setRoot(MC);
8009 
8010   // Check if Size needs to be truncated or extended.
8011   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8012 
8013   // Adjust return pointer to point just past the last dst byte.
8014   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8015                                     Dst, Size);
8016   setValue(&I, DstPlusSize);
8017   return true;
8018 }
8019 
8020 /// See if we can lower a strcpy call into an optimized form.  If so, return
8021 /// true and lower it, otherwise return false and it will be lowered like a
8022 /// normal call.
8023 /// The caller already checked that \p I calls the appropriate LibFunc with a
8024 /// correct prototype.
8025 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8026   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8027 
8028   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8029   std::pair<SDValue, SDValue> Res =
8030     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
8031                                 getValue(Arg0), getValue(Arg1),
8032                                 MachinePointerInfo(Arg0),
8033                                 MachinePointerInfo(Arg1), isStpcpy);
8034   if (Res.first.getNode()) {
8035     setValue(&I, Res.first);
8036     DAG.setRoot(Res.second);
8037     return true;
8038   }
8039 
8040   return false;
8041 }
8042 
8043 /// See if we can lower a strcmp call into an optimized form.  If so, return
8044 /// true and lower it, otherwise return false and it will be lowered like a
8045 /// normal call.
8046 /// The caller already checked that \p I calls the appropriate LibFunc with a
8047 /// correct prototype.
8048 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8049   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8050 
8051   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8052   std::pair<SDValue, SDValue> Res =
8053     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
8054                                 getValue(Arg0), getValue(Arg1),
8055                                 MachinePointerInfo(Arg0),
8056                                 MachinePointerInfo(Arg1));
8057   if (Res.first.getNode()) {
8058     processIntegerCallValue(I, Res.first, true);
8059     PendingLoads.push_back(Res.second);
8060     return true;
8061   }
8062 
8063   return false;
8064 }
8065 
8066 /// See if we can lower a strlen call into an optimized form.  If so, return
8067 /// true and lower it, otherwise return false and it will be lowered like a
8068 /// normal call.
8069 /// The caller already checked that \p I calls the appropriate LibFunc with a
8070 /// correct prototype.
8071 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8072   const Value *Arg0 = I.getArgOperand(0);
8073 
8074   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8075   std::pair<SDValue, SDValue> Res =
8076     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
8077                                 getValue(Arg0), MachinePointerInfo(Arg0));
8078   if (Res.first.getNode()) {
8079     processIntegerCallValue(I, Res.first, false);
8080     PendingLoads.push_back(Res.second);
8081     return true;
8082   }
8083 
8084   return false;
8085 }
8086 
8087 /// See if we can lower a strnlen call into an optimized form.  If so, return
8088 /// true and lower it, otherwise return false and it will be lowered like a
8089 /// normal call.
8090 /// The caller already checked that \p I calls the appropriate LibFunc with a
8091 /// correct prototype.
8092 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8093   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8094 
8095   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8096   std::pair<SDValue, SDValue> Res =
8097     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
8098                                  getValue(Arg0), getValue(Arg1),
8099                                  MachinePointerInfo(Arg0));
8100   if (Res.first.getNode()) {
8101     processIntegerCallValue(I, Res.first, false);
8102     PendingLoads.push_back(Res.second);
8103     return true;
8104   }
8105 
8106   return false;
8107 }
8108 
8109 /// See if we can lower a unary floating-point operation into an SDNode with
8110 /// the specified Opcode.  If so, return true and lower it, otherwise return
8111 /// false and it will be lowered like a normal call.
8112 /// The caller already checked that \p I calls the appropriate LibFunc with a
8113 /// correct prototype.
8114 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8115                                               unsigned Opcode) {
8116   // We already checked this call's prototype; verify it doesn't modify errno.
8117   if (!I.onlyReadsMemory())
8118     return false;
8119 
8120   SDNodeFlags Flags;
8121   Flags.copyFMF(cast<FPMathOperator>(I));
8122 
8123   SDValue Tmp = getValue(I.getArgOperand(0));
8124   setValue(&I,
8125            DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8126   return true;
8127 }
8128 
8129 /// See if we can lower a binary floating-point operation into an SDNode with
8130 /// the specified Opcode. If so, return true and lower it. Otherwise return
8131 /// false, and it will be lowered like a normal call.
8132 /// The caller already checked that \p I calls the appropriate LibFunc with a
8133 /// correct prototype.
8134 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8135                                                unsigned Opcode) {
8136   // We already checked this call's prototype; verify it doesn't modify errno.
8137   if (!I.onlyReadsMemory())
8138     return false;
8139 
8140   SDNodeFlags Flags;
8141   Flags.copyFMF(cast<FPMathOperator>(I));
8142 
8143   SDValue Tmp0 = getValue(I.getArgOperand(0));
8144   SDValue Tmp1 = getValue(I.getArgOperand(1));
8145   EVT VT = Tmp0.getValueType();
8146   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8147   return true;
8148 }
8149 
8150 void SelectionDAGBuilder::visitCall(const CallInst &I) {
8151   // Handle inline assembly differently.
8152   if (I.isInlineAsm()) {
8153     visitInlineAsm(I);
8154     return;
8155   }
8156 
8157   if (Function *F = I.getCalledFunction()) {
8158     diagnoseDontCall(I);
8159 
8160     if (F->isDeclaration()) {
8161       // Is this an LLVM intrinsic or a target-specific intrinsic?
8162       unsigned IID = F->getIntrinsicID();
8163       if (!IID)
8164         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8165           IID = II->getIntrinsicID(F);
8166 
8167       if (IID) {
8168         visitIntrinsicCall(I, IID);
8169         return;
8170       }
8171     }
8172 
8173     // Check for well-known libc/libm calls.  If the function is internal, it
8174     // can't be a library call.  Don't do the check if marked as nobuiltin for
8175     // some reason or the call site requires strict floating point semantics.
8176     LibFunc Func;
8177     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
8178         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
8179         LibInfo->hasOptimizedCodeGen(Func)) {
8180       switch (Func) {
8181       default: break;
8182       case LibFunc_bcmp:
8183         if (visitMemCmpBCmpCall(I))
8184           return;
8185         break;
8186       case LibFunc_copysign:
8187       case LibFunc_copysignf:
8188       case LibFunc_copysignl:
8189         // We already checked this call's prototype; verify it doesn't modify
8190         // errno.
8191         if (I.onlyReadsMemory()) {
8192           SDValue LHS = getValue(I.getArgOperand(0));
8193           SDValue RHS = getValue(I.getArgOperand(1));
8194           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
8195                                    LHS.getValueType(), LHS, RHS));
8196           return;
8197         }
8198         break;
8199       case LibFunc_fabs:
8200       case LibFunc_fabsf:
8201       case LibFunc_fabsl:
8202         if (visitUnaryFloatCall(I, ISD::FABS))
8203           return;
8204         break;
8205       case LibFunc_fmin:
8206       case LibFunc_fminf:
8207       case LibFunc_fminl:
8208         if (visitBinaryFloatCall(I, ISD::FMINNUM))
8209           return;
8210         break;
8211       case LibFunc_fmax:
8212       case LibFunc_fmaxf:
8213       case LibFunc_fmaxl:
8214         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
8215           return;
8216         break;
8217       case LibFunc_sin:
8218       case LibFunc_sinf:
8219       case LibFunc_sinl:
8220         if (visitUnaryFloatCall(I, ISD::FSIN))
8221           return;
8222         break;
8223       case LibFunc_cos:
8224       case LibFunc_cosf:
8225       case LibFunc_cosl:
8226         if (visitUnaryFloatCall(I, ISD::FCOS))
8227           return;
8228         break;
8229       case LibFunc_sqrt:
8230       case LibFunc_sqrtf:
8231       case LibFunc_sqrtl:
8232       case LibFunc_sqrt_finite:
8233       case LibFunc_sqrtf_finite:
8234       case LibFunc_sqrtl_finite:
8235         if (visitUnaryFloatCall(I, ISD::FSQRT))
8236           return;
8237         break;
8238       case LibFunc_floor:
8239       case LibFunc_floorf:
8240       case LibFunc_floorl:
8241         if (visitUnaryFloatCall(I, ISD::FFLOOR))
8242           return;
8243         break;
8244       case LibFunc_nearbyint:
8245       case LibFunc_nearbyintf:
8246       case LibFunc_nearbyintl:
8247         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
8248           return;
8249         break;
8250       case LibFunc_ceil:
8251       case LibFunc_ceilf:
8252       case LibFunc_ceill:
8253         if (visitUnaryFloatCall(I, ISD::FCEIL))
8254           return;
8255         break;
8256       case LibFunc_rint:
8257       case LibFunc_rintf:
8258       case LibFunc_rintl:
8259         if (visitUnaryFloatCall(I, ISD::FRINT))
8260           return;
8261         break;
8262       case LibFunc_round:
8263       case LibFunc_roundf:
8264       case LibFunc_roundl:
8265         if (visitUnaryFloatCall(I, ISD::FROUND))
8266           return;
8267         break;
8268       case LibFunc_trunc:
8269       case LibFunc_truncf:
8270       case LibFunc_truncl:
8271         if (visitUnaryFloatCall(I, ISD::FTRUNC))
8272           return;
8273         break;
8274       case LibFunc_log2:
8275       case LibFunc_log2f:
8276       case LibFunc_log2l:
8277         if (visitUnaryFloatCall(I, ISD::FLOG2))
8278           return;
8279         break;
8280       case LibFunc_exp2:
8281       case LibFunc_exp2f:
8282       case LibFunc_exp2l:
8283         if (visitUnaryFloatCall(I, ISD::FEXP2))
8284           return;
8285         break;
8286       case LibFunc_memcmp:
8287         if (visitMemCmpBCmpCall(I))
8288           return;
8289         break;
8290       case LibFunc_mempcpy:
8291         if (visitMemPCpyCall(I))
8292           return;
8293         break;
8294       case LibFunc_memchr:
8295         if (visitMemChrCall(I))
8296           return;
8297         break;
8298       case LibFunc_strcpy:
8299         if (visitStrCpyCall(I, false))
8300           return;
8301         break;
8302       case LibFunc_stpcpy:
8303         if (visitStrCpyCall(I, true))
8304           return;
8305         break;
8306       case LibFunc_strcmp:
8307         if (visitStrCmpCall(I))
8308           return;
8309         break;
8310       case LibFunc_strlen:
8311         if (visitStrLenCall(I))
8312           return;
8313         break;
8314       case LibFunc_strnlen:
8315         if (visitStrNLenCall(I))
8316           return;
8317         break;
8318       }
8319     }
8320   }
8321 
8322   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
8323   // have to do anything here to lower funclet bundles.
8324   // CFGuardTarget bundles are lowered in LowerCallTo.
8325   assert(!I.hasOperandBundlesOtherThan(
8326              {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
8327               LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
8328               LLVMContext::OB_clang_arc_attachedcall}) &&
8329          "Cannot lower calls with arbitrary operand bundles!");
8330 
8331   SDValue Callee = getValue(I.getCalledOperand());
8332 
8333   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
8334     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
8335   else
8336     // Check if we can potentially perform a tail call. More detailed checking
8337     // is be done within LowerCallTo, after more information about the call is
8338     // known.
8339     LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
8340 }
8341 
8342 namespace {
8343 
8344 /// AsmOperandInfo - This contains information for each constraint that we are
8345 /// lowering.
8346 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
8347 public:
8348   /// CallOperand - If this is the result output operand or a clobber
8349   /// this is null, otherwise it is the incoming operand to the CallInst.
8350   /// This gets modified as the asm is processed.
8351   SDValue CallOperand;
8352 
8353   /// AssignedRegs - If this is a register or register class operand, this
8354   /// contains the set of register corresponding to the operand.
8355   RegsForValue AssignedRegs;
8356 
8357   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
8358     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
8359   }
8360 
8361   /// Whether or not this operand accesses memory
8362   bool hasMemory(const TargetLowering &TLI) const {
8363     // Indirect operand accesses access memory.
8364     if (isIndirect)
8365       return true;
8366 
8367     for (const auto &Code : Codes)
8368       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
8369         return true;
8370 
8371     return false;
8372   }
8373 
8374   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
8375   /// corresponds to.  If there is no Value* for this operand, it returns
8376   /// MVT::Other.
8377   EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
8378                            const DataLayout &DL,
8379                            llvm::Type *ParamElemType) const {
8380     if (!CallOperandVal) return MVT::Other;
8381 
8382     if (isa<BasicBlock>(CallOperandVal))
8383       return TLI.getProgramPointerTy(DL);
8384 
8385     llvm::Type *OpTy = CallOperandVal->getType();
8386 
8387     // FIXME: code duplicated from TargetLowering::ParseConstraints().
8388     // If this is an indirect operand, the operand is a pointer to the
8389     // accessed type.
8390     if (isIndirect) {
8391       OpTy = ParamElemType;
8392       assert(OpTy && "Indirect operand must have elementtype attribute");
8393     }
8394 
8395     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
8396     if (StructType *STy = dyn_cast<StructType>(OpTy))
8397       if (STy->getNumElements() == 1)
8398         OpTy = STy->getElementType(0);
8399 
8400     // If OpTy is not a single value, it may be a struct/union that we
8401     // can tile with integers.
8402     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
8403       unsigned BitSize = DL.getTypeSizeInBits(OpTy);
8404       switch (BitSize) {
8405       default: break;
8406       case 1:
8407       case 8:
8408       case 16:
8409       case 32:
8410       case 64:
8411       case 128:
8412         OpTy = IntegerType::get(Context, BitSize);
8413         break;
8414       }
8415     }
8416 
8417     return TLI.getAsmOperandValueType(DL, OpTy, true);
8418   }
8419 };
8420 
8421 
8422 } // end anonymous namespace
8423 
8424 /// Make sure that the output operand \p OpInfo and its corresponding input
8425 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
8426 /// out).
8427 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
8428                                SDISelAsmOperandInfo &MatchingOpInfo,
8429                                SelectionDAG &DAG) {
8430   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
8431     return;
8432 
8433   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
8434   const auto &TLI = DAG.getTargetLoweringInfo();
8435 
8436   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
8437       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
8438                                        OpInfo.ConstraintVT);
8439   std::pair<unsigned, const TargetRegisterClass *> InputRC =
8440       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
8441                                        MatchingOpInfo.ConstraintVT);
8442   if ((OpInfo.ConstraintVT.isInteger() !=
8443        MatchingOpInfo.ConstraintVT.isInteger()) ||
8444       (MatchRC.second != InputRC.second)) {
8445     // FIXME: error out in a more elegant fashion
8446     report_fatal_error("Unsupported asm: input constraint"
8447                        " with a matching output constraint of"
8448                        " incompatible type!");
8449   }
8450   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
8451 }
8452 
8453 /// Get a direct memory input to behave well as an indirect operand.
8454 /// This may introduce stores, hence the need for a \p Chain.
8455 /// \return The (possibly updated) chain.
8456 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
8457                                         SDISelAsmOperandInfo &OpInfo,
8458                                         SelectionDAG &DAG) {
8459   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8460 
8461   // If we don't have an indirect input, put it in the constpool if we can,
8462   // otherwise spill it to a stack slot.
8463   // TODO: This isn't quite right. We need to handle these according to
8464   // the addressing mode that the constraint wants. Also, this may take
8465   // an additional register for the computation and we don't want that
8466   // either.
8467 
8468   // If the operand is a float, integer, or vector constant, spill to a
8469   // constant pool entry to get its address.
8470   const Value *OpVal = OpInfo.CallOperandVal;
8471   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
8472       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
8473     OpInfo.CallOperand = DAG.getConstantPool(
8474         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
8475     return Chain;
8476   }
8477 
8478   // Otherwise, create a stack slot and emit a store to it before the asm.
8479   Type *Ty = OpVal->getType();
8480   auto &DL = DAG.getDataLayout();
8481   uint64_t TySize = DL.getTypeAllocSize(Ty);
8482   MachineFunction &MF = DAG.getMachineFunction();
8483   int SSFI = MF.getFrameInfo().CreateStackObject(
8484       TySize, DL.getPrefTypeAlign(Ty), false);
8485   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
8486   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
8487                             MachinePointerInfo::getFixedStack(MF, SSFI),
8488                             TLI.getMemValueType(DL, Ty));
8489   OpInfo.CallOperand = StackSlot;
8490 
8491   return Chain;
8492 }
8493 
8494 /// GetRegistersForValue - Assign registers (virtual or physical) for the
8495 /// specified operand.  We prefer to assign virtual registers, to allow the
8496 /// register allocator to handle the assignment process.  However, if the asm
8497 /// uses features that we can't model on machineinstrs, we have SDISel do the
8498 /// allocation.  This produces generally horrible, but correct, code.
8499 ///
8500 ///   OpInfo describes the operand
8501 ///   RefOpInfo describes the matching operand if any, the operand otherwise
8502 static llvm::Optional<unsigned>
8503 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
8504                      SDISelAsmOperandInfo &OpInfo,
8505                      SDISelAsmOperandInfo &RefOpInfo) {
8506   LLVMContext &Context = *DAG.getContext();
8507   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8508 
8509   MachineFunction &MF = DAG.getMachineFunction();
8510   SmallVector<unsigned, 4> Regs;
8511   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8512 
8513   // No work to do for memory/address operands.
8514   if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8515       OpInfo.ConstraintType == TargetLowering::C_Address)
8516     return None;
8517 
8518   // If this is a constraint for a single physreg, or a constraint for a
8519   // register class, find it.
8520   unsigned AssignedReg;
8521   const TargetRegisterClass *RC;
8522   std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
8523       &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
8524   // RC is unset only on failure. Return immediately.
8525   if (!RC)
8526     return None;
8527 
8528   // Get the actual register value type.  This is important, because the user
8529   // may have asked for (e.g.) the AX register in i32 type.  We need to
8530   // remember that AX is actually i16 to get the right extension.
8531   const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
8532 
8533   if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
8534     // If this is an FP operand in an integer register (or visa versa), or more
8535     // generally if the operand value disagrees with the register class we plan
8536     // to stick it in, fix the operand type.
8537     //
8538     // If this is an input value, the bitcast to the new type is done now.
8539     // Bitcast for output value is done at the end of visitInlineAsm().
8540     if ((OpInfo.Type == InlineAsm::isOutput ||
8541          OpInfo.Type == InlineAsm::isInput) &&
8542         !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
8543       // Try to convert to the first EVT that the reg class contains.  If the
8544       // types are identical size, use a bitcast to convert (e.g. two differing
8545       // vector types).  Note: output bitcast is done at the end of
8546       // visitInlineAsm().
8547       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
8548         // Exclude indirect inputs while they are unsupported because the code
8549         // to perform the load is missing and thus OpInfo.CallOperand still
8550         // refers to the input address rather than the pointed-to value.
8551         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
8552           OpInfo.CallOperand =
8553               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
8554         OpInfo.ConstraintVT = RegVT;
8555         // If the operand is an FP value and we want it in integer registers,
8556         // use the corresponding integer type. This turns an f64 value into
8557         // i64, which can be passed with two i32 values on a 32-bit machine.
8558       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
8559         MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
8560         if (OpInfo.Type == InlineAsm::isInput)
8561           OpInfo.CallOperand =
8562               DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
8563         OpInfo.ConstraintVT = VT;
8564       }
8565     }
8566   }
8567 
8568   // No need to allocate a matching input constraint since the constraint it's
8569   // matching to has already been allocated.
8570   if (OpInfo.isMatchingInputConstraint())
8571     return None;
8572 
8573   EVT ValueVT = OpInfo.ConstraintVT;
8574   if (OpInfo.ConstraintVT == MVT::Other)
8575     ValueVT = RegVT;
8576 
8577   // Initialize NumRegs.
8578   unsigned NumRegs = 1;
8579   if (OpInfo.ConstraintVT != MVT::Other)
8580     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
8581 
8582   // If this is a constraint for a specific physical register, like {r17},
8583   // assign it now.
8584 
8585   // If this associated to a specific register, initialize iterator to correct
8586   // place. If virtual, make sure we have enough registers
8587 
8588   // Initialize iterator if necessary
8589   TargetRegisterClass::iterator I = RC->begin();
8590   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8591 
8592   // Do not check for single registers.
8593   if (AssignedReg) {
8594     I = std::find(I, RC->end(), AssignedReg);
8595     if (I == RC->end()) {
8596       // RC does not contain the selected register, which indicates a
8597       // mismatch between the register and the required type/bitwidth.
8598       return {AssignedReg};
8599     }
8600   }
8601 
8602   for (; NumRegs; --NumRegs, ++I) {
8603     assert(I != RC->end() && "Ran out of registers to allocate!");
8604     Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
8605     Regs.push_back(R);
8606   }
8607 
8608   OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
8609   return None;
8610 }
8611 
8612 static unsigned
8613 findMatchingInlineAsmOperand(unsigned OperandNo,
8614                              const std::vector<SDValue> &AsmNodeOperands) {
8615   // Scan until we find the definition we already emitted of this operand.
8616   unsigned CurOp = InlineAsm::Op_FirstOperand;
8617   for (; OperandNo; --OperandNo) {
8618     // Advance to the next operand.
8619     unsigned OpFlag =
8620         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8621     assert((InlineAsm::isRegDefKind(OpFlag) ||
8622             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
8623             InlineAsm::isMemKind(OpFlag)) &&
8624            "Skipped past definitions?");
8625     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
8626   }
8627   return CurOp;
8628 }
8629 
8630 namespace {
8631 
8632 class ExtraFlags {
8633   unsigned Flags = 0;
8634 
8635 public:
8636   explicit ExtraFlags(const CallBase &Call) {
8637     const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8638     if (IA->hasSideEffects())
8639       Flags |= InlineAsm::Extra_HasSideEffects;
8640     if (IA->isAlignStack())
8641       Flags |= InlineAsm::Extra_IsAlignStack;
8642     if (Call.isConvergent())
8643       Flags |= InlineAsm::Extra_IsConvergent;
8644     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
8645   }
8646 
8647   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
8648     // Ideally, we would only check against memory constraints.  However, the
8649     // meaning of an Other constraint can be target-specific and we can't easily
8650     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
8651     // for Other constraints as well.
8652     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8653         OpInfo.ConstraintType == TargetLowering::C_Other) {
8654       if (OpInfo.Type == InlineAsm::isInput)
8655         Flags |= InlineAsm::Extra_MayLoad;
8656       else if (OpInfo.Type == InlineAsm::isOutput)
8657         Flags |= InlineAsm::Extra_MayStore;
8658       else if (OpInfo.Type == InlineAsm::isClobber)
8659         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
8660     }
8661   }
8662 
8663   unsigned get() const { return Flags; }
8664 };
8665 
8666 } // end anonymous namespace
8667 
8668 /// visitInlineAsm - Handle a call to an InlineAsm object.
8669 void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
8670                                          const BasicBlock *EHPadBB) {
8671   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8672 
8673   /// ConstraintOperands - Information about all of the constraints.
8674   SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
8675 
8676   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8677   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
8678       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
8679 
8680   // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
8681   // AsmDialect, MayLoad, MayStore).
8682   bool HasSideEffect = IA->hasSideEffects();
8683   ExtraFlags ExtraInfo(Call);
8684 
8685   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
8686   unsigned ResNo = 0;   // ResNo - The result number of the next output.
8687   for (auto &T : TargetConstraints) {
8688     ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
8689     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
8690 
8691     // Compute the value type for each operand.
8692     if (OpInfo.hasArg()) {
8693       OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
8694       OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
8695       Type *ParamElemTy = Call.getParamElementType(ArgNo);
8696       EVT VT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI,
8697                                            DAG.getDataLayout(), ParamElemTy);
8698       OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
8699       ArgNo++;
8700     } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
8701       // The return value of the call is this value.  As such, there is no
8702       // corresponding argument.
8703       assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
8704       if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
8705         OpInfo.ConstraintVT = TLI.getSimpleValueType(
8706             DAG.getDataLayout(), STy->getElementType(ResNo));
8707       } else {
8708         assert(ResNo == 0 && "Asm only has one result!");
8709         OpInfo.ConstraintVT = TLI.getAsmOperandValueType(
8710             DAG.getDataLayout(), Call.getType()).getSimpleVT();
8711       }
8712       ++ResNo;
8713     } else {
8714       OpInfo.ConstraintVT = MVT::Other;
8715     }
8716 
8717     if (!HasSideEffect)
8718       HasSideEffect = OpInfo.hasMemory(TLI);
8719 
8720     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
8721     // FIXME: Could we compute this on OpInfo rather than T?
8722 
8723     // Compute the constraint code and ConstraintType to use.
8724     TLI.ComputeConstraintToUse(T, SDValue());
8725 
8726     if (T.ConstraintType == TargetLowering::C_Immediate &&
8727         OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
8728       // We've delayed emitting a diagnostic like the "n" constraint because
8729       // inlining could cause an integer showing up.
8730       return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
8731                                           "' expects an integer constant "
8732                                           "expression");
8733 
8734     ExtraInfo.update(T);
8735   }
8736 
8737   // We won't need to flush pending loads if this asm doesn't touch
8738   // memory and is nonvolatile.
8739   SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
8740 
8741   bool EmitEHLabels = isa<InvokeInst>(Call) && IA->canThrow();
8742   if (EmitEHLabels) {
8743     assert(EHPadBB && "InvokeInst must have an EHPadBB");
8744   }
8745   bool IsCallBr = isa<CallBrInst>(Call);
8746 
8747   if (IsCallBr || EmitEHLabels) {
8748     // If this is a callbr or invoke we need to flush pending exports since
8749     // inlineasm_br and invoke are terminators.
8750     // We need to do this before nodes are glued to the inlineasm_br node.
8751     Chain = getControlRoot();
8752   }
8753 
8754   MCSymbol *BeginLabel = nullptr;
8755   if (EmitEHLabels) {
8756     Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
8757   }
8758 
8759   // Second pass over the constraints: compute which constraint option to use.
8760   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8761     // If this is an output operand with a matching input operand, look up the
8762     // matching input. If their types mismatch, e.g. one is an integer, the
8763     // other is floating point, or their sizes are different, flag it as an
8764     // error.
8765     if (OpInfo.hasMatchingInput()) {
8766       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
8767       patchMatchingInput(OpInfo, Input, DAG);
8768     }
8769 
8770     // Compute the constraint code and ConstraintType to use.
8771     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
8772 
8773     if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
8774          OpInfo.Type == InlineAsm::isClobber) ||
8775         OpInfo.ConstraintType == TargetLowering::C_Address)
8776       continue;
8777 
8778     // If this is a memory input, and if the operand is not indirect, do what we
8779     // need to provide an address for the memory input.
8780     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
8781         !OpInfo.isIndirect) {
8782       assert((OpInfo.isMultipleAlternative ||
8783               (OpInfo.Type == InlineAsm::isInput)) &&
8784              "Can only indirectify direct input operands!");
8785 
8786       // Memory operands really want the address of the value.
8787       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
8788 
8789       // There is no longer a Value* corresponding to this operand.
8790       OpInfo.CallOperandVal = nullptr;
8791 
8792       // It is now an indirect operand.
8793       OpInfo.isIndirect = true;
8794     }
8795 
8796   }
8797 
8798   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
8799   std::vector<SDValue> AsmNodeOperands;
8800   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
8801   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
8802       IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
8803 
8804   // If we have a !srcloc metadata node associated with it, we want to attach
8805   // this to the ultimately generated inline asm machineinstr.  To do this, we
8806   // pass in the third operand as this (potentially null) inline asm MDNode.
8807   const MDNode *SrcLoc = Call.getMetadata("srcloc");
8808   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
8809 
8810   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
8811   // bits as operand 3.
8812   AsmNodeOperands.push_back(DAG.getTargetConstant(
8813       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8814 
8815   // Third pass: Loop over operands to prepare DAG-level operands.. As part of
8816   // this, assign virtual and physical registers for inputs and otput.
8817   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8818     // Assign Registers.
8819     SDISelAsmOperandInfo &RefOpInfo =
8820         OpInfo.isMatchingInputConstraint()
8821             ? ConstraintOperands[OpInfo.getMatchedOperand()]
8822             : OpInfo;
8823     const auto RegError =
8824         getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
8825     if (RegError.hasValue()) {
8826       const MachineFunction &MF = DAG.getMachineFunction();
8827       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8828       const char *RegName = TRI.getName(RegError.getValue());
8829       emitInlineAsmError(Call, "register '" + Twine(RegName) +
8830                                    "' allocated for constraint '" +
8831                                    Twine(OpInfo.ConstraintCode) +
8832                                    "' does not match required type");
8833       return;
8834     }
8835 
8836     auto DetectWriteToReservedRegister = [&]() {
8837       const MachineFunction &MF = DAG.getMachineFunction();
8838       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8839       for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
8840         if (Register::isPhysicalRegister(Reg) &&
8841             TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
8842           const char *RegName = TRI.getName(Reg);
8843           emitInlineAsmError(Call, "write to reserved register '" +
8844                                        Twine(RegName) + "'");
8845           return true;
8846         }
8847       }
8848       return false;
8849     };
8850     assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
8851             (OpInfo.Type == InlineAsm::isInput &&
8852              !OpInfo.isMatchingInputConstraint())) &&
8853            "Only address as input operand is allowed.");
8854 
8855     switch (OpInfo.Type) {
8856     case InlineAsm::isOutput:
8857       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
8858         unsigned ConstraintID =
8859             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8860         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8861                "Failed to convert memory constraint code to constraint id.");
8862 
8863         // Add information to the INLINEASM node to know about this output.
8864         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
8865         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
8866         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
8867                                                         MVT::i32));
8868         AsmNodeOperands.push_back(OpInfo.CallOperand);
8869       } else {
8870         // Otherwise, this outputs to a register (directly for C_Register /
8871         // C_RegisterClass, and a target-defined fashion for
8872         // C_Immediate/C_Other). Find a register that we can use.
8873         if (OpInfo.AssignedRegs.Regs.empty()) {
8874           emitInlineAsmError(
8875               Call, "couldn't allocate output register for constraint '" +
8876                         Twine(OpInfo.ConstraintCode) + "'");
8877           return;
8878         }
8879 
8880         if (DetectWriteToReservedRegister())
8881           return;
8882 
8883         // Add information to the INLINEASM node to know that this register is
8884         // set.
8885         OpInfo.AssignedRegs.AddInlineAsmOperands(
8886             OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
8887                                   : InlineAsm::Kind_RegDef,
8888             false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
8889       }
8890       break;
8891 
8892     case InlineAsm::isInput: {
8893       SDValue InOperandVal = OpInfo.CallOperand;
8894 
8895       if (OpInfo.isMatchingInputConstraint()) {
8896         // If this is required to match an output register we have already set,
8897         // just use its register.
8898         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
8899                                                   AsmNodeOperands);
8900         unsigned OpFlag =
8901           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8902         if (InlineAsm::isRegDefKind(OpFlag) ||
8903             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
8904           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
8905           if (OpInfo.isIndirect) {
8906             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
8907             emitInlineAsmError(Call, "inline asm not supported yet: "
8908                                      "don't know how to handle tied "
8909                                      "indirect register inputs");
8910             return;
8911           }
8912 
8913           SmallVector<unsigned, 4> Regs;
8914           MachineFunction &MF = DAG.getMachineFunction();
8915           MachineRegisterInfo &MRI = MF.getRegInfo();
8916           const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8917           auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
8918           Register TiedReg = R->getReg();
8919           MVT RegVT = R->getSimpleValueType(0);
8920           const TargetRegisterClass *RC =
8921               TiedReg.isVirtual()     ? MRI.getRegClass(TiedReg)
8922               : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
8923                                       : TRI.getMinimalPhysRegClass(TiedReg);
8924           unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
8925           for (unsigned i = 0; i != NumRegs; ++i)
8926             Regs.push_back(MRI.createVirtualRegister(RC));
8927 
8928           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
8929 
8930           SDLoc dl = getCurSDLoc();
8931           // Use the produced MatchedRegs object to
8932           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call);
8933           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
8934                                            true, OpInfo.getMatchedOperand(), dl,
8935                                            DAG, AsmNodeOperands);
8936           break;
8937         }
8938 
8939         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
8940         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
8941                "Unexpected number of operands");
8942         // Add information to the INLINEASM node to know about this input.
8943         // See InlineAsm.h isUseOperandTiedToDef.
8944         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
8945         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
8946                                                     OpInfo.getMatchedOperand());
8947         AsmNodeOperands.push_back(DAG.getTargetConstant(
8948             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8949         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
8950         break;
8951       }
8952 
8953       // Treat indirect 'X' constraint as memory.
8954       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
8955           OpInfo.isIndirect)
8956         OpInfo.ConstraintType = TargetLowering::C_Memory;
8957 
8958       if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
8959           OpInfo.ConstraintType == TargetLowering::C_Other) {
8960         std::vector<SDValue> Ops;
8961         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
8962                                           Ops, DAG);
8963         if (Ops.empty()) {
8964           if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
8965             if (isa<ConstantSDNode>(InOperandVal)) {
8966               emitInlineAsmError(Call, "value out of range for constraint '" +
8967                                            Twine(OpInfo.ConstraintCode) + "'");
8968               return;
8969             }
8970 
8971           emitInlineAsmError(Call,
8972                              "invalid operand for inline asm constraint '" +
8973                                  Twine(OpInfo.ConstraintCode) + "'");
8974           return;
8975         }
8976 
8977         // Add information to the INLINEASM node to know about this input.
8978         unsigned ResOpType =
8979           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
8980         AsmNodeOperands.push_back(DAG.getTargetConstant(
8981             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8982         llvm::append_range(AsmNodeOperands, Ops);
8983         break;
8984       }
8985 
8986       if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8987           OpInfo.ConstraintType == TargetLowering::C_Address) {
8988         assert((OpInfo.isIndirect ||
8989                 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
8990                "Operand must be indirect to be a mem!");
8991         assert(InOperandVal.getValueType() ==
8992                    TLI.getPointerTy(DAG.getDataLayout()) &&
8993                "Memory operands expect pointer values");
8994 
8995         unsigned ConstraintID =
8996             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8997         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8998                "Failed to convert memory constraint code to constraint id.");
8999 
9000         // Add information to the INLINEASM node to know about this input.
9001         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
9002         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
9003         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9004                                                         getCurSDLoc(),
9005                                                         MVT::i32));
9006         AsmNodeOperands.push_back(InOperandVal);
9007         break;
9008       }
9009 
9010       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9011               OpInfo.ConstraintType == TargetLowering::C_Register) &&
9012              "Unknown constraint type!");
9013 
9014       // TODO: Support this.
9015       if (OpInfo.isIndirect) {
9016         emitInlineAsmError(
9017             Call, "Don't know how to handle indirect register inputs yet "
9018                   "for constraint '" +
9019                       Twine(OpInfo.ConstraintCode) + "'");
9020         return;
9021       }
9022 
9023       // Copy the input into the appropriate registers.
9024       if (OpInfo.AssignedRegs.Regs.empty()) {
9025         emitInlineAsmError(Call,
9026                            "couldn't allocate input reg for constraint '" +
9027                                Twine(OpInfo.ConstraintCode) + "'");
9028         return;
9029       }
9030 
9031       if (DetectWriteToReservedRegister())
9032         return;
9033 
9034       SDLoc dl = getCurSDLoc();
9035 
9036       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
9037                                         &Call);
9038 
9039       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
9040                                                dl, DAG, AsmNodeOperands);
9041       break;
9042     }
9043     case InlineAsm::isClobber:
9044       // Add the clobbered value to the operand list, so that the register
9045       // allocator is aware that the physreg got clobbered.
9046       if (!OpInfo.AssignedRegs.Regs.empty())
9047         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
9048                                                  false, 0, getCurSDLoc(), DAG,
9049                                                  AsmNodeOperands);
9050       break;
9051     }
9052   }
9053 
9054   // Finish up input operands.  Set the input chain and add the flag last.
9055   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9056   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
9057 
9058   unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9059   Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9060                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9061   Flag = Chain.getValue(1);
9062 
9063   // Do additional work to generate outputs.
9064 
9065   SmallVector<EVT, 1> ResultVTs;
9066   SmallVector<SDValue, 1> ResultValues;
9067   SmallVector<SDValue, 8> OutChains;
9068 
9069   llvm::Type *CallResultType = Call.getType();
9070   ArrayRef<Type *> ResultTypes;
9071   if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9072     ResultTypes = StructResult->elements();
9073   else if (!CallResultType->isVoidTy())
9074     ResultTypes = makeArrayRef(CallResultType);
9075 
9076   auto CurResultType = ResultTypes.begin();
9077   auto handleRegAssign = [&](SDValue V) {
9078     assert(CurResultType != ResultTypes.end() && "Unexpected value");
9079     assert((*CurResultType)->isSized() && "Unexpected unsized type");
9080     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9081     ++CurResultType;
9082     // If the type of the inline asm call site return value is different but has
9083     // same size as the type of the asm output bitcast it.  One example of this
9084     // is for vectors with different width / number of elements.  This can
9085     // happen for register classes that can contain multiple different value
9086     // types.  The preg or vreg allocated may not have the same VT as was
9087     // expected.
9088     //
9089     // This can also happen for a return value that disagrees with the register
9090     // class it is put in, eg. a double in a general-purpose register on a
9091     // 32-bit machine.
9092     if (ResultVT != V.getValueType() &&
9093         ResultVT.getSizeInBits() == V.getValueSizeInBits())
9094       V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9095     else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9096              V.getValueType().isInteger()) {
9097       // If a result value was tied to an input value, the computed result
9098       // may have a wider width than the expected result.  Extract the
9099       // relevant portion.
9100       V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9101     }
9102     assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9103     ResultVTs.push_back(ResultVT);
9104     ResultValues.push_back(V);
9105   };
9106 
9107   // Deal with output operands.
9108   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9109     if (OpInfo.Type == InlineAsm::isOutput) {
9110       SDValue Val;
9111       // Skip trivial output operands.
9112       if (OpInfo.AssignedRegs.Regs.empty())
9113         continue;
9114 
9115       switch (OpInfo.ConstraintType) {
9116       case TargetLowering::C_Register:
9117       case TargetLowering::C_RegisterClass:
9118         Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9119                                                   Chain, &Flag, &Call);
9120         break;
9121       case TargetLowering::C_Immediate:
9122       case TargetLowering::C_Other:
9123         Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
9124                                               OpInfo, DAG);
9125         break;
9126       case TargetLowering::C_Memory:
9127         break; // Already handled.
9128       case TargetLowering::C_Address:
9129         break; // Silence warning.
9130       case TargetLowering::C_Unknown:
9131         assert(false && "Unexpected unknown constraint");
9132       }
9133 
9134       // Indirect output manifest as stores. Record output chains.
9135       if (OpInfo.isIndirect) {
9136         const Value *Ptr = OpInfo.CallOperandVal;
9137         assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9138         SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9139                                      MachinePointerInfo(Ptr));
9140         OutChains.push_back(Store);
9141       } else {
9142         // generate CopyFromRegs to associated registers.
9143         assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9144         if (Val.getOpcode() == ISD::MERGE_VALUES) {
9145           for (const SDValue &V : Val->op_values())
9146             handleRegAssign(V);
9147         } else
9148           handleRegAssign(Val);
9149       }
9150     }
9151   }
9152 
9153   // Set results.
9154   if (!ResultValues.empty()) {
9155     assert(CurResultType == ResultTypes.end() &&
9156            "Mismatch in number of ResultTypes");
9157     assert(ResultValues.size() == ResultTypes.size() &&
9158            "Mismatch in number of output operands in asm result");
9159 
9160     SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
9161                             DAG.getVTList(ResultVTs), ResultValues);
9162     setValue(&Call, V);
9163   }
9164 
9165   // Collect store chains.
9166   if (!OutChains.empty())
9167     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
9168 
9169   if (EmitEHLabels) {
9170     Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
9171   }
9172 
9173   // Only Update Root if inline assembly has a memory effect.
9174   if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
9175       EmitEHLabels)
9176     DAG.setRoot(Chain);
9177 }
9178 
9179 void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
9180                                              const Twine &Message) {
9181   LLVMContext &Ctx = *DAG.getContext();
9182   Ctx.emitError(&Call, Message);
9183 
9184   // Make sure we leave the DAG in a valid state
9185   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9186   SmallVector<EVT, 1> ValueVTs;
9187   ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
9188 
9189   if (ValueVTs.empty())
9190     return;
9191 
9192   SmallVector<SDValue, 1> Ops;
9193   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
9194     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
9195 
9196   setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
9197 }
9198 
9199 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
9200   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
9201                           MVT::Other, getRoot(),
9202                           getValue(I.getArgOperand(0)),
9203                           DAG.getSrcValue(I.getArgOperand(0))));
9204 }
9205 
9206 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
9207   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9208   const DataLayout &DL = DAG.getDataLayout();
9209   SDValue V = DAG.getVAArg(
9210       TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
9211       getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
9212       DL.getABITypeAlign(I.getType()).value());
9213   DAG.setRoot(V.getValue(1));
9214 
9215   if (I.getType()->isPointerTy())
9216     V = DAG.getPtrExtOrTrunc(
9217         V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
9218   setValue(&I, V);
9219 }
9220 
9221 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
9222   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
9223                           MVT::Other, getRoot(),
9224                           getValue(I.getArgOperand(0)),
9225                           DAG.getSrcValue(I.getArgOperand(0))));
9226 }
9227 
9228 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
9229   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
9230                           MVT::Other, getRoot(),
9231                           getValue(I.getArgOperand(0)),
9232                           getValue(I.getArgOperand(1)),
9233                           DAG.getSrcValue(I.getArgOperand(0)),
9234                           DAG.getSrcValue(I.getArgOperand(1))));
9235 }
9236 
9237 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
9238                                                     const Instruction &I,
9239                                                     SDValue Op) {
9240   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
9241   if (!Range)
9242     return Op;
9243 
9244   ConstantRange CR = getConstantRangeFromMetadata(*Range);
9245   if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
9246     return Op;
9247 
9248   APInt Lo = CR.getUnsignedMin();
9249   if (!Lo.isMinValue())
9250     return Op;
9251 
9252   APInt Hi = CR.getUnsignedMax();
9253   unsigned Bits = std::max(Hi.getActiveBits(),
9254                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
9255 
9256   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
9257 
9258   SDLoc SL = getCurSDLoc();
9259 
9260   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
9261                              DAG.getValueType(SmallVT));
9262   unsigned NumVals = Op.getNode()->getNumValues();
9263   if (NumVals == 1)
9264     return ZExt;
9265 
9266   SmallVector<SDValue, 4> Ops;
9267 
9268   Ops.push_back(ZExt);
9269   for (unsigned I = 1; I != NumVals; ++I)
9270     Ops.push_back(Op.getValue(I));
9271 
9272   return DAG.getMergeValues(Ops, SL);
9273 }
9274 
9275 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
9276 /// the call being lowered.
9277 ///
9278 /// This is a helper for lowering intrinsics that follow a target calling
9279 /// convention or require stack pointer adjustment. Only a subset of the
9280 /// intrinsic's operands need to participate in the calling convention.
9281 void SelectionDAGBuilder::populateCallLoweringInfo(
9282     TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
9283     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
9284     bool IsPatchPoint) {
9285   TargetLowering::ArgListTy Args;
9286   Args.reserve(NumArgs);
9287 
9288   // Populate the argument list.
9289   // Attributes for args start at offset 1, after the return attribute.
9290   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
9291        ArgI != ArgE; ++ArgI) {
9292     const Value *V = Call->getOperand(ArgI);
9293 
9294     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
9295 
9296     TargetLowering::ArgListEntry Entry;
9297     Entry.Node = getValue(V);
9298     Entry.Ty = V->getType();
9299     Entry.setAttributes(Call, ArgI);
9300     Args.push_back(Entry);
9301   }
9302 
9303   CLI.setDebugLoc(getCurSDLoc())
9304       .setChain(getRoot())
9305       .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
9306       .setDiscardResult(Call->use_empty())
9307       .setIsPatchPoint(IsPatchPoint)
9308       .setIsPreallocated(
9309           Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
9310 }
9311 
9312 /// Add a stack map intrinsic call's live variable operands to a stackmap
9313 /// or patchpoint target node's operand list.
9314 ///
9315 /// Constants are converted to TargetConstants purely as an optimization to
9316 /// avoid constant materialization and register allocation.
9317 ///
9318 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
9319 /// generate addess computation nodes, and so FinalizeISel can convert the
9320 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
9321 /// address materialization and register allocation, but may also be required
9322 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
9323 /// alloca in the entry block, then the runtime may assume that the alloca's
9324 /// StackMap location can be read immediately after compilation and that the
9325 /// location is valid at any point during execution (this is similar to the
9326 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
9327 /// only available in a register, then the runtime would need to trap when
9328 /// execution reaches the StackMap in order to read the alloca's location.
9329 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
9330                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
9331                                 SelectionDAGBuilder &Builder) {
9332   for (unsigned i = StartIdx, e = Call.arg_size(); i != e; ++i) {
9333     SDValue OpVal = Builder.getValue(Call.getArgOperand(i));
9334     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
9335       Ops.push_back(
9336         Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
9337       Ops.push_back(
9338         Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
9339     } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
9340       const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
9341       Ops.push_back(Builder.DAG.getTargetFrameIndex(
9342           FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
9343     } else
9344       Ops.push_back(OpVal);
9345   }
9346 }
9347 
9348 /// Lower llvm.experimental.stackmap directly to its target opcode.
9349 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
9350   // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
9351   //                                  [live variables...])
9352 
9353   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
9354 
9355   SDValue Chain, InFlag, Callee, NullPtr;
9356   SmallVector<SDValue, 32> Ops;
9357 
9358   SDLoc DL = getCurSDLoc();
9359   Callee = getValue(CI.getCalledOperand());
9360   NullPtr = DAG.getIntPtrConstant(0, DL, true);
9361 
9362   // The stackmap intrinsic only records the live variables (the arguments
9363   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
9364   // intrinsic, this won't be lowered to a function call. This means we don't
9365   // have to worry about calling conventions and target specific lowering code.
9366   // Instead we perform the call lowering right here.
9367   //
9368   // chain, flag = CALLSEQ_START(chain, 0, 0)
9369   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
9370   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
9371   //
9372   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
9373   InFlag = Chain.getValue(1);
9374 
9375   // Add the <id> and <numBytes> constants.
9376   SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
9377   Ops.push_back(DAG.getTargetConstant(
9378                   cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
9379   SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
9380   Ops.push_back(DAG.getTargetConstant(
9381                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
9382                   MVT::i32));
9383 
9384   // Push live variables for the stack map.
9385   addStackMapLiveVars(CI, 2, DL, Ops, *this);
9386 
9387   // We are not pushing any register mask info here on the operands list,
9388   // because the stackmap doesn't clobber anything.
9389 
9390   // Push the chain and the glue flag.
9391   Ops.push_back(Chain);
9392   Ops.push_back(InFlag);
9393 
9394   // Create the STACKMAP node.
9395   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9396   SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
9397   Chain = SDValue(SM, 0);
9398   InFlag = Chain.getValue(1);
9399 
9400   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
9401 
9402   // Stackmaps don't generate values, so nothing goes into the NodeMap.
9403 
9404   // Set the root to the target-lowered call chain.
9405   DAG.setRoot(Chain);
9406 
9407   // Inform the Frame Information that we have a stackmap in this function.
9408   FuncInfo.MF->getFrameInfo().setHasStackMap();
9409 }
9410 
9411 /// Lower llvm.experimental.patchpoint directly to its target opcode.
9412 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
9413                                           const BasicBlock *EHPadBB) {
9414   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
9415   //                                                 i32 <numBytes>,
9416   //                                                 i8* <target>,
9417   //                                                 i32 <numArgs>,
9418   //                                                 [Args...],
9419   //                                                 [live variables...])
9420 
9421   CallingConv::ID CC = CB.getCallingConv();
9422   bool IsAnyRegCC = CC == CallingConv::AnyReg;
9423   bool HasDef = !CB.getType()->isVoidTy();
9424   SDLoc dl = getCurSDLoc();
9425   SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));
9426 
9427   // Handle immediate and symbolic callees.
9428   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
9429     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
9430                                    /*isTarget=*/true);
9431   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
9432     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
9433                                          SDLoc(SymbolicCallee),
9434                                          SymbolicCallee->getValueType(0));
9435 
9436   // Get the real number of arguments participating in the call <numArgs>
9437   SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
9438   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
9439 
9440   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
9441   // Intrinsics include all meta-operands up to but not including CC.
9442   unsigned NumMetaOpers = PatchPointOpers::CCPos;
9443   assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
9444          "Not enough arguments provided to the patchpoint intrinsic");
9445 
9446   // For AnyRegCC the arguments are lowered later on manually.
9447   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
9448   Type *ReturnTy =
9449       IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
9450 
9451   TargetLowering::CallLoweringInfo CLI(DAG);
9452   populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
9453                            ReturnTy, true);
9454   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9455 
9456   SDNode *CallEnd = Result.second.getNode();
9457   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
9458     CallEnd = CallEnd->getOperand(0).getNode();
9459 
9460   /// Get a call instruction from the call sequence chain.
9461   /// Tail calls are not allowed.
9462   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
9463          "Expected a callseq node.");
9464   SDNode *Call = CallEnd->getOperand(0).getNode();
9465   bool HasGlue = Call->getGluedNode();
9466 
9467   // Replace the target specific call node with the patchable intrinsic.
9468   SmallVector<SDValue, 8> Ops;
9469 
9470   // Add the <id> and <numBytes> constants.
9471   SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
9472   Ops.push_back(DAG.getTargetConstant(
9473                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
9474   SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
9475   Ops.push_back(DAG.getTargetConstant(
9476                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
9477                   MVT::i32));
9478 
9479   // Add the callee.
9480   Ops.push_back(Callee);
9481 
9482   // Adjust <numArgs> to account for any arguments that have been passed on the
9483   // stack instead.
9484   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
9485   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
9486   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
9487   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
9488 
9489   // Add the calling convention
9490   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
9491 
9492   // Add the arguments we omitted previously. The register allocator should
9493   // place these in any free register.
9494   if (IsAnyRegCC)
9495     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
9496       Ops.push_back(getValue(CB.getArgOperand(i)));
9497 
9498   // Push the arguments from the call instruction up to the register mask.
9499   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
9500   Ops.append(Call->op_begin() + 2, e);
9501 
9502   // Push live variables for the stack map.
9503   addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
9504 
9505   // Push the register mask info.
9506   if (HasGlue)
9507     Ops.push_back(*(Call->op_end()-2));
9508   else
9509     Ops.push_back(*(Call->op_end()-1));
9510 
9511   // Push the chain (this is originally the first operand of the call, but
9512   // becomes now the last or second to last operand).
9513   Ops.push_back(*(Call->op_begin()));
9514 
9515   // Push the glue flag (last operand).
9516   if (HasGlue)
9517     Ops.push_back(*(Call->op_end()-1));
9518 
9519   SDVTList NodeTys;
9520   if (IsAnyRegCC && HasDef) {
9521     // Create the return types based on the intrinsic definition
9522     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9523     SmallVector<EVT, 3> ValueVTs;
9524     ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
9525     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
9526 
9527     // There is always a chain and a glue type at the end
9528     ValueVTs.push_back(MVT::Other);
9529     ValueVTs.push_back(MVT::Glue);
9530     NodeTys = DAG.getVTList(ValueVTs);
9531   } else
9532     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9533 
9534   // Replace the target specific call node with a PATCHPOINT node.
9535   MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
9536                                          dl, NodeTys, Ops);
9537 
9538   // Update the NodeMap.
9539   if (HasDef) {
9540     if (IsAnyRegCC)
9541       setValue(&CB, SDValue(MN, 0));
9542     else
9543       setValue(&CB, Result.first);
9544   }
9545 
9546   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
9547   // call sequence. Furthermore the location of the chain and glue can change
9548   // when the AnyReg calling convention is used and the intrinsic returns a
9549   // value.
9550   if (IsAnyRegCC && HasDef) {
9551     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
9552     SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
9553     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
9554   } else
9555     DAG.ReplaceAllUsesWith(Call, MN);
9556   DAG.DeleteNode(Call);
9557 
9558   // Inform the Frame Information that we have a patchpoint in this function.
9559   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
9560 }
9561 
9562 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
9563                                             unsigned Intrinsic) {
9564   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9565   SDValue Op1 = getValue(I.getArgOperand(0));
9566   SDValue Op2;
9567   if (I.arg_size() > 1)
9568     Op2 = getValue(I.getArgOperand(1));
9569   SDLoc dl = getCurSDLoc();
9570   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
9571   SDValue Res;
9572   SDNodeFlags SDFlags;
9573   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
9574     SDFlags.copyFMF(*FPMO);
9575 
9576   switch (Intrinsic) {
9577   case Intrinsic::vector_reduce_fadd:
9578     if (SDFlags.hasAllowReassociation())
9579       Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
9580                         DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
9581                         SDFlags);
9582     else
9583       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
9584     break;
9585   case Intrinsic::vector_reduce_fmul:
9586     if (SDFlags.hasAllowReassociation())
9587       Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
9588                         DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
9589                         SDFlags);
9590     else
9591       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
9592     break;
9593   case Intrinsic::vector_reduce_add:
9594     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
9595     break;
9596   case Intrinsic::vector_reduce_mul:
9597     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
9598     break;
9599   case Intrinsic::vector_reduce_and:
9600     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
9601     break;
9602   case Intrinsic::vector_reduce_or:
9603     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
9604     break;
9605   case Intrinsic::vector_reduce_xor:
9606     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
9607     break;
9608   case Intrinsic::vector_reduce_smax:
9609     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
9610     break;
9611   case Intrinsic::vector_reduce_smin:
9612     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
9613     break;
9614   case Intrinsic::vector_reduce_umax:
9615     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
9616     break;
9617   case Intrinsic::vector_reduce_umin:
9618     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
9619     break;
9620   case Intrinsic::vector_reduce_fmax:
9621     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
9622     break;
9623   case Intrinsic::vector_reduce_fmin:
9624     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
9625     break;
9626   default:
9627     llvm_unreachable("Unhandled vector reduce intrinsic");
9628   }
9629   setValue(&I, Res);
9630 }
9631 
9632 /// Returns an AttributeList representing the attributes applied to the return
9633 /// value of the given call.
9634 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
9635   SmallVector<Attribute::AttrKind, 2> Attrs;
9636   if (CLI.RetSExt)
9637     Attrs.push_back(Attribute::SExt);
9638   if (CLI.RetZExt)
9639     Attrs.push_back(Attribute::ZExt);
9640   if (CLI.IsInReg)
9641     Attrs.push_back(Attribute::InReg);
9642 
9643   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
9644                             Attrs);
9645 }
9646 
9647 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
9648 /// implementation, which just calls LowerCall.
9649 /// FIXME: When all targets are
9650 /// migrated to using LowerCall, this hook should be integrated into SDISel.
9651 std::pair<SDValue, SDValue>
9652 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
9653   // Handle the incoming return values from the call.
9654   CLI.Ins.clear();
9655   Type *OrigRetTy = CLI.RetTy;
9656   SmallVector<EVT, 4> RetTys;
9657   SmallVector<uint64_t, 4> Offsets;
9658   auto &DL = CLI.DAG.getDataLayout();
9659   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
9660 
9661   if (CLI.IsPostTypeLegalization) {
9662     // If we are lowering a libcall after legalization, split the return type.
9663     SmallVector<EVT, 4> OldRetTys;
9664     SmallVector<uint64_t, 4> OldOffsets;
9665     RetTys.swap(OldRetTys);
9666     Offsets.swap(OldOffsets);
9667 
9668     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
9669       EVT RetVT = OldRetTys[i];
9670       uint64_t Offset = OldOffsets[i];
9671       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
9672       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
9673       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
9674       RetTys.append(NumRegs, RegisterVT);
9675       for (unsigned j = 0; j != NumRegs; ++j)
9676         Offsets.push_back(Offset + j * RegisterVTByteSZ);
9677     }
9678   }
9679 
9680   SmallVector<ISD::OutputArg, 4> Outs;
9681   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
9682 
9683   bool CanLowerReturn =
9684       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
9685                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
9686 
9687   SDValue DemoteStackSlot;
9688   int DemoteStackIdx = -100;
9689   if (!CanLowerReturn) {
9690     // FIXME: equivalent assert?
9691     // assert(!CS.hasInAllocaArgument() &&
9692     //        "sret demotion is incompatible with inalloca");
9693     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
9694     Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
9695     MachineFunction &MF = CLI.DAG.getMachineFunction();
9696     DemoteStackIdx =
9697         MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
9698     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
9699                                               DL.getAllocaAddrSpace());
9700 
9701     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
9702     ArgListEntry Entry;
9703     Entry.Node = DemoteStackSlot;
9704     Entry.Ty = StackSlotPtrType;
9705     Entry.IsSExt = false;
9706     Entry.IsZExt = false;
9707     Entry.IsInReg = false;
9708     Entry.IsSRet = true;
9709     Entry.IsNest = false;
9710     Entry.IsByVal = false;
9711     Entry.IsByRef = false;
9712     Entry.IsReturned = false;
9713     Entry.IsSwiftSelf = false;
9714     Entry.IsSwiftAsync = false;
9715     Entry.IsSwiftError = false;
9716     Entry.IsCFGuardTarget = false;
9717     Entry.Alignment = Alignment;
9718     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
9719     CLI.NumFixedArgs += 1;
9720     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
9721 
9722     // sret demotion isn't compatible with tail-calls, since the sret argument
9723     // points into the callers stack frame.
9724     CLI.IsTailCall = false;
9725   } else {
9726     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9727         CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
9728     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
9729       ISD::ArgFlagsTy Flags;
9730       if (NeedsRegBlock) {
9731         Flags.setInConsecutiveRegs();
9732         if (I == RetTys.size() - 1)
9733           Flags.setInConsecutiveRegsLast();
9734       }
9735       EVT VT = RetTys[I];
9736       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9737                                                      CLI.CallConv, VT);
9738       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9739                                                        CLI.CallConv, VT);
9740       for (unsigned i = 0; i != NumRegs; ++i) {
9741         ISD::InputArg MyFlags;
9742         MyFlags.Flags = Flags;
9743         MyFlags.VT = RegisterVT;
9744         MyFlags.ArgVT = VT;
9745         MyFlags.Used = CLI.IsReturnValueUsed;
9746         if (CLI.RetTy->isPointerTy()) {
9747           MyFlags.Flags.setPointer();
9748           MyFlags.Flags.setPointerAddrSpace(
9749               cast<PointerType>(CLI.RetTy)->getAddressSpace());
9750         }
9751         if (CLI.RetSExt)
9752           MyFlags.Flags.setSExt();
9753         if (CLI.RetZExt)
9754           MyFlags.Flags.setZExt();
9755         if (CLI.IsInReg)
9756           MyFlags.Flags.setInReg();
9757         CLI.Ins.push_back(MyFlags);
9758       }
9759     }
9760   }
9761 
9762   // We push in swifterror return as the last element of CLI.Ins.
9763   ArgListTy &Args = CLI.getArgs();
9764   if (supportSwiftError()) {
9765     for (const ArgListEntry &Arg : Args) {
9766       if (Arg.IsSwiftError) {
9767         ISD::InputArg MyFlags;
9768         MyFlags.VT = getPointerTy(DL);
9769         MyFlags.ArgVT = EVT(getPointerTy(DL));
9770         MyFlags.Flags.setSwiftError();
9771         CLI.Ins.push_back(MyFlags);
9772       }
9773     }
9774   }
9775 
9776   // Handle all of the outgoing arguments.
9777   CLI.Outs.clear();
9778   CLI.OutVals.clear();
9779   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
9780     SmallVector<EVT, 4> ValueVTs;
9781     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
9782     // FIXME: Split arguments if CLI.IsPostTypeLegalization
9783     Type *FinalType = Args[i].Ty;
9784     if (Args[i].IsByVal)
9785       FinalType = Args[i].IndirectType;
9786     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9787         FinalType, CLI.CallConv, CLI.IsVarArg, DL);
9788     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
9789          ++Value) {
9790       EVT VT = ValueVTs[Value];
9791       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
9792       SDValue Op = SDValue(Args[i].Node.getNode(),
9793                            Args[i].Node.getResNo() + Value);
9794       ISD::ArgFlagsTy Flags;
9795 
9796       // Certain targets (such as MIPS), may have a different ABI alignment
9797       // for a type depending on the context. Give the target a chance to
9798       // specify the alignment it wants.
9799       const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
9800       Flags.setOrigAlign(OriginalAlignment);
9801 
9802       if (Args[i].Ty->isPointerTy()) {
9803         Flags.setPointer();
9804         Flags.setPointerAddrSpace(
9805             cast<PointerType>(Args[i].Ty)->getAddressSpace());
9806       }
9807       if (Args[i].IsZExt)
9808         Flags.setZExt();
9809       if (Args[i].IsSExt)
9810         Flags.setSExt();
9811       if (Args[i].IsInReg) {
9812         // If we are using vectorcall calling convention, a structure that is
9813         // passed InReg - is surely an HVA
9814         if (CLI.CallConv == CallingConv::X86_VectorCall &&
9815             isa<StructType>(FinalType)) {
9816           // The first value of a structure is marked
9817           if (0 == Value)
9818             Flags.setHvaStart();
9819           Flags.setHva();
9820         }
9821         // Set InReg Flag
9822         Flags.setInReg();
9823       }
9824       if (Args[i].IsSRet)
9825         Flags.setSRet();
9826       if (Args[i].IsSwiftSelf)
9827         Flags.setSwiftSelf();
9828       if (Args[i].IsSwiftAsync)
9829         Flags.setSwiftAsync();
9830       if (Args[i].IsSwiftError)
9831         Flags.setSwiftError();
9832       if (Args[i].IsCFGuardTarget)
9833         Flags.setCFGuardTarget();
9834       if (Args[i].IsByVal)
9835         Flags.setByVal();
9836       if (Args[i].IsByRef)
9837         Flags.setByRef();
9838       if (Args[i].IsPreallocated) {
9839         Flags.setPreallocated();
9840         // Set the byval flag for CCAssignFn callbacks that don't know about
9841         // preallocated.  This way we can know how many bytes we should've
9842         // allocated and how many bytes a callee cleanup function will pop.  If
9843         // we port preallocated to more targets, we'll have to add custom
9844         // preallocated handling in the various CC lowering callbacks.
9845         Flags.setByVal();
9846       }
9847       if (Args[i].IsInAlloca) {
9848         Flags.setInAlloca();
9849         // Set the byval flag for CCAssignFn callbacks that don't know about
9850         // inalloca.  This way we can know how many bytes we should've allocated
9851         // and how many bytes a callee cleanup function will pop.  If we port
9852         // inalloca to more targets, we'll have to add custom inalloca handling
9853         // in the various CC lowering callbacks.
9854         Flags.setByVal();
9855       }
9856       Align MemAlign;
9857       if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
9858         unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
9859         Flags.setByValSize(FrameSize);
9860 
9861         // info is not there but there are cases it cannot get right.
9862         if (auto MA = Args[i].Alignment)
9863           MemAlign = *MA;
9864         else
9865           MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
9866       } else if (auto MA = Args[i].Alignment) {
9867         MemAlign = *MA;
9868       } else {
9869         MemAlign = OriginalAlignment;
9870       }
9871       Flags.setMemAlign(MemAlign);
9872       if (Args[i].IsNest)
9873         Flags.setNest();
9874       if (NeedsRegBlock)
9875         Flags.setInConsecutiveRegs();
9876 
9877       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9878                                                  CLI.CallConv, VT);
9879       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9880                                                         CLI.CallConv, VT);
9881       SmallVector<SDValue, 4> Parts(NumParts);
9882       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
9883 
9884       if (Args[i].IsSExt)
9885         ExtendKind = ISD::SIGN_EXTEND;
9886       else if (Args[i].IsZExt)
9887         ExtendKind = ISD::ZERO_EXTEND;
9888 
9889       // Conservatively only handle 'returned' on non-vectors that can be lowered,
9890       // for now.
9891       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
9892           CanLowerReturn) {
9893         assert((CLI.RetTy == Args[i].Ty ||
9894                 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
9895                  CLI.RetTy->getPointerAddressSpace() ==
9896                      Args[i].Ty->getPointerAddressSpace())) &&
9897                RetTys.size() == NumValues && "unexpected use of 'returned'");
9898         // Before passing 'returned' to the target lowering code, ensure that
9899         // either the register MVT and the actual EVT are the same size or that
9900         // the return value and argument are extended in the same way; in these
9901         // cases it's safe to pass the argument register value unchanged as the
9902         // return register value (although it's at the target's option whether
9903         // to do so)
9904         // TODO: allow code generation to take advantage of partially preserved
9905         // registers rather than clobbering the entire register when the
9906         // parameter extension method is not compatible with the return
9907         // extension method
9908         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
9909             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
9910              CLI.RetZExt == Args[i].IsZExt))
9911           Flags.setReturned();
9912       }
9913 
9914       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
9915                      CLI.CallConv, ExtendKind);
9916 
9917       for (unsigned j = 0; j != NumParts; ++j) {
9918         // if it isn't first piece, alignment must be 1
9919         // For scalable vectors the scalable part is currently handled
9920         // by individual targets, so we just use the known minimum size here.
9921         ISD::OutputArg MyFlags(
9922             Flags, Parts[j].getValueType().getSimpleVT(), VT,
9923             i < CLI.NumFixedArgs, i,
9924             j * Parts[j].getValueType().getStoreSize().getKnownMinSize());
9925         if (NumParts > 1 && j == 0)
9926           MyFlags.Flags.setSplit();
9927         else if (j != 0) {
9928           MyFlags.Flags.setOrigAlign(Align(1));
9929           if (j == NumParts - 1)
9930             MyFlags.Flags.setSplitEnd();
9931         }
9932 
9933         CLI.Outs.push_back(MyFlags);
9934         CLI.OutVals.push_back(Parts[j]);
9935       }
9936 
9937       if (NeedsRegBlock && Value == NumValues - 1)
9938         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
9939     }
9940   }
9941 
9942   SmallVector<SDValue, 4> InVals;
9943   CLI.Chain = LowerCall(CLI, InVals);
9944 
9945   // Update CLI.InVals to use outside of this function.
9946   CLI.InVals = InVals;
9947 
9948   // Verify that the target's LowerCall behaved as expected.
9949   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
9950          "LowerCall didn't return a valid chain!");
9951   assert((!CLI.IsTailCall || InVals.empty()) &&
9952          "LowerCall emitted a return value for a tail call!");
9953   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
9954          "LowerCall didn't emit the correct number of values!");
9955 
9956   // For a tail call, the return value is merely live-out and there aren't
9957   // any nodes in the DAG representing it. Return a special value to
9958   // indicate that a tail call has been emitted and no more Instructions
9959   // should be processed in the current block.
9960   if (CLI.IsTailCall) {
9961     CLI.DAG.setRoot(CLI.Chain);
9962     return std::make_pair(SDValue(), SDValue());
9963   }
9964 
9965 #ifndef NDEBUG
9966   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
9967     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
9968     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
9969            "LowerCall emitted a value with the wrong type!");
9970   }
9971 #endif
9972 
9973   SmallVector<SDValue, 4> ReturnValues;
9974   if (!CanLowerReturn) {
9975     // The instruction result is the result of loading from the
9976     // hidden sret parameter.
9977     SmallVector<EVT, 1> PVTs;
9978     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
9979 
9980     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
9981     assert(PVTs.size() == 1 && "Pointers should fit in one register");
9982     EVT PtrVT = PVTs[0];
9983 
9984     unsigned NumValues = RetTys.size();
9985     ReturnValues.resize(NumValues);
9986     SmallVector<SDValue, 4> Chains(NumValues);
9987 
9988     // An aggregate return value cannot wrap around the address space, so
9989     // offsets to its parts don't wrap either.
9990     SDNodeFlags Flags;
9991     Flags.setNoUnsignedWrap(true);
9992 
9993     MachineFunction &MF = CLI.DAG.getMachineFunction();
9994     Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
9995     for (unsigned i = 0; i < NumValues; ++i) {
9996       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
9997                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
9998                                                         PtrVT), Flags);
9999       SDValue L = CLI.DAG.getLoad(
10000           RetTys[i], CLI.DL, CLI.Chain, Add,
10001           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
10002                                             DemoteStackIdx, Offsets[i]),
10003           HiddenSRetAlign);
10004       ReturnValues[i] = L;
10005       Chains[i] = L.getValue(1);
10006     }
10007 
10008     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10009   } else {
10010     // Collect the legal value parts into potentially illegal values
10011     // that correspond to the original function's return values.
10012     Optional<ISD::NodeType> AssertOp;
10013     if (CLI.RetSExt)
10014       AssertOp = ISD::AssertSext;
10015     else if (CLI.RetZExt)
10016       AssertOp = ISD::AssertZext;
10017     unsigned CurReg = 0;
10018     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10019       EVT VT = RetTys[I];
10020       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
10021                                                      CLI.CallConv, VT);
10022       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10023                                                        CLI.CallConv, VT);
10024 
10025       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
10026                                               NumRegs, RegisterVT, VT, nullptr,
10027                                               CLI.CallConv, AssertOp));
10028       CurReg += NumRegs;
10029     }
10030 
10031     // For a function returning void, there is no return value. We can't create
10032     // such a node, so we just return a null return value in that case. In
10033     // that case, nothing will actually look at the value.
10034     if (ReturnValues.empty())
10035       return std::make_pair(SDValue(), CLI.Chain);
10036   }
10037 
10038   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10039                                 CLI.DAG.getVTList(RetTys), ReturnValues);
10040   return std::make_pair(Res, CLI.Chain);
10041 }
10042 
10043 /// Places new result values for the node in Results (their number
10044 /// and types must exactly match those of the original return values of
10045 /// the node), or leaves Results empty, which indicates that the node is not
10046 /// to be custom lowered after all.
10047 void TargetLowering::LowerOperationWrapper(SDNode *N,
10048                                            SmallVectorImpl<SDValue> &Results,
10049                                            SelectionDAG &DAG) const {
10050   SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10051 
10052   if (!Res.getNode())
10053     return;
10054 
10055   // If the original node has one result, take the return value from
10056   // LowerOperation as is. It might not be result number 0.
10057   if (N->getNumValues() == 1) {
10058     Results.push_back(Res);
10059     return;
10060   }
10061 
10062   // If the original node has multiple results, then the return node should
10063   // have the same number of results.
10064   assert((N->getNumValues() == Res->getNumValues()) &&
10065       "Lowering returned the wrong number of results!");
10066 
10067   // Places new result values base on N result number.
10068   for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10069     Results.push_back(Res.getValue(I));
10070 }
10071 
10072 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10073   llvm_unreachable("LowerOperation not implemented for this target!");
10074 }
10075 
10076 void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
10077                                                      unsigned Reg,
10078                                                      ISD::NodeType ExtendType) {
10079   SDValue Op = getNonRegisterValue(V);
10080   assert((Op.getOpcode() != ISD::CopyFromReg ||
10081           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10082          "Copy from a reg to the same reg!");
10083   assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10084 
10085   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10086   // If this is an InlineAsm we have to match the registers required, not the
10087   // notional registers required by the type.
10088 
10089   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10090                    None); // This is not an ABI copy.
10091   SDValue Chain = DAG.getEntryNode();
10092 
10093   if (ExtendType == ISD::ANY_EXTEND) {
10094     auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10095     if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10096       ExtendType = PreferredExtendIt->second;
10097   }
10098   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10099   PendingExports.push_back(Chain);
10100 }
10101 
10102 #include "llvm/CodeGen/SelectionDAGISel.h"
10103 
10104 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10105 /// entry block, return true.  This includes arguments used by switches, since
10106 /// the switch may expand into multiple basic blocks.
10107 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10108   // With FastISel active, we may be splitting blocks, so force creation
10109   // of virtual registers for all non-dead arguments.
10110   if (FastISel)
10111     return A->use_empty();
10112 
10113   const BasicBlock &Entry = A->getParent()->front();
10114   for (const User *U : A->users())
10115     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10116       return false;  // Use not in entry block.
10117 
10118   return true;
10119 }
10120 
10121 using ArgCopyElisionMapTy =
10122     DenseMap<const Argument *,
10123              std::pair<const AllocaInst *, const StoreInst *>>;
10124 
10125 /// Scan the entry block of the function in FuncInfo for arguments that look
10126 /// like copies into a local alloca. Record any copied arguments in
10127 /// ArgCopyElisionCandidates.
10128 static void
10129 findArgumentCopyElisionCandidates(const DataLayout &DL,
10130                                   FunctionLoweringInfo *FuncInfo,
10131                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10132   // Record the state of every static alloca used in the entry block. Argument
10133   // allocas are all used in the entry block, so we need approximately as many
10134   // entries as we have arguments.
10135   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10136   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
10137   unsigned NumArgs = FuncInfo->Fn->arg_size();
10138   StaticAllocas.reserve(NumArgs * 2);
10139 
10140   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10141     if (!V)
10142       return nullptr;
10143     V = V->stripPointerCasts();
10144     const auto *AI = dyn_cast<AllocaInst>(V);
10145     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10146       return nullptr;
10147     auto Iter = StaticAllocas.insert({AI, Unknown});
10148     return &Iter.first->second;
10149   };
10150 
10151   // Look for stores of arguments to static allocas. Look through bitcasts and
10152   // GEPs to handle type coercions, as long as the alloca is fully initialized
10153   // by the store. Any non-store use of an alloca escapes it and any subsequent
10154   // unanalyzed store might write it.
10155   // FIXME: Handle structs initialized with multiple stores.
10156   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
10157     // Look for stores, and handle non-store uses conservatively.
10158     const auto *SI = dyn_cast<StoreInst>(&I);
10159     if (!SI) {
10160       // We will look through cast uses, so ignore them completely.
10161       if (I.isCast())
10162         continue;
10163       // Ignore debug info and pseudo op intrinsics, they don't escape or store
10164       // to allocas.
10165       if (I.isDebugOrPseudoInst())
10166         continue;
10167       // This is an unknown instruction. Assume it escapes or writes to all
10168       // static alloca operands.
10169       for (const Use &U : I.operands()) {
10170         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
10171           *Info = StaticAllocaInfo::Clobbered;
10172       }
10173       continue;
10174     }
10175 
10176     // If the stored value is a static alloca, mark it as escaped.
10177     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
10178       *Info = StaticAllocaInfo::Clobbered;
10179 
10180     // Check if the destination is a static alloca.
10181     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
10182     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
10183     if (!Info)
10184       continue;
10185     const AllocaInst *AI = cast<AllocaInst>(Dst);
10186 
10187     // Skip allocas that have been initialized or clobbered.
10188     if (*Info != StaticAllocaInfo::Unknown)
10189       continue;
10190 
10191     // Check if the stored value is an argument, and that this store fully
10192     // initializes the alloca.
10193     // If the argument type has padding bits we can't directly forward a pointer
10194     // as the upper bits may contain garbage.
10195     // Don't elide copies from the same argument twice.
10196     const Value *Val = SI->getValueOperand()->stripPointerCasts();
10197     const auto *Arg = dyn_cast<Argument>(Val);
10198     if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
10199         Arg->getType()->isEmptyTy() ||
10200         DL.getTypeStoreSize(Arg->getType()) !=
10201             DL.getTypeAllocSize(AI->getAllocatedType()) ||
10202         !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
10203         ArgCopyElisionCandidates.count(Arg)) {
10204       *Info = StaticAllocaInfo::Clobbered;
10205       continue;
10206     }
10207 
10208     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
10209                       << '\n');
10210 
10211     // Mark this alloca and store for argument copy elision.
10212     *Info = StaticAllocaInfo::Elidable;
10213     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
10214 
10215     // Stop scanning if we've seen all arguments. This will happen early in -O0
10216     // builds, which is useful, because -O0 builds have large entry blocks and
10217     // many allocas.
10218     if (ArgCopyElisionCandidates.size() == NumArgs)
10219       break;
10220   }
10221 }
10222 
10223 /// Try to elide argument copies from memory into a local alloca. Succeeds if
10224 /// ArgVal is a load from a suitable fixed stack object.
10225 static void tryToElideArgumentCopy(
10226     FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
10227     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
10228     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
10229     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
10230     SDValue ArgVal, bool &ArgHasUses) {
10231   // Check if this is a load from a fixed stack object.
10232   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
10233   if (!LNode)
10234     return;
10235   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
10236   if (!FINode)
10237     return;
10238 
10239   // Check that the fixed stack object is the right size and alignment.
10240   // Look at the alignment that the user wrote on the alloca instead of looking
10241   // at the stack object.
10242   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
10243   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
10244   const AllocaInst *AI = ArgCopyIter->second.first;
10245   int FixedIndex = FINode->getIndex();
10246   int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
10247   int OldIndex = AllocaIndex;
10248   MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
10249   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
10250     LLVM_DEBUG(
10251         dbgs() << "  argument copy elision failed due to bad fixed stack "
10252                   "object size\n");
10253     return;
10254   }
10255   Align RequiredAlignment = AI->getAlign();
10256   if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
10257     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
10258                          "greater than stack argument alignment ("
10259                       << DebugStr(RequiredAlignment) << " vs "
10260                       << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
10261     return;
10262   }
10263 
10264   // Perform the elision. Delete the old stack object and replace its only use
10265   // in the variable info map. Mark the stack object as mutable.
10266   LLVM_DEBUG({
10267     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
10268            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
10269            << '\n';
10270   });
10271   MFI.RemoveStackObject(OldIndex);
10272   MFI.setIsImmutableObjectIndex(FixedIndex, false);
10273   AllocaIndex = FixedIndex;
10274   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
10275   Chains.push_back(ArgVal.getValue(1));
10276 
10277   // Avoid emitting code for the store implementing the copy.
10278   const StoreInst *SI = ArgCopyIter->second.second;
10279   ElidedArgCopyInstrs.insert(SI);
10280 
10281   // Check for uses of the argument again so that we can avoid exporting ArgVal
10282   // if it is't used by anything other than the store.
10283   for (const Value *U : Arg.users()) {
10284     if (U != SI) {
10285       ArgHasUses = true;
10286       break;
10287     }
10288   }
10289 }
10290 
10291 void SelectionDAGISel::LowerArguments(const Function &F) {
10292   SelectionDAG &DAG = SDB->DAG;
10293   SDLoc dl = SDB->getCurSDLoc();
10294   const DataLayout &DL = DAG.getDataLayout();
10295   SmallVector<ISD::InputArg, 16> Ins;
10296 
10297   // In Naked functions we aren't going to save any registers.
10298   if (F.hasFnAttribute(Attribute::Naked))
10299     return;
10300 
10301   if (!FuncInfo->CanLowerReturn) {
10302     // Put in an sret pointer parameter before all the other parameters.
10303     SmallVector<EVT, 1> ValueVTs;
10304     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10305                     F.getReturnType()->getPointerTo(
10306                         DAG.getDataLayout().getAllocaAddrSpace()),
10307                     ValueVTs);
10308 
10309     // NOTE: Assuming that a pointer will never break down to more than one VT
10310     // or one register.
10311     ISD::ArgFlagsTy Flags;
10312     Flags.setSRet();
10313     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
10314     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
10315                          ISD::InputArg::NoArgIndex, 0);
10316     Ins.push_back(RetArg);
10317   }
10318 
10319   // Look for stores of arguments to static allocas. Mark such arguments with a
10320   // flag to ask the target to give us the memory location of that argument if
10321   // available.
10322   ArgCopyElisionMapTy ArgCopyElisionCandidates;
10323   findArgumentCopyElisionCandidates(DL, FuncInfo.get(),
10324                                     ArgCopyElisionCandidates);
10325 
10326   // Set up the incoming argument description vector.
10327   for (const Argument &Arg : F.args()) {
10328     unsigned ArgNo = Arg.getArgNo();
10329     SmallVector<EVT, 4> ValueVTs;
10330     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10331     bool isArgValueUsed = !Arg.use_empty();
10332     unsigned PartBase = 0;
10333     Type *FinalType = Arg.getType();
10334     if (Arg.hasAttribute(Attribute::ByVal))
10335       FinalType = Arg.getParamByValType();
10336     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
10337         FinalType, F.getCallingConv(), F.isVarArg(), DL);
10338     for (unsigned Value = 0, NumValues = ValueVTs.size();
10339          Value != NumValues; ++Value) {
10340       EVT VT = ValueVTs[Value];
10341       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
10342       ISD::ArgFlagsTy Flags;
10343 
10344 
10345       if (Arg.getType()->isPointerTy()) {
10346         Flags.setPointer();
10347         Flags.setPointerAddrSpace(
10348             cast<PointerType>(Arg.getType())->getAddressSpace());
10349       }
10350       if (Arg.hasAttribute(Attribute::ZExt))
10351         Flags.setZExt();
10352       if (Arg.hasAttribute(Attribute::SExt))
10353         Flags.setSExt();
10354       if (Arg.hasAttribute(Attribute::InReg)) {
10355         // If we are using vectorcall calling convention, a structure that is
10356         // passed InReg - is surely an HVA
10357         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
10358             isa<StructType>(Arg.getType())) {
10359           // The first value of a structure is marked
10360           if (0 == Value)
10361             Flags.setHvaStart();
10362           Flags.setHva();
10363         }
10364         // Set InReg Flag
10365         Flags.setInReg();
10366       }
10367       if (Arg.hasAttribute(Attribute::StructRet))
10368         Flags.setSRet();
10369       if (Arg.hasAttribute(Attribute::SwiftSelf))
10370         Flags.setSwiftSelf();
10371       if (Arg.hasAttribute(Attribute::SwiftAsync))
10372         Flags.setSwiftAsync();
10373       if (Arg.hasAttribute(Attribute::SwiftError))
10374         Flags.setSwiftError();
10375       if (Arg.hasAttribute(Attribute::ByVal))
10376         Flags.setByVal();
10377       if (Arg.hasAttribute(Attribute::ByRef))
10378         Flags.setByRef();
10379       if (Arg.hasAttribute(Attribute::InAlloca)) {
10380         Flags.setInAlloca();
10381         // Set the byval flag for CCAssignFn callbacks that don't know about
10382         // inalloca.  This way we can know how many bytes we should've allocated
10383         // and how many bytes a callee cleanup function will pop.  If we port
10384         // inalloca to more targets, we'll have to add custom inalloca handling
10385         // in the various CC lowering callbacks.
10386         Flags.setByVal();
10387       }
10388       if (Arg.hasAttribute(Attribute::Preallocated)) {
10389         Flags.setPreallocated();
10390         // Set the byval flag for CCAssignFn callbacks that don't know about
10391         // preallocated.  This way we can know how many bytes we should've
10392         // allocated and how many bytes a callee cleanup function will pop.  If
10393         // we port preallocated to more targets, we'll have to add custom
10394         // preallocated handling in the various CC lowering callbacks.
10395         Flags.setByVal();
10396       }
10397 
10398       // Certain targets (such as MIPS), may have a different ABI alignment
10399       // for a type depending on the context. Give the target a chance to
10400       // specify the alignment it wants.
10401       const Align OriginalAlignment(
10402           TLI->getABIAlignmentForCallingConv(ArgTy, DL));
10403       Flags.setOrigAlign(OriginalAlignment);
10404 
10405       Align MemAlign;
10406       Type *ArgMemTy = nullptr;
10407       if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
10408           Flags.isByRef()) {
10409         if (!ArgMemTy)
10410           ArgMemTy = Arg.getPointeeInMemoryValueType();
10411 
10412         uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
10413 
10414         // For in-memory arguments, size and alignment should be passed from FE.
10415         // BE will guess if this info is not there but there are cases it cannot
10416         // get right.
10417         if (auto ParamAlign = Arg.getParamStackAlign())
10418           MemAlign = *ParamAlign;
10419         else if ((ParamAlign = Arg.getParamAlign()))
10420           MemAlign = *ParamAlign;
10421         else
10422           MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
10423         if (Flags.isByRef())
10424           Flags.setByRefSize(MemSize);
10425         else
10426           Flags.setByValSize(MemSize);
10427       } else if (auto ParamAlign = Arg.getParamStackAlign()) {
10428         MemAlign = *ParamAlign;
10429       } else {
10430         MemAlign = OriginalAlignment;
10431       }
10432       Flags.setMemAlign(MemAlign);
10433 
10434       if (Arg.hasAttribute(Attribute::Nest))
10435         Flags.setNest();
10436       if (NeedsRegBlock)
10437         Flags.setInConsecutiveRegs();
10438       if (ArgCopyElisionCandidates.count(&Arg))
10439         Flags.setCopyElisionCandidate();
10440       if (Arg.hasAttribute(Attribute::Returned))
10441         Flags.setReturned();
10442 
10443       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
10444           *CurDAG->getContext(), F.getCallingConv(), VT);
10445       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
10446           *CurDAG->getContext(), F.getCallingConv(), VT);
10447       for (unsigned i = 0; i != NumRegs; ++i) {
10448         // For scalable vectors, use the minimum size; individual targets
10449         // are responsible for handling scalable vector arguments and
10450         // return values.
10451         ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
10452                  ArgNo, PartBase+i*RegisterVT.getStoreSize().getKnownMinSize());
10453         if (NumRegs > 1 && i == 0)
10454           MyFlags.Flags.setSplit();
10455         // if it isn't first piece, alignment must be 1
10456         else if (i > 0) {
10457           MyFlags.Flags.setOrigAlign(Align(1));
10458           if (i == NumRegs - 1)
10459             MyFlags.Flags.setSplitEnd();
10460         }
10461         Ins.push_back(MyFlags);
10462       }
10463       if (NeedsRegBlock && Value == NumValues - 1)
10464         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
10465       PartBase += VT.getStoreSize().getKnownMinSize();
10466     }
10467   }
10468 
10469   // Call the target to set up the argument values.
10470   SmallVector<SDValue, 8> InVals;
10471   SDValue NewRoot = TLI->LowerFormalArguments(
10472       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
10473 
10474   // Verify that the target's LowerFormalArguments behaved as expected.
10475   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
10476          "LowerFormalArguments didn't return a valid chain!");
10477   assert(InVals.size() == Ins.size() &&
10478          "LowerFormalArguments didn't emit the correct number of values!");
10479   LLVM_DEBUG({
10480     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10481       assert(InVals[i].getNode() &&
10482              "LowerFormalArguments emitted a null value!");
10483       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
10484              "LowerFormalArguments emitted a value with the wrong type!");
10485     }
10486   });
10487 
10488   // Update the DAG with the new chain value resulting from argument lowering.
10489   DAG.setRoot(NewRoot);
10490 
10491   // Set up the argument values.
10492   unsigned i = 0;
10493   if (!FuncInfo->CanLowerReturn) {
10494     // Create a virtual register for the sret pointer, and put in a copy
10495     // from the sret argument into it.
10496     SmallVector<EVT, 1> ValueVTs;
10497     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10498                     F.getReturnType()->getPointerTo(
10499                         DAG.getDataLayout().getAllocaAddrSpace()),
10500                     ValueVTs);
10501     MVT VT = ValueVTs[0].getSimpleVT();
10502     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
10503     Optional<ISD::NodeType> AssertOp = None;
10504     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
10505                                         nullptr, F.getCallingConv(), AssertOp);
10506 
10507     MachineFunction& MF = SDB->DAG.getMachineFunction();
10508     MachineRegisterInfo& RegInfo = MF.getRegInfo();
10509     Register SRetReg =
10510         RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
10511     FuncInfo->DemoteRegister = SRetReg;
10512     NewRoot =
10513         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
10514     DAG.setRoot(NewRoot);
10515 
10516     // i indexes lowered arguments.  Bump it past the hidden sret argument.
10517     ++i;
10518   }
10519 
10520   SmallVector<SDValue, 4> Chains;
10521   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
10522   for (const Argument &Arg : F.args()) {
10523     SmallVector<SDValue, 4> ArgValues;
10524     SmallVector<EVT, 4> ValueVTs;
10525     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10526     unsigned NumValues = ValueVTs.size();
10527     if (NumValues == 0)
10528       continue;
10529 
10530     bool ArgHasUses = !Arg.use_empty();
10531 
10532     // Elide the copying store if the target loaded this argument from a
10533     // suitable fixed stack object.
10534     if (Ins[i].Flags.isCopyElisionCandidate()) {
10535       tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
10536                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
10537                              InVals[i], ArgHasUses);
10538     }
10539 
10540     // If this argument is unused then remember its value. It is used to generate
10541     // debugging information.
10542     bool isSwiftErrorArg =
10543         TLI->supportSwiftError() &&
10544         Arg.hasAttribute(Attribute::SwiftError);
10545     if (!ArgHasUses && !isSwiftErrorArg) {
10546       SDB->setUnusedArgValue(&Arg, InVals[i]);
10547 
10548       // Also remember any frame index for use in FastISel.
10549       if (FrameIndexSDNode *FI =
10550           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
10551         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10552     }
10553 
10554     for (unsigned Val = 0; Val != NumValues; ++Val) {
10555       EVT VT = ValueVTs[Val];
10556       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
10557                                                       F.getCallingConv(), VT);
10558       unsigned NumParts = TLI->getNumRegistersForCallingConv(
10559           *CurDAG->getContext(), F.getCallingConv(), VT);
10560 
10561       // Even an apparent 'unused' swifterror argument needs to be returned. So
10562       // we do generate a copy for it that can be used on return from the
10563       // function.
10564       if (ArgHasUses || isSwiftErrorArg) {
10565         Optional<ISD::NodeType> AssertOp;
10566         if (Arg.hasAttribute(Attribute::SExt))
10567           AssertOp = ISD::AssertSext;
10568         else if (Arg.hasAttribute(Attribute::ZExt))
10569           AssertOp = ISD::AssertZext;
10570 
10571         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
10572                                              PartVT, VT, nullptr,
10573                                              F.getCallingConv(), AssertOp));
10574       }
10575 
10576       i += NumParts;
10577     }
10578 
10579     // We don't need to do anything else for unused arguments.
10580     if (ArgValues.empty())
10581       continue;
10582 
10583     // Note down frame index.
10584     if (FrameIndexSDNode *FI =
10585         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
10586       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10587 
10588     SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
10589                                      SDB->getCurSDLoc());
10590 
10591     SDB->setValue(&Arg, Res);
10592     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
10593       // We want to associate the argument with the frame index, among
10594       // involved operands, that correspond to the lowest address. The
10595       // getCopyFromParts function, called earlier, is swapping the order of
10596       // the operands to BUILD_PAIR depending on endianness. The result of
10597       // that swapping is that the least significant bits of the argument will
10598       // be in the first operand of the BUILD_PAIR node, and the most
10599       // significant bits will be in the second operand.
10600       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
10601       if (LoadSDNode *LNode =
10602           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
10603         if (FrameIndexSDNode *FI =
10604             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
10605           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10606     }
10607 
10608     // Analyses past this point are naive and don't expect an assertion.
10609     if (Res.getOpcode() == ISD::AssertZext)
10610       Res = Res.getOperand(0);
10611 
10612     // Update the SwiftErrorVRegDefMap.
10613     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
10614       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10615       if (Register::isVirtualRegister(Reg))
10616         SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
10617                                    Reg);
10618     }
10619 
10620     // If this argument is live outside of the entry block, insert a copy from
10621     // wherever we got it to the vreg that other BB's will reference it as.
10622     if (Res.getOpcode() == ISD::CopyFromReg) {
10623       // If we can, though, try to skip creating an unnecessary vreg.
10624       // FIXME: This isn't very clean... it would be nice to make this more
10625       // general.
10626       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10627       if (Register::isVirtualRegister(Reg)) {
10628         FuncInfo->ValueMap[&Arg] = Reg;
10629         continue;
10630       }
10631     }
10632     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
10633       FuncInfo->InitializeRegForValue(&Arg);
10634       SDB->CopyToExportRegsIfNeeded(&Arg);
10635     }
10636   }
10637 
10638   if (!Chains.empty()) {
10639     Chains.push_back(NewRoot);
10640     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
10641   }
10642 
10643   DAG.setRoot(NewRoot);
10644 
10645   assert(i == InVals.size() && "Argument register count mismatch!");
10646 
10647   // If any argument copy elisions occurred and we have debug info, update the
10648   // stale frame indices used in the dbg.declare variable info table.
10649   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
10650   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
10651     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
10652       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
10653       if (I != ArgCopyElisionFrameIndexMap.end())
10654         VI.Slot = I->second;
10655     }
10656   }
10657 
10658   // Finally, if the target has anything special to do, allow it to do so.
10659   emitFunctionEntryCode();
10660 }
10661 
10662 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
10663 /// ensure constants are generated when needed.  Remember the virtual registers
10664 /// that need to be added to the Machine PHI nodes as input.  We cannot just
10665 /// directly add them, because expansion might result in multiple MBB's for one
10666 /// BB.  As such, the start of the BB might correspond to a different MBB than
10667 /// the end.
10668 void
10669 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
10670   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10671   const Instruction *TI = LLVMBB->getTerminator();
10672 
10673   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
10674 
10675   // Check PHI nodes in successors that expect a value to be available from this
10676   // block.
10677   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
10678     const BasicBlock *SuccBB = TI->getSuccessor(succ);
10679     if (!isa<PHINode>(SuccBB->begin())) continue;
10680     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
10681 
10682     // If this terminator has multiple identical successors (common for
10683     // switches), only handle each succ once.
10684     if (!SuccsHandled.insert(SuccMBB).second)
10685       continue;
10686 
10687     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
10688 
10689     // At this point we know that there is a 1-1 correspondence between LLVM PHI
10690     // nodes and Machine PHI nodes, but the incoming operands have not been
10691     // emitted yet.
10692     for (const PHINode &PN : SuccBB->phis()) {
10693       // Ignore dead phi's.
10694       if (PN.use_empty())
10695         continue;
10696 
10697       // Skip empty types
10698       if (PN.getType()->isEmptyTy())
10699         continue;
10700 
10701       unsigned Reg;
10702       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
10703 
10704       if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
10705         unsigned &RegOut = ConstantsOut[C];
10706         if (RegOut == 0) {
10707           RegOut = FuncInfo.CreateRegs(C);
10708           // We need to zero/sign extend ConstantInt phi operands to match
10709           // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
10710           ISD::NodeType ExtendType = ISD::ANY_EXTEND;
10711           if (auto *CI = dyn_cast<ConstantInt>(C))
10712             ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
10713                                                     : ISD::ZERO_EXTEND;
10714           CopyValueToVirtualRegister(C, RegOut, ExtendType);
10715         }
10716         Reg = RegOut;
10717       } else {
10718         DenseMap<const Value *, Register>::iterator I =
10719           FuncInfo.ValueMap.find(PHIOp);
10720         if (I != FuncInfo.ValueMap.end())
10721           Reg = I->second;
10722         else {
10723           assert(isa<AllocaInst>(PHIOp) &&
10724                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
10725                  "Didn't codegen value into a register!??");
10726           Reg = FuncInfo.CreateRegs(PHIOp);
10727           CopyValueToVirtualRegister(PHIOp, Reg);
10728         }
10729       }
10730 
10731       // Remember that this register needs to added to the machine PHI node as
10732       // the input for this MBB.
10733       SmallVector<EVT, 4> ValueVTs;
10734       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
10735       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
10736         EVT VT = ValueVTs[vti];
10737         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
10738         for (unsigned i = 0, e = NumRegisters; i != e; ++i)
10739           FuncInfo.PHINodesToUpdate.push_back(
10740               std::make_pair(&*MBBI++, Reg + i));
10741         Reg += NumRegisters;
10742       }
10743     }
10744   }
10745 
10746   ConstantsOut.clear();
10747 }
10748 
10749 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
10750   MachineFunction::iterator I(MBB);
10751   if (++I == FuncInfo.MF->end())
10752     return nullptr;
10753   return &*I;
10754 }
10755 
10756 /// During lowering new call nodes can be created (such as memset, etc.).
10757 /// Those will become new roots of the current DAG, but complications arise
10758 /// when they are tail calls. In such cases, the call lowering will update
10759 /// the root, but the builder still needs to know that a tail call has been
10760 /// lowered in order to avoid generating an additional return.
10761 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
10762   // If the node is null, we do have a tail call.
10763   if (MaybeTC.getNode() != nullptr)
10764     DAG.setRoot(MaybeTC);
10765   else
10766     HasTailCall = true;
10767 }
10768 
10769 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
10770                                         MachineBasicBlock *SwitchMBB,
10771                                         MachineBasicBlock *DefaultMBB) {
10772   MachineFunction *CurMF = FuncInfo.MF;
10773   MachineBasicBlock *NextMBB = nullptr;
10774   MachineFunction::iterator BBI(W.MBB);
10775   if (++BBI != FuncInfo.MF->end())
10776     NextMBB = &*BBI;
10777 
10778   unsigned Size = W.LastCluster - W.FirstCluster + 1;
10779 
10780   BranchProbabilityInfo *BPI = FuncInfo.BPI;
10781 
10782   if (Size == 2 && W.MBB == SwitchMBB) {
10783     // If any two of the cases has the same destination, and if one value
10784     // is the same as the other, but has one bit unset that the other has set,
10785     // use bit manipulation to do two compares at once.  For example:
10786     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
10787     // TODO: This could be extended to merge any 2 cases in switches with 3
10788     // cases.
10789     // TODO: Handle cases where W.CaseBB != SwitchBB.
10790     CaseCluster &Small = *W.FirstCluster;
10791     CaseCluster &Big = *W.LastCluster;
10792 
10793     if (Small.Low == Small.High && Big.Low == Big.High &&
10794         Small.MBB == Big.MBB) {
10795       const APInt &SmallValue = Small.Low->getValue();
10796       const APInt &BigValue = Big.Low->getValue();
10797 
10798       // Check that there is only one bit different.
10799       APInt CommonBit = BigValue ^ SmallValue;
10800       if (CommonBit.isPowerOf2()) {
10801         SDValue CondLHS = getValue(Cond);
10802         EVT VT = CondLHS.getValueType();
10803         SDLoc DL = getCurSDLoc();
10804 
10805         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
10806                                  DAG.getConstant(CommonBit, DL, VT));
10807         SDValue Cond = DAG.getSetCC(
10808             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
10809             ISD::SETEQ);
10810 
10811         // Update successor info.
10812         // Both Small and Big will jump to Small.BB, so we sum up the
10813         // probabilities.
10814         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
10815         if (BPI)
10816           addSuccessorWithProb(
10817               SwitchMBB, DefaultMBB,
10818               // The default destination is the first successor in IR.
10819               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
10820         else
10821           addSuccessorWithProb(SwitchMBB, DefaultMBB);
10822 
10823         // Insert the true branch.
10824         SDValue BrCond =
10825             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
10826                         DAG.getBasicBlock(Small.MBB));
10827         // Insert the false branch.
10828         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
10829                              DAG.getBasicBlock(DefaultMBB));
10830 
10831         DAG.setRoot(BrCond);
10832         return;
10833       }
10834     }
10835   }
10836 
10837   if (TM.getOptLevel() != CodeGenOpt::None) {
10838     // Here, we order cases by probability so the most likely case will be
10839     // checked first. However, two clusters can have the same probability in
10840     // which case their relative ordering is non-deterministic. So we use Low
10841     // as a tie-breaker as clusters are guaranteed to never overlap.
10842     llvm::sort(W.FirstCluster, W.LastCluster + 1,
10843                [](const CaseCluster &a, const CaseCluster &b) {
10844       return a.Prob != b.Prob ?
10845              a.Prob > b.Prob :
10846              a.Low->getValue().slt(b.Low->getValue());
10847     });
10848 
10849     // Rearrange the case blocks so that the last one falls through if possible
10850     // without changing the order of probabilities.
10851     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
10852       --I;
10853       if (I->Prob > W.LastCluster->Prob)
10854         break;
10855       if (I->Kind == CC_Range && I->MBB == NextMBB) {
10856         std::swap(*I, *W.LastCluster);
10857         break;
10858       }
10859     }
10860   }
10861 
10862   // Compute total probability.
10863   BranchProbability DefaultProb = W.DefaultProb;
10864   BranchProbability UnhandledProbs = DefaultProb;
10865   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
10866     UnhandledProbs += I->Prob;
10867 
10868   MachineBasicBlock *CurMBB = W.MBB;
10869   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
10870     bool FallthroughUnreachable = false;
10871     MachineBasicBlock *Fallthrough;
10872     if (I == W.LastCluster) {
10873       // For the last cluster, fall through to the default destination.
10874       Fallthrough = DefaultMBB;
10875       FallthroughUnreachable = isa<UnreachableInst>(
10876           DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
10877     } else {
10878       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
10879       CurMF->insert(BBI, Fallthrough);
10880       // Put Cond in a virtual register to make it available from the new blocks.
10881       ExportFromCurrentBlock(Cond);
10882     }
10883     UnhandledProbs -= I->Prob;
10884 
10885     switch (I->Kind) {
10886       case CC_JumpTable: {
10887         // FIXME: Optimize away range check based on pivot comparisons.
10888         JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
10889         SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
10890 
10891         // The jump block hasn't been inserted yet; insert it here.
10892         MachineBasicBlock *JumpMBB = JT->MBB;
10893         CurMF->insert(BBI, JumpMBB);
10894 
10895         auto JumpProb = I->Prob;
10896         auto FallthroughProb = UnhandledProbs;
10897 
10898         // If the default statement is a target of the jump table, we evenly
10899         // distribute the default probability to successors of CurMBB. Also
10900         // update the probability on the edge from JumpMBB to Fallthrough.
10901         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
10902                                               SE = JumpMBB->succ_end();
10903              SI != SE; ++SI) {
10904           if (*SI == DefaultMBB) {
10905             JumpProb += DefaultProb / 2;
10906             FallthroughProb -= DefaultProb / 2;
10907             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
10908             JumpMBB->normalizeSuccProbs();
10909             break;
10910           }
10911         }
10912 
10913         if (FallthroughUnreachable)
10914           JTH->FallthroughUnreachable = true;
10915 
10916         if (!JTH->FallthroughUnreachable)
10917           addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
10918         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
10919         CurMBB->normalizeSuccProbs();
10920 
10921         // The jump table header will be inserted in our current block, do the
10922         // range check, and fall through to our fallthrough block.
10923         JTH->HeaderBB = CurMBB;
10924         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
10925 
10926         // If we're in the right place, emit the jump table header right now.
10927         if (CurMBB == SwitchMBB) {
10928           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
10929           JTH->Emitted = true;
10930         }
10931         break;
10932       }
10933       case CC_BitTests: {
10934         // FIXME: Optimize away range check based on pivot comparisons.
10935         BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
10936 
10937         // The bit test blocks haven't been inserted yet; insert them here.
10938         for (BitTestCase &BTC : BTB->Cases)
10939           CurMF->insert(BBI, BTC.ThisBB);
10940 
10941         // Fill in fields of the BitTestBlock.
10942         BTB->Parent = CurMBB;
10943         BTB->Default = Fallthrough;
10944 
10945         BTB->DefaultProb = UnhandledProbs;
10946         // If the cases in bit test don't form a contiguous range, we evenly
10947         // distribute the probability on the edge to Fallthrough to two
10948         // successors of CurMBB.
10949         if (!BTB->ContiguousRange) {
10950           BTB->Prob += DefaultProb / 2;
10951           BTB->DefaultProb -= DefaultProb / 2;
10952         }
10953 
10954         if (FallthroughUnreachable)
10955           BTB->FallthroughUnreachable = true;
10956 
10957         // If we're in the right place, emit the bit test header right now.
10958         if (CurMBB == SwitchMBB) {
10959           visitBitTestHeader(*BTB, SwitchMBB);
10960           BTB->Emitted = true;
10961         }
10962         break;
10963       }
10964       case CC_Range: {
10965         const Value *RHS, *LHS, *MHS;
10966         ISD::CondCode CC;
10967         if (I->Low == I->High) {
10968           // Check Cond == I->Low.
10969           CC = ISD::SETEQ;
10970           LHS = Cond;
10971           RHS=I->Low;
10972           MHS = nullptr;
10973         } else {
10974           // Check I->Low <= Cond <= I->High.
10975           CC = ISD::SETLE;
10976           LHS = I->Low;
10977           MHS = Cond;
10978           RHS = I->High;
10979         }
10980 
10981         // If Fallthrough is unreachable, fold away the comparison.
10982         if (FallthroughUnreachable)
10983           CC = ISD::SETTRUE;
10984 
10985         // The false probability is the sum of all unhandled cases.
10986         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
10987                      getCurSDLoc(), I->Prob, UnhandledProbs);
10988 
10989         if (CurMBB == SwitchMBB)
10990           visitSwitchCase(CB, SwitchMBB);
10991         else
10992           SL->SwitchCases.push_back(CB);
10993 
10994         break;
10995       }
10996     }
10997     CurMBB = Fallthrough;
10998   }
10999 }
11000 
11001 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
11002                                               CaseClusterIt First,
11003                                               CaseClusterIt Last) {
11004   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
11005     if (X.Prob != CC.Prob)
11006       return X.Prob > CC.Prob;
11007 
11008     // Ties are broken by comparing the case value.
11009     return X.Low->getValue().slt(CC.Low->getValue());
11010   });
11011 }
11012 
11013 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11014                                         const SwitchWorkListItem &W,
11015                                         Value *Cond,
11016                                         MachineBasicBlock *SwitchMBB) {
11017   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11018          "Clusters not sorted?");
11019 
11020   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11021 
11022   // Balance the tree based on branch probabilities to create a near-optimal (in
11023   // terms of search time given key frequency) binary search tree. See e.g. Kurt
11024   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
11025   CaseClusterIt LastLeft = W.FirstCluster;
11026   CaseClusterIt FirstRight = W.LastCluster;
11027   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
11028   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
11029 
11030   // Move LastLeft and FirstRight towards each other from opposite directions to
11031   // find a partitioning of the clusters which balances the probability on both
11032   // sides. If LeftProb and RightProb are equal, alternate which side is
11033   // taken to ensure 0-probability nodes are distributed evenly.
11034   unsigned I = 0;
11035   while (LastLeft + 1 < FirstRight) {
11036     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
11037       LeftProb += (++LastLeft)->Prob;
11038     else
11039       RightProb += (--FirstRight)->Prob;
11040     I++;
11041   }
11042 
11043   while (true) {
11044     // Our binary search tree differs from a typical BST in that ours can have up
11045     // to three values in each leaf. The pivot selection above doesn't take that
11046     // into account, which means the tree might require more nodes and be less
11047     // efficient. We compensate for this here.
11048 
11049     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
11050     unsigned NumRight = W.LastCluster - FirstRight + 1;
11051 
11052     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
11053       // If one side has less than 3 clusters, and the other has more than 3,
11054       // consider taking a cluster from the other side.
11055 
11056       if (NumLeft < NumRight) {
11057         // Consider moving the first cluster on the right to the left side.
11058         CaseCluster &CC = *FirstRight;
11059         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11060         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11061         if (LeftSideRank <= RightSideRank) {
11062           // Moving the cluster to the left does not demote it.
11063           ++LastLeft;
11064           ++FirstRight;
11065           continue;
11066         }
11067       } else {
11068         assert(NumRight < NumLeft);
11069         // Consider moving the last element on the left to the right side.
11070         CaseCluster &CC = *LastLeft;
11071         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11072         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11073         if (RightSideRank <= LeftSideRank) {
11074           // Moving the cluster to the right does not demot it.
11075           --LastLeft;
11076           --FirstRight;
11077           continue;
11078         }
11079       }
11080     }
11081     break;
11082   }
11083 
11084   assert(LastLeft + 1 == FirstRight);
11085   assert(LastLeft >= W.FirstCluster);
11086   assert(FirstRight <= W.LastCluster);
11087 
11088   // Use the first element on the right as pivot since we will make less-than
11089   // comparisons against it.
11090   CaseClusterIt PivotCluster = FirstRight;
11091   assert(PivotCluster > W.FirstCluster);
11092   assert(PivotCluster <= W.LastCluster);
11093 
11094   CaseClusterIt FirstLeft = W.FirstCluster;
11095   CaseClusterIt LastRight = W.LastCluster;
11096 
11097   const ConstantInt *Pivot = PivotCluster->Low;
11098 
11099   // New blocks will be inserted immediately after the current one.
11100   MachineFunction::iterator BBI(W.MBB);
11101   ++BBI;
11102 
11103   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11104   // we can branch to its destination directly if it's squeezed exactly in
11105   // between the known lower bound and Pivot - 1.
11106   MachineBasicBlock *LeftMBB;
11107   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11108       FirstLeft->Low == W.GE &&
11109       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11110     LeftMBB = FirstLeft->MBB;
11111   } else {
11112     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11113     FuncInfo.MF->insert(BBI, LeftMBB);
11114     WorkList.push_back(
11115         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11116     // Put Cond in a virtual register to make it available from the new blocks.
11117     ExportFromCurrentBlock(Cond);
11118   }
11119 
11120   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11121   // single cluster, RHS.Low == Pivot, and we can branch to its destination
11122   // directly if RHS.High equals the current upper bound.
11123   MachineBasicBlock *RightMBB;
11124   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11125       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11126     RightMBB = FirstRight->MBB;
11127   } else {
11128     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11129     FuncInfo.MF->insert(BBI, RightMBB);
11130     WorkList.push_back(
11131         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11132     // Put Cond in a virtual register to make it available from the new blocks.
11133     ExportFromCurrentBlock(Cond);
11134   }
11135 
11136   // Create the CaseBlock record that will be used to lower the branch.
11137   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11138                getCurSDLoc(), LeftProb, RightProb);
11139 
11140   if (W.MBB == SwitchMBB)
11141     visitSwitchCase(CB, SwitchMBB);
11142   else
11143     SL->SwitchCases.push_back(CB);
11144 }
11145 
11146 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11147 // from the swith statement.
11148 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
11149                                             BranchProbability PeeledCaseProb) {
11150   if (PeeledCaseProb == BranchProbability::getOne())
11151     return BranchProbability::getZero();
11152   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11153 
11154   uint32_t Numerator = CaseProb.getNumerator();
11155   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11156   return BranchProbability(Numerator, std::max(Numerator, Denominator));
11157 }
11158 
11159 // Try to peel the top probability case if it exceeds the threshold.
11160 // Return current MachineBasicBlock for the switch statement if the peeling
11161 // does not occur.
11162 // If the peeling is performed, return the newly created MachineBasicBlock
11163 // for the peeled switch statement. Also update Clusters to remove the peeled
11164 // case. PeeledCaseProb is the BranchProbability for the peeled case.
11165 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11166     const SwitchInst &SI, CaseClusterVector &Clusters,
11167     BranchProbability &PeeledCaseProb) {
11168   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11169   // Don't perform if there is only one cluster or optimizing for size.
11170   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11171       TM.getOptLevel() == CodeGenOpt::None ||
11172       SwitchMBB->getParent()->getFunction().hasMinSize())
11173     return SwitchMBB;
11174 
11175   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
11176   unsigned PeeledCaseIndex = 0;
11177   bool SwitchPeeled = false;
11178   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11179     CaseCluster &CC = Clusters[Index];
11180     if (CC.Prob < TopCaseProb)
11181       continue;
11182     TopCaseProb = CC.Prob;
11183     PeeledCaseIndex = Index;
11184     SwitchPeeled = true;
11185   }
11186   if (!SwitchPeeled)
11187     return SwitchMBB;
11188 
11189   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11190                     << TopCaseProb << "\n");
11191 
11192   // Record the MBB for the peeled switch statement.
11193   MachineFunction::iterator BBI(SwitchMBB);
11194   ++BBI;
11195   MachineBasicBlock *PeeledSwitchMBB =
11196       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
11197   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
11198 
11199   ExportFromCurrentBlock(SI.getCondition());
11200   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
11201   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
11202                           nullptr,   nullptr,      TopCaseProb.getCompl()};
11203   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
11204 
11205   Clusters.erase(PeeledCaseIt);
11206   for (CaseCluster &CC : Clusters) {
11207     LLVM_DEBUG(
11208         dbgs() << "Scale the probablity for one cluster, before scaling: "
11209                << CC.Prob << "\n");
11210     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
11211     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
11212   }
11213   PeeledCaseProb = TopCaseProb;
11214   return PeeledSwitchMBB;
11215 }
11216 
11217 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
11218   // Extract cases from the switch.
11219   BranchProbabilityInfo *BPI = FuncInfo.BPI;
11220   CaseClusterVector Clusters;
11221   Clusters.reserve(SI.getNumCases());
11222   for (auto I : SI.cases()) {
11223     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
11224     const ConstantInt *CaseVal = I.getCaseValue();
11225     BranchProbability Prob =
11226         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
11227             : BranchProbability(1, SI.getNumCases() + 1);
11228     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
11229   }
11230 
11231   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
11232 
11233   // Cluster adjacent cases with the same destination. We do this at all
11234   // optimization levels because it's cheap to do and will make codegen faster
11235   // if there are many clusters.
11236   sortAndRangeify(Clusters);
11237 
11238   // The branch probablity of the peeled case.
11239   BranchProbability PeeledCaseProb = BranchProbability::getZero();
11240   MachineBasicBlock *PeeledSwitchMBB =
11241       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
11242 
11243   // If there is only the default destination, jump there directly.
11244   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11245   if (Clusters.empty()) {
11246     assert(PeeledSwitchMBB == SwitchMBB);
11247     SwitchMBB->addSuccessor(DefaultMBB);
11248     if (DefaultMBB != NextBlock(SwitchMBB)) {
11249       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
11250                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
11251     }
11252     return;
11253   }
11254 
11255   SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI());
11256   SL->findBitTestClusters(Clusters, &SI);
11257 
11258   LLVM_DEBUG({
11259     dbgs() << "Case clusters: ";
11260     for (const CaseCluster &C : Clusters) {
11261       if (C.Kind == CC_JumpTable)
11262         dbgs() << "JT:";
11263       if (C.Kind == CC_BitTests)
11264         dbgs() << "BT:";
11265 
11266       C.Low->getValue().print(dbgs(), true);
11267       if (C.Low != C.High) {
11268         dbgs() << '-';
11269         C.High->getValue().print(dbgs(), true);
11270       }
11271       dbgs() << ' ';
11272     }
11273     dbgs() << '\n';
11274   });
11275 
11276   assert(!Clusters.empty());
11277   SwitchWorkList WorkList;
11278   CaseClusterIt First = Clusters.begin();
11279   CaseClusterIt Last = Clusters.end() - 1;
11280   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
11281   // Scale the branchprobability for DefaultMBB if the peel occurs and
11282   // DefaultMBB is not replaced.
11283   if (PeeledCaseProb != BranchProbability::getZero() &&
11284       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
11285     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
11286   WorkList.push_back(
11287       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
11288 
11289   while (!WorkList.empty()) {
11290     SwitchWorkListItem W = WorkList.pop_back_val();
11291     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
11292 
11293     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
11294         !DefaultMBB->getParent()->getFunction().hasMinSize()) {
11295       // For optimized builds, lower large range as a balanced binary tree.
11296       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
11297       continue;
11298     }
11299 
11300     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
11301   }
11302 }
11303 
11304 void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
11305   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11306   auto DL = getCurSDLoc();
11307   EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11308   setValue(&I, DAG.getStepVector(DL, ResultVT));
11309 }
11310 
11311 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
11312   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11313   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11314 
11315   SDLoc DL = getCurSDLoc();
11316   SDValue V = getValue(I.getOperand(0));
11317   assert(VT == V.getValueType() && "Malformed vector.reverse!");
11318 
11319   if (VT.isScalableVector()) {
11320     setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
11321     return;
11322   }
11323 
11324   // Use VECTOR_SHUFFLE for the fixed-length vector
11325   // to maintain existing behavior.
11326   SmallVector<int, 8> Mask;
11327   unsigned NumElts = VT.getVectorMinNumElements();
11328   for (unsigned i = 0; i != NumElts; ++i)
11329     Mask.push_back(NumElts - 1 - i);
11330 
11331   setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
11332 }
11333 
11334 void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
11335   SmallVector<EVT, 4> ValueVTs;
11336   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
11337                   ValueVTs);
11338   unsigned NumValues = ValueVTs.size();
11339   if (NumValues == 0) return;
11340 
11341   SmallVector<SDValue, 4> Values(NumValues);
11342   SDValue Op = getValue(I.getOperand(0));
11343 
11344   for (unsigned i = 0; i != NumValues; ++i)
11345     Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
11346                             SDValue(Op.getNode(), Op.getResNo() + i));
11347 
11348   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
11349                            DAG.getVTList(ValueVTs), Values));
11350 }
11351 
11352 void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
11353   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11354   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11355 
11356   SDLoc DL = getCurSDLoc();
11357   SDValue V1 = getValue(I.getOperand(0));
11358   SDValue V2 = getValue(I.getOperand(1));
11359   int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
11360 
11361   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
11362   if (VT.isScalableVector()) {
11363     MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
11364     setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
11365                              DAG.getConstant(Imm, DL, IdxVT)));
11366     return;
11367   }
11368 
11369   unsigned NumElts = VT.getVectorNumElements();
11370 
11371   uint64_t Idx = (NumElts + Imm) % NumElts;
11372 
11373   // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
11374   SmallVector<int, 8> Mask;
11375   for (unsigned i = 0; i < NumElts; ++i)
11376     Mask.push_back(Idx + i);
11377   setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
11378 }
11379