xref: /llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision 1b8177232813a048407bbb43eef18d7f72794b35)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/APInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/None.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/ADT/Triple.h"
29 #include "llvm/ADT/Twine.h"
30 #include "llvm/Analysis/AliasAnalysis.h"
31 #include "llvm/Analysis/BranchProbabilityInfo.h"
32 #include "llvm/Analysis/ConstantFolding.h"
33 #include "llvm/Analysis/EHPersonalities.h"
34 #include "llvm/Analysis/Loads.h"
35 #include "llvm/Analysis/MemoryLocation.h"
36 #include "llvm/Analysis/TargetLibraryInfo.h"
37 #include "llvm/Analysis/ValueTracking.h"
38 #include "llvm/Analysis/VectorUtils.h"
39 #include "llvm/CodeGen/Analysis.h"
40 #include "llvm/CodeGen/FunctionLoweringInfo.h"
41 #include "llvm/CodeGen/GCMetadata.h"
42 #include "llvm/CodeGen/ISDOpcodes.h"
43 #include "llvm/CodeGen/MachineBasicBlock.h"
44 #include "llvm/CodeGen/MachineFrameInfo.h"
45 #include "llvm/CodeGen/MachineFunction.h"
46 #include "llvm/CodeGen/MachineInstr.h"
47 #include "llvm/CodeGen/MachineInstrBuilder.h"
48 #include "llvm/CodeGen/MachineJumpTableInfo.h"
49 #include "llvm/CodeGen/MachineMemOperand.h"
50 #include "llvm/CodeGen/MachineModuleInfo.h"
51 #include "llvm/CodeGen/MachineOperand.h"
52 #include "llvm/CodeGen/MachineRegisterInfo.h"
53 #include "llvm/CodeGen/RuntimeLibcalls.h"
54 #include "llvm/CodeGen/SelectionDAG.h"
55 #include "llvm/CodeGen/SelectionDAGNodes.h"
56 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
57 #include "llvm/CodeGen/StackMaps.h"
58 #include "llvm/CodeGen/TargetFrameLowering.h"
59 #include "llvm/CodeGen/TargetInstrInfo.h"
60 #include "llvm/CodeGen/TargetLowering.h"
61 #include "llvm/CodeGen/TargetOpcodes.h"
62 #include "llvm/CodeGen/TargetRegisterInfo.h"
63 #include "llvm/CodeGen/TargetSubtargetInfo.h"
64 #include "llvm/CodeGen/ValueTypes.h"
65 #include "llvm/CodeGen/WinEHFuncInfo.h"
66 #include "llvm/IR/Argument.h"
67 #include "llvm/IR/Attributes.h"
68 #include "llvm/IR/BasicBlock.h"
69 #include "llvm/IR/CFG.h"
70 #include "llvm/IR/CallSite.h"
71 #include "llvm/IR/CallingConv.h"
72 #include "llvm/IR/Constant.h"
73 #include "llvm/IR/ConstantRange.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfoMetadata.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
80 #include "llvm/IR/GetElementPtrTypeIterator.h"
81 #include "llvm/IR/InlineAsm.h"
82 #include "llvm/IR/InstrTypes.h"
83 #include "llvm/IR/Instruction.h"
84 #include "llvm/IR/Instructions.h"
85 #include "llvm/IR/IntrinsicInst.h"
86 #include "llvm/IR/Intrinsics.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Metadata.h"
89 #include "llvm/IR/Module.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Statepoint.h"
93 #include "llvm/IR/Type.h"
94 #include "llvm/IR/User.h"
95 #include "llvm/IR/Value.h"
96 #include "llvm/MC/MCContext.h"
97 #include "llvm/MC/MCSymbol.h"
98 #include "llvm/Support/AtomicOrdering.h"
99 #include "llvm/Support/BranchProbability.h"
100 #include "llvm/Support/Casting.h"
101 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/CommandLine.h"
103 #include "llvm/Support/Compiler.h"
104 #include "llvm/Support/Debug.h"
105 #include "llvm/Support/ErrorHandling.h"
106 #include "llvm/Support/MachineValueType.h"
107 #include "llvm/Support/MathExtras.h"
108 #include "llvm/Support/raw_ostream.h"
109 #include "llvm/Target/TargetIntrinsicInfo.h"
110 #include "llvm/Target/TargetMachine.h"
111 #include "llvm/Target/TargetOptions.h"
112 #include <algorithm>
113 #include <cassert>
114 #include <cstddef>
115 #include <cstdint>
116 #include <cstring>
117 #include <iterator>
118 #include <limits>
119 #include <numeric>
120 #include <tuple>
121 #include <utility>
122 #include <vector>
123 
124 using namespace llvm;
125 using namespace PatternMatch;
126 
127 #define DEBUG_TYPE "isel"
128 
129 /// LimitFloatPrecision - Generate low-precision inline sequences for
130 /// some float libcalls (6, 8 or 12 bits).
131 static unsigned LimitFloatPrecision;
132 
133 static cl::opt<unsigned, true>
134     LimitFPPrecision("limit-float-precision",
135                      cl::desc("Generate low-precision inline sequences "
136                               "for some float libcalls"),
137                      cl::location(LimitFloatPrecision), cl::Hidden,
138                      cl::init(0));
139 
140 static cl::opt<unsigned> SwitchPeelThreshold(
141     "switch-peel-threshold", cl::Hidden, cl::init(66),
142     cl::desc("Set the case probability threshold for peeling the case from a "
143              "switch statement. A value greater than 100 will void this "
144              "optimization"));
145 
146 // Limit the width of DAG chains. This is important in general to prevent
147 // DAG-based analysis from blowing up. For example, alias analysis and
148 // load clustering may not complete in reasonable time. It is difficult to
149 // recognize and avoid this situation within each individual analysis, and
150 // future analyses are likely to have the same behavior. Limiting DAG width is
151 // the safe approach and will be especially important with global DAGs.
152 //
153 // MaxParallelChains default is arbitrarily high to avoid affecting
154 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
155 // sequence over this should have been converted to llvm.memcpy by the
156 // frontend. It is easy to induce this behavior with .ll code such as:
157 // %buffer = alloca [4096 x i8]
158 // %data = load [4096 x i8]* %argPtr
159 // store [4096 x i8] %data, [4096 x i8]* %buffer
160 static const unsigned MaxParallelChains = 64;
161 
162 // Return the calling convention if the Value passed requires ABI mangling as it
163 // is a parameter to a function or a return value from a function which is not
164 // an intrinsic.
165 static Optional<CallingConv::ID> getABIRegCopyCC(const Value *V) {
166   if (auto *R = dyn_cast<ReturnInst>(V))
167     return R->getParent()->getParent()->getCallingConv();
168 
169   if (auto *CI = dyn_cast<CallInst>(V)) {
170     const bool IsInlineAsm = CI->isInlineAsm();
171     const bool IsIndirectFunctionCall =
172         !IsInlineAsm && !CI->getCalledFunction();
173 
174     // It is possible that the call instruction is an inline asm statement or an
175     // indirect function call in which case the return value of
176     // getCalledFunction() would be nullptr.
177     const bool IsInstrinsicCall =
178         !IsInlineAsm && !IsIndirectFunctionCall &&
179         CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
180 
181     if (!IsInlineAsm && !IsInstrinsicCall)
182       return CI->getCallingConv();
183   }
184 
185   return None;
186 }
187 
188 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
189                                       const SDValue *Parts, unsigned NumParts,
190                                       MVT PartVT, EVT ValueVT, const Value *V,
191                                       Optional<CallingConv::ID> CC);
192 
193 /// getCopyFromParts - Create a value that contains the specified legal parts
194 /// combined into the value they represent.  If the parts combine to a type
195 /// larger than ValueVT then AssertOp can be used to specify whether the extra
196 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
197 /// (ISD::AssertSext).
198 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
199                                 const SDValue *Parts, unsigned NumParts,
200                                 MVT PartVT, EVT ValueVT, const Value *V,
201                                 Optional<CallingConv::ID> CC = None,
202                                 Optional<ISD::NodeType> AssertOp = None) {
203   if (ValueVT.isVector())
204     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
205                                   CC);
206 
207   assert(NumParts > 0 && "No parts to assemble!");
208   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
209   SDValue Val = Parts[0];
210 
211   if (NumParts > 1) {
212     // Assemble the value from multiple parts.
213     if (ValueVT.isInteger()) {
214       unsigned PartBits = PartVT.getSizeInBits();
215       unsigned ValueBits = ValueVT.getSizeInBits();
216 
217       // Assemble the power of 2 part.
218       unsigned RoundParts = NumParts & (NumParts - 1) ?
219         1 << Log2_32(NumParts) : NumParts;
220       unsigned RoundBits = PartBits * RoundParts;
221       EVT RoundVT = RoundBits == ValueBits ?
222         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
223       SDValue Lo, Hi;
224 
225       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
226 
227       if (RoundParts > 2) {
228         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
229                               PartVT, HalfVT, V);
230         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
231                               RoundParts / 2, PartVT, HalfVT, V);
232       } else {
233         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
234         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
235       }
236 
237       if (DAG.getDataLayout().isBigEndian())
238         std::swap(Lo, Hi);
239 
240       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
241 
242       if (RoundParts < NumParts) {
243         // Assemble the trailing non-power-of-2 part.
244         unsigned OddParts = NumParts - RoundParts;
245         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
246         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
247                               OddVT, V, CC);
248 
249         // Combine the round and odd parts.
250         Lo = Val;
251         if (DAG.getDataLayout().isBigEndian())
252           std::swap(Lo, Hi);
253         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
254         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
255         Hi =
256             DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
257                         DAG.getConstant(Lo.getValueSizeInBits(), DL,
258                                         TLI.getPointerTy(DAG.getDataLayout())));
259         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
260         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
261       }
262     } else if (PartVT.isFloatingPoint()) {
263       // FP split into multiple FP parts (for ppcf128)
264       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
265              "Unexpected split");
266       SDValue Lo, Hi;
267       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
268       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
269       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
270         std::swap(Lo, Hi);
271       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
272     } else {
273       // FP split into integer parts (soft fp)
274       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
275              !PartVT.isVector() && "Unexpected split");
276       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
277       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
278     }
279   }
280 
281   // There is now one part, held in Val.  Correct it to match ValueVT.
282   // PartEVT is the type of the register class that holds the value.
283   // ValueVT is the type of the inline asm operation.
284   EVT PartEVT = Val.getValueType();
285 
286   if (PartEVT == ValueVT)
287     return Val;
288 
289   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
290       ValueVT.bitsLT(PartEVT)) {
291     // For an FP value in an integer part, we need to truncate to the right
292     // width first.
293     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
294     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
295   }
296 
297   // Handle types that have the same size.
298   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
299     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
300 
301   // Handle types with different sizes.
302   if (PartEVT.isInteger() && ValueVT.isInteger()) {
303     if (ValueVT.bitsLT(PartEVT)) {
304       // For a truncate, see if we have any information to
305       // indicate whether the truncated bits will always be
306       // zero or sign-extension.
307       if (AssertOp.hasValue())
308         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
309                           DAG.getValueType(ValueVT));
310       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
311     }
312     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
313   }
314 
315   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
316     // FP_ROUND's are always exact here.
317     if (ValueVT.bitsLT(Val.getValueType()))
318       return DAG.getNode(
319           ISD::FP_ROUND, DL, ValueVT, Val,
320           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
321 
322     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
323   }
324 
325   llvm_unreachable("Unknown mismatch!");
326 }
327 
328 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
329                                               const Twine &ErrMsg) {
330   const Instruction *I = dyn_cast_or_null<Instruction>(V);
331   if (!V)
332     return Ctx.emitError(ErrMsg);
333 
334   const char *AsmError = ", possible invalid constraint for vector type";
335   if (const CallInst *CI = dyn_cast<CallInst>(I))
336     if (isa<InlineAsm>(CI->getCalledValue()))
337       return Ctx.emitError(I, ErrMsg + AsmError);
338 
339   return Ctx.emitError(I, ErrMsg);
340 }
341 
342 /// getCopyFromPartsVector - Create a value that contains the specified legal
343 /// parts combined into the value they represent.  If the parts combine to a
344 /// type larger than ValueVT then AssertOp can be used to specify whether the
345 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
346 /// ValueVT (ISD::AssertSext).
347 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
348                                       const SDValue *Parts, unsigned NumParts,
349                                       MVT PartVT, EVT ValueVT, const Value *V,
350                                       Optional<CallingConv::ID> CallConv) {
351   assert(ValueVT.isVector() && "Not a vector value");
352   assert(NumParts > 0 && "No parts to assemble!");
353   const bool IsABIRegCopy = CallConv.hasValue();
354 
355   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
356   SDValue Val = Parts[0];
357 
358   // Handle a multi-element vector.
359   if (NumParts > 1) {
360     EVT IntermediateVT;
361     MVT RegisterVT;
362     unsigned NumIntermediates;
363     unsigned NumRegs;
364 
365     if (IsABIRegCopy) {
366       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
367           *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
368           NumIntermediates, RegisterVT);
369     } else {
370       NumRegs =
371           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
372                                      NumIntermediates, RegisterVT);
373     }
374 
375     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
376     NumParts = NumRegs; // Silence a compiler warning.
377     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
378     assert(RegisterVT.getSizeInBits() ==
379            Parts[0].getSimpleValueType().getSizeInBits() &&
380            "Part type sizes don't match!");
381 
382     // Assemble the parts into intermediate operands.
383     SmallVector<SDValue, 8> Ops(NumIntermediates);
384     if (NumIntermediates == NumParts) {
385       // If the register was not expanded, truncate or copy the value,
386       // as appropriate.
387       for (unsigned i = 0; i != NumParts; ++i)
388         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
389                                   PartVT, IntermediateVT, V);
390     } else if (NumParts > 0) {
391       // If the intermediate type was expanded, build the intermediate
392       // operands from the parts.
393       assert(NumParts % NumIntermediates == 0 &&
394              "Must expand into a divisible number of parts!");
395       unsigned Factor = NumParts / NumIntermediates;
396       for (unsigned i = 0; i != NumIntermediates; ++i)
397         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
398                                   PartVT, IntermediateVT, V);
399     }
400 
401     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
402     // intermediate operands.
403     EVT BuiltVectorTy =
404         EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
405                          (IntermediateVT.isVector()
406                               ? IntermediateVT.getVectorNumElements() * NumParts
407                               : NumIntermediates));
408     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
409                                                 : ISD::BUILD_VECTOR,
410                       DL, BuiltVectorTy, Ops);
411   }
412 
413   // There is now one part, held in Val.  Correct it to match ValueVT.
414   EVT PartEVT = Val.getValueType();
415 
416   if (PartEVT == ValueVT)
417     return Val;
418 
419   if (PartEVT.isVector()) {
420     // If the element type of the source/dest vectors are the same, but the
421     // parts vector has more elements than the value vector, then we have a
422     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
423     // elements we want.
424     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
425       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
426              "Cannot narrow, it would be a lossy transformation");
427       return DAG.getNode(
428           ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
429           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
430     }
431 
432     // Vector/Vector bitcast.
433     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
434       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435 
436     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
437       "Cannot handle this kind of promotion");
438     // Promoted vector extract
439     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
440 
441   }
442 
443   // Trivial bitcast if the types are the same size and the destination
444   // vector type is legal.
445   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
446       TLI.isTypeLegal(ValueVT))
447     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
448 
449   if (ValueVT.getVectorNumElements() != 1) {
450      // Certain ABIs require that vectors are passed as integers. For vectors
451      // are the same size, this is an obvious bitcast.
452      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
453        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
454      } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
455        // Bitcast Val back the original type and extract the corresponding
456        // vector we want.
457        unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
458        EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
459                                            ValueVT.getVectorElementType(), Elts);
460        Val = DAG.getBitcast(WiderVecType, Val);
461        return DAG.getNode(
462            ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
463            DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
464      }
465 
466      diagnosePossiblyInvalidConstraint(
467          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
468      return DAG.getUNDEF(ValueVT);
469   }
470 
471   // Handle cases such as i8 -> <1 x i1>
472   EVT ValueSVT = ValueVT.getVectorElementType();
473   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
474     Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
475                                     : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
476 
477   return DAG.getBuildVector(ValueVT, DL, Val);
478 }
479 
480 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
481                                  SDValue Val, SDValue *Parts, unsigned NumParts,
482                                  MVT PartVT, const Value *V,
483                                  Optional<CallingConv::ID> CallConv);
484 
485 /// getCopyToParts - Create a series of nodes that contain the specified value
486 /// split into legal parts.  If the parts contain more bits than Val, then, for
487 /// integers, ExtendKind can be used to specify how to generate the extra bits.
488 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
489                            SDValue *Parts, unsigned NumParts, MVT PartVT,
490                            const Value *V,
491                            Optional<CallingConv::ID> CallConv = None,
492                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
493   EVT ValueVT = Val.getValueType();
494 
495   // Handle the vector case separately.
496   if (ValueVT.isVector())
497     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
498                                 CallConv);
499 
500   unsigned PartBits = PartVT.getSizeInBits();
501   unsigned OrigNumParts = NumParts;
502   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
503          "Copying to an illegal type!");
504 
505   if (NumParts == 0)
506     return;
507 
508   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
509   EVT PartEVT = PartVT;
510   if (PartEVT == ValueVT) {
511     assert(NumParts == 1 && "No-op copy with multiple parts!");
512     Parts[0] = Val;
513     return;
514   }
515 
516   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
517     // If the parts cover more bits than the value has, promote the value.
518     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
519       assert(NumParts == 1 && "Do not know what to promote to!");
520       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
521     } else {
522       if (ValueVT.isFloatingPoint()) {
523         // FP values need to be bitcast, then extended if they are being put
524         // into a larger container.
525         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
526         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
527       }
528       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529              ValueVT.isInteger() &&
530              "Unknown mismatch!");
531       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
533       if (PartVT == MVT::x86mmx)
534         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535     }
536   } else if (PartBits == ValueVT.getSizeInBits()) {
537     // Different types of the same size.
538     assert(NumParts == 1 && PartEVT != ValueVT);
539     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
540   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
541     // If the parts cover less bits than value has, truncate the value.
542     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
543            ValueVT.isInteger() &&
544            "Unknown mismatch!");
545     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
546     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
547     if (PartVT == MVT::x86mmx)
548       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
549   }
550 
551   // The value may have changed - recompute ValueVT.
552   ValueVT = Val.getValueType();
553   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
554          "Failed to tile the value with PartVT!");
555 
556   if (NumParts == 1) {
557     if (PartEVT != ValueVT) {
558       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
559                                         "scalar-to-vector conversion failed");
560       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561     }
562 
563     Parts[0] = Val;
564     return;
565   }
566 
567   // Expand the value into multiple parts.
568   if (NumParts & (NumParts - 1)) {
569     // The number of parts is not a power of 2.  Split off and copy the tail.
570     assert(PartVT.isInteger() && ValueVT.isInteger() &&
571            "Do not know what to expand to!");
572     unsigned RoundParts = 1 << Log2_32(NumParts);
573     unsigned RoundBits = RoundParts * PartBits;
574     unsigned OddParts = NumParts - RoundParts;
575     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
576                                  DAG.getIntPtrConstant(RoundBits, DL));
577     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
578                    CallConv);
579 
580     if (DAG.getDataLayout().isBigEndian())
581       // The odd parts were reversed by getCopyToParts - unreverse them.
582       std::reverse(Parts + RoundParts, Parts + NumParts);
583 
584     NumParts = RoundParts;
585     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
586     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
587   }
588 
589   // The number of parts is a power of 2.  Repeatedly bisect the value using
590   // EXTRACT_ELEMENT.
591   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
592                          EVT::getIntegerVT(*DAG.getContext(),
593                                            ValueVT.getSizeInBits()),
594                          Val);
595 
596   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
597     for (unsigned i = 0; i < NumParts; i += StepSize) {
598       unsigned ThisBits = StepSize * PartBits / 2;
599       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
600       SDValue &Part0 = Parts[i];
601       SDValue &Part1 = Parts[i+StepSize/2];
602 
603       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
605       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
606                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
607 
608       if (ThisBits == PartBits && ThisVT != PartVT) {
609         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
610         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
611       }
612     }
613   }
614 
615   if (DAG.getDataLayout().isBigEndian())
616     std::reverse(Parts, Parts + OrigNumParts);
617 }
618 
619 static SDValue widenVectorToPartType(SelectionDAG &DAG,
620                                      SDValue Val, const SDLoc &DL, EVT PartVT) {
621   if (!PartVT.isVector())
622     return SDValue();
623 
624   EVT ValueVT = Val.getValueType();
625   unsigned PartNumElts = PartVT.getVectorNumElements();
626   unsigned ValueNumElts = ValueVT.getVectorNumElements();
627   if (PartNumElts > ValueNumElts &&
628       PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
629     EVT ElementVT = PartVT.getVectorElementType();
630     // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
631     // undef elements.
632     SmallVector<SDValue, 16> Ops;
633     DAG.ExtractVectorElements(Val, Ops);
634     SDValue EltUndef = DAG.getUNDEF(ElementVT);
635     for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
636       Ops.push_back(EltUndef);
637 
638     // FIXME: Use CONCAT for 2x -> 4x.
639     return DAG.getBuildVector(PartVT, DL, Ops);
640   }
641 
642   return SDValue();
643 }
644 
645 /// getCopyToPartsVector - Create a series of nodes that contain the specified
646 /// value split into legal parts.
647 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
648                                  SDValue Val, SDValue *Parts, unsigned NumParts,
649                                  MVT PartVT, const Value *V,
650                                  Optional<CallingConv::ID> CallConv) {
651   EVT ValueVT = Val.getValueType();
652   assert(ValueVT.isVector() && "Not a vector");
653   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
654   const bool IsABIRegCopy = CallConv.hasValue();
655 
656   if (NumParts == 1) {
657     EVT PartEVT = PartVT;
658     if (PartEVT == ValueVT) {
659       // Nothing to do.
660     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
661       // Bitconvert vector->vector case.
662       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
663     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
664       Val = Widened;
665     } else if (PartVT.isVector() &&
666                PartEVT.getVectorElementType().bitsGE(
667                  ValueVT.getVectorElementType()) &&
668                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
669 
670       // Promoted vector extract
671       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
672     } else {
673       if (ValueVT.getVectorNumElements() == 1) {
674         Val = DAG.getNode(
675             ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
676             DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
677       } else {
678         assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
679                "lossy conversion of vector to scalar type");
680         EVT IntermediateType =
681             EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
682         Val = DAG.getBitcast(IntermediateType, Val);
683         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
684       }
685     }
686 
687     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
688     Parts[0] = Val;
689     return;
690   }
691 
692   // Handle a multi-element vector.
693   EVT IntermediateVT;
694   MVT RegisterVT;
695   unsigned NumIntermediates;
696   unsigned NumRegs;
697   if (IsABIRegCopy) {
698     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
699         *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
700         NumIntermediates, RegisterVT);
701   } else {
702     NumRegs =
703         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
704                                    NumIntermediates, RegisterVT);
705   }
706 
707   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
708   NumParts = NumRegs; // Silence a compiler warning.
709   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
710 
711   unsigned IntermediateNumElts = IntermediateVT.isVector() ?
712     IntermediateVT.getVectorNumElements() : 1;
713 
714   // Convert the vector to the appropiate type if necessary.
715   unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
716 
717   EVT BuiltVectorTy = EVT::getVectorVT(
718       *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
719   MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
720   if (ValueVT != BuiltVectorTy) {
721     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
722       Val = Widened;
723 
724     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
725   }
726 
727   // Split the vector into intermediate operands.
728   SmallVector<SDValue, 8> Ops(NumIntermediates);
729   for (unsigned i = 0; i != NumIntermediates; ++i) {
730     if (IntermediateVT.isVector()) {
731       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
732                            DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
733     } else {
734       Ops[i] = DAG.getNode(
735           ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
736           DAG.getConstant(i, DL, IdxVT));
737     }
738   }
739 
740   // Split the intermediate operands into legal parts.
741   if (NumParts == NumIntermediates) {
742     // If the register was not expanded, promote or copy the value,
743     // as appropriate.
744     for (unsigned i = 0; i != NumParts; ++i)
745       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
746   } else if (NumParts > 0) {
747     // If the intermediate type was expanded, split each the value into
748     // legal parts.
749     assert(NumIntermediates != 0 && "division by zero");
750     assert(NumParts % NumIntermediates == 0 &&
751            "Must expand into a divisible number of parts!");
752     unsigned Factor = NumParts / NumIntermediates;
753     for (unsigned i = 0; i != NumIntermediates; ++i)
754       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
755                      CallConv);
756   }
757 }
758 
759 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
760                            EVT valuevt, Optional<CallingConv::ID> CC)
761     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
762       RegCount(1, regs.size()), CallConv(CC) {}
763 
764 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
765                            const DataLayout &DL, unsigned Reg, Type *Ty,
766                            Optional<CallingConv::ID> CC) {
767   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
768 
769   CallConv = CC;
770 
771   for (EVT ValueVT : ValueVTs) {
772     unsigned NumRegs =
773         isABIMangled()
774             ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
775             : TLI.getNumRegisters(Context, ValueVT);
776     MVT RegisterVT =
777         isABIMangled()
778             ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
779             : TLI.getRegisterType(Context, ValueVT);
780     for (unsigned i = 0; i != NumRegs; ++i)
781       Regs.push_back(Reg + i);
782     RegVTs.push_back(RegisterVT);
783     RegCount.push_back(NumRegs);
784     Reg += NumRegs;
785   }
786 }
787 
788 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
789                                       FunctionLoweringInfo &FuncInfo,
790                                       const SDLoc &dl, SDValue &Chain,
791                                       SDValue *Flag, const Value *V) const {
792   // A Value with type {} or [0 x %t] needs no registers.
793   if (ValueVTs.empty())
794     return SDValue();
795 
796   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
797 
798   // Assemble the legal parts into the final values.
799   SmallVector<SDValue, 4> Values(ValueVTs.size());
800   SmallVector<SDValue, 8> Parts;
801   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
802     // Copy the legal parts from the registers.
803     EVT ValueVT = ValueVTs[Value];
804     unsigned NumRegs = RegCount[Value];
805     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
806                                           *DAG.getContext(),
807                                           CallConv.getValue(), RegVTs[Value])
808                                     : RegVTs[Value];
809 
810     Parts.resize(NumRegs);
811     for (unsigned i = 0; i != NumRegs; ++i) {
812       SDValue P;
813       if (!Flag) {
814         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
815       } else {
816         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
817         *Flag = P.getValue(2);
818       }
819 
820       Chain = P.getValue(1);
821       Parts[i] = P;
822 
823       // If the source register was virtual and if we know something about it,
824       // add an assert node.
825       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
826           !RegisterVT.isInteger())
827         continue;
828 
829       const FunctionLoweringInfo::LiveOutInfo *LOI =
830         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
831       if (!LOI)
832         continue;
833 
834       unsigned RegSize = RegisterVT.getScalarSizeInBits();
835       unsigned NumSignBits = LOI->NumSignBits;
836       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
837 
838       if (NumZeroBits == RegSize) {
839         // The current value is a zero.
840         // Explicitly express that as it would be easier for
841         // optimizations to kick in.
842         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
843         continue;
844       }
845 
846       // FIXME: We capture more information than the dag can represent.  For
847       // now, just use the tightest assertzext/assertsext possible.
848       bool isSExt;
849       EVT FromVT(MVT::Other);
850       if (NumZeroBits) {
851         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
852         isSExt = false;
853       } else if (NumSignBits > 1) {
854         FromVT =
855             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
856         isSExt = true;
857       } else {
858         continue;
859       }
860       // Add an assertion node.
861       assert(FromVT != MVT::Other);
862       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
863                              RegisterVT, P, DAG.getValueType(FromVT));
864     }
865 
866     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
867                                      RegisterVT, ValueVT, V, CallConv);
868     Part += NumRegs;
869     Parts.clear();
870   }
871 
872   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
873 }
874 
875 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
876                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
877                                  const Value *V,
878                                  ISD::NodeType PreferredExtendType) const {
879   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
880   ISD::NodeType ExtendKind = PreferredExtendType;
881 
882   // Get the list of the values's legal parts.
883   unsigned NumRegs = Regs.size();
884   SmallVector<SDValue, 8> Parts(NumRegs);
885   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
886     unsigned NumParts = RegCount[Value];
887 
888     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
889                                           *DAG.getContext(),
890                                           CallConv.getValue(), RegVTs[Value])
891                                     : RegVTs[Value];
892 
893     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
894       ExtendKind = ISD::ZERO_EXTEND;
895 
896     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
897                    NumParts, RegisterVT, V, CallConv, ExtendKind);
898     Part += NumParts;
899   }
900 
901   // Copy the parts into the registers.
902   SmallVector<SDValue, 8> Chains(NumRegs);
903   for (unsigned i = 0; i != NumRegs; ++i) {
904     SDValue Part;
905     if (!Flag) {
906       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
907     } else {
908       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
909       *Flag = Part.getValue(1);
910     }
911 
912     Chains[i] = Part.getValue(0);
913   }
914 
915   if (NumRegs == 1 || Flag)
916     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
917     // flagged to it. That is the CopyToReg nodes and the user are considered
918     // a single scheduling unit. If we create a TokenFactor and return it as
919     // chain, then the TokenFactor is both a predecessor (operand) of the
920     // user as well as a successor (the TF operands are flagged to the user).
921     // c1, f1 = CopyToReg
922     // c2, f2 = CopyToReg
923     // c3     = TokenFactor c1, c2
924     // ...
925     //        = op c3, ..., f2
926     Chain = Chains[NumRegs-1];
927   else
928     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
929 }
930 
931 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
932                                         unsigned MatchingIdx, const SDLoc &dl,
933                                         SelectionDAG &DAG,
934                                         std::vector<SDValue> &Ops) const {
935   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
936 
937   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
938   if (HasMatching)
939     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
940   else if (!Regs.empty() &&
941            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
942     // Put the register class of the virtual registers in the flag word.  That
943     // way, later passes can recompute register class constraints for inline
944     // assembly as well as normal instructions.
945     // Don't do this for tied operands that can use the regclass information
946     // from the def.
947     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
948     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
949     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
950   }
951 
952   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
953   Ops.push_back(Res);
954 
955   if (Code == InlineAsm::Kind_Clobber) {
956     // Clobbers should always have a 1:1 mapping with registers, and may
957     // reference registers that have illegal (e.g. vector) types. Hence, we
958     // shouldn't try to apply any sort of splitting logic to them.
959     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
960            "No 1:1 mapping from clobbers to regs?");
961     unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
962     (void)SP;
963     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
964       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
965       assert(
966           (Regs[I] != SP ||
967            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
968           "If we clobbered the stack pointer, MFI should know about it.");
969     }
970     return;
971   }
972 
973   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
974     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
975     MVT RegisterVT = RegVTs[Value];
976     for (unsigned i = 0; i != NumRegs; ++i) {
977       assert(Reg < Regs.size() && "Mismatch in # registers expected");
978       unsigned TheReg = Regs[Reg++];
979       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
980     }
981   }
982 }
983 
984 SmallVector<std::pair<unsigned, unsigned>, 4>
985 RegsForValue::getRegsAndSizes() const {
986   SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
987   unsigned I = 0;
988   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
989     unsigned RegCount = std::get<0>(CountAndVT);
990     MVT RegisterVT = std::get<1>(CountAndVT);
991     unsigned RegisterSize = RegisterVT.getSizeInBits();
992     for (unsigned E = I + RegCount; I != E; ++I)
993       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
994   }
995   return OutVec;
996 }
997 
998 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
999                                const TargetLibraryInfo *li) {
1000   AA = aa;
1001   GFI = gfi;
1002   LibInfo = li;
1003   DL = &DAG.getDataLayout();
1004   Context = DAG.getContext();
1005   LPadToCallSiteMap.clear();
1006 }
1007 
1008 void SelectionDAGBuilder::clear() {
1009   NodeMap.clear();
1010   UnusedArgNodeMap.clear();
1011   PendingLoads.clear();
1012   PendingExports.clear();
1013   CurInst = nullptr;
1014   HasTailCall = false;
1015   SDNodeOrder = LowestSDNodeOrder;
1016   StatepointLowering.clear();
1017 }
1018 
1019 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1020   DanglingDebugInfoMap.clear();
1021 }
1022 
1023 SDValue SelectionDAGBuilder::getRoot() {
1024   if (PendingLoads.empty())
1025     return DAG.getRoot();
1026 
1027   if (PendingLoads.size() == 1) {
1028     SDValue Root = PendingLoads[0];
1029     DAG.setRoot(Root);
1030     PendingLoads.clear();
1031     return Root;
1032   }
1033 
1034   // Otherwise, we have to make a token factor node.
1035   // If we have >= 2^16 loads then split across multiple token factors as
1036   // there's a 64k limit on the number of SDNode operands.
1037   SDValue Root;
1038   size_t Limit = SDNode::getMaxNumOperands();
1039   while (PendingLoads.size() > Limit) {
1040     unsigned SliceIdx = PendingLoads.size() - Limit;
1041     auto ExtractedTFs = ArrayRef<SDValue>(PendingLoads).slice(SliceIdx, Limit);
1042     SDValue NewTF =
1043         DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, ExtractedTFs);
1044     PendingLoads.erase(PendingLoads.begin() + SliceIdx, PendingLoads.end());
1045     PendingLoads.emplace_back(NewTF);
1046   }
1047   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads);
1048   PendingLoads.clear();
1049   DAG.setRoot(Root);
1050   return Root;
1051 }
1052 
1053 SDValue SelectionDAGBuilder::getControlRoot() {
1054   SDValue Root = DAG.getRoot();
1055 
1056   if (PendingExports.empty())
1057     return Root;
1058 
1059   // Turn all of the CopyToReg chains into one factored node.
1060   if (Root.getOpcode() != ISD::EntryToken) {
1061     unsigned i = 0, e = PendingExports.size();
1062     for (; i != e; ++i) {
1063       assert(PendingExports[i].getNode()->getNumOperands() > 1);
1064       if (PendingExports[i].getNode()->getOperand(0) == Root)
1065         break;  // Don't add the root if we already indirectly depend on it.
1066     }
1067 
1068     if (i == e)
1069       PendingExports.push_back(Root);
1070   }
1071 
1072   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1073                      PendingExports);
1074   PendingExports.clear();
1075   DAG.setRoot(Root);
1076   return Root;
1077 }
1078 
1079 void SelectionDAGBuilder::visit(const Instruction &I) {
1080   // Set up outgoing PHI node register values before emitting the terminator.
1081   if (I.isTerminator()) {
1082     HandlePHINodesInSuccessorBlocks(I.getParent());
1083   }
1084 
1085   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1086   if (!isa<DbgInfoIntrinsic>(I))
1087     ++SDNodeOrder;
1088 
1089   CurInst = &I;
1090 
1091   visit(I.getOpcode(), I);
1092 
1093   if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1094     // Propagate the fast-math-flags of this IR instruction to the DAG node that
1095     // maps to this instruction.
1096     // TODO: We could handle all flags (nsw, etc) here.
1097     // TODO: If an IR instruction maps to >1 node, only the final node will have
1098     //       flags set.
1099     if (SDNode *Node = getNodeForIRValue(&I)) {
1100       SDNodeFlags IncomingFlags;
1101       IncomingFlags.copyFMF(*FPMO);
1102       if (!Node->getFlags().isDefined())
1103         Node->setFlags(IncomingFlags);
1104       else
1105         Node->intersectFlagsWith(IncomingFlags);
1106     }
1107   }
1108 
1109   if (!I.isTerminator() && !HasTailCall &&
1110       !isStatepoint(&I)) // statepoints handle their exports internally
1111     CopyToExportRegsIfNeeded(&I);
1112 
1113   CurInst = nullptr;
1114 }
1115 
1116 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1117   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1118 }
1119 
1120 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1121   // Note: this doesn't use InstVisitor, because it has to work with
1122   // ConstantExpr's in addition to instructions.
1123   switch (Opcode) {
1124   default: llvm_unreachable("Unknown instruction type encountered!");
1125     // Build the switch statement using the Instruction.def file.
1126 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1127     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1128 #include "llvm/IR/Instruction.def"
1129   }
1130 }
1131 
1132 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1133                                                 const DIExpression *Expr) {
1134   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1135     const DbgValueInst *DI = DDI.getDI();
1136     DIVariable *DanglingVariable = DI->getVariable();
1137     DIExpression *DanglingExpr = DI->getExpression();
1138     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1139       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1140       return true;
1141     }
1142     return false;
1143   };
1144 
1145   for (auto &DDIMI : DanglingDebugInfoMap) {
1146     DanglingDebugInfoVector &DDIV = DDIMI.second;
1147     DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1148   }
1149 }
1150 
1151 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1152 // generate the debug data structures now that we've seen its definition.
1153 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1154                                                    SDValue Val) {
1155   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1156   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1157     return;
1158 
1159   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1160   for (auto &DDI : DDIV) {
1161     const DbgValueInst *DI = DDI.getDI();
1162     assert(DI && "Ill-formed DanglingDebugInfo");
1163     DebugLoc dl = DDI.getdl();
1164     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1165     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1166     DILocalVariable *Variable = DI->getVariable();
1167     DIExpression *Expr = DI->getExpression();
1168     assert(Variable->isValidLocationForIntrinsic(dl) &&
1169            "Expected inlined-at fields to agree");
1170     SDDbgValue *SDV;
1171     if (Val.getNode()) {
1172       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1173         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1174                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1175         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1176         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1177         // inserted after the definition of Val when emitting the instructions
1178         // after ISel. An alternative could be to teach
1179         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1180         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1181                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1182                    << ValSDNodeOrder << "\n");
1183         SDV = getDbgValue(Val, Variable, Expr, dl,
1184                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1185         DAG.AddDbgValue(SDV, Val.getNode(), false);
1186       } else
1187         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1188                           << "in EmitFuncArgumentDbgValue\n");
1189     } else
1190       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1191   }
1192   DDIV.clear();
1193 }
1194 
1195 /// getCopyFromRegs - If there was virtual register allocated for the value V
1196 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1197 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1198   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1199   SDValue Result;
1200 
1201   if (It != FuncInfo.ValueMap.end()) {
1202     unsigned InReg = It->second;
1203 
1204     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1205                      DAG.getDataLayout(), InReg, Ty,
1206                      None); // This is not an ABI copy.
1207     SDValue Chain = DAG.getEntryNode();
1208     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1209                                  V);
1210     resolveDanglingDebugInfo(V, Result);
1211   }
1212 
1213   return Result;
1214 }
1215 
1216 /// getValue - Return an SDValue for the given Value.
1217 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1218   // If we already have an SDValue for this value, use it. It's important
1219   // to do this first, so that we don't create a CopyFromReg if we already
1220   // have a regular SDValue.
1221   SDValue &N = NodeMap[V];
1222   if (N.getNode()) return N;
1223 
1224   // If there's a virtual register allocated and initialized for this
1225   // value, use it.
1226   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1227     return copyFromReg;
1228 
1229   // Otherwise create a new SDValue and remember it.
1230   SDValue Val = getValueImpl(V);
1231   NodeMap[V] = Val;
1232   resolveDanglingDebugInfo(V, Val);
1233   return Val;
1234 }
1235 
1236 // Return true if SDValue exists for the given Value
1237 bool SelectionDAGBuilder::findValue(const Value *V) const {
1238   return (NodeMap.find(V) != NodeMap.end()) ||
1239     (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1240 }
1241 
1242 /// getNonRegisterValue - Return an SDValue for the given Value, but
1243 /// don't look in FuncInfo.ValueMap for a virtual register.
1244 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1245   // If we already have an SDValue for this value, use it.
1246   SDValue &N = NodeMap[V];
1247   if (N.getNode()) {
1248     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1249       // Remove the debug location from the node as the node is about to be used
1250       // in a location which may differ from the original debug location.  This
1251       // is relevant to Constant and ConstantFP nodes because they can appear
1252       // as constant expressions inside PHI nodes.
1253       N->setDebugLoc(DebugLoc());
1254     }
1255     return N;
1256   }
1257 
1258   // Otherwise create a new SDValue and remember it.
1259   SDValue Val = getValueImpl(V);
1260   NodeMap[V] = Val;
1261   resolveDanglingDebugInfo(V, Val);
1262   return Val;
1263 }
1264 
1265 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1266 /// Create an SDValue for the given value.
1267 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1268   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1269 
1270   if (const Constant *C = dyn_cast<Constant>(V)) {
1271     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1272 
1273     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1274       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1275 
1276     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1277       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1278 
1279     if (isa<ConstantPointerNull>(C)) {
1280       unsigned AS = V->getType()->getPointerAddressSpace();
1281       return DAG.getConstant(0, getCurSDLoc(),
1282                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1283     }
1284 
1285     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1286       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1287 
1288     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1289       return DAG.getUNDEF(VT);
1290 
1291     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1292       visit(CE->getOpcode(), *CE);
1293       SDValue N1 = NodeMap[V];
1294       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1295       return N1;
1296     }
1297 
1298     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1299       SmallVector<SDValue, 4> Constants;
1300       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1301            OI != OE; ++OI) {
1302         SDNode *Val = getValue(*OI).getNode();
1303         // If the operand is an empty aggregate, there are no values.
1304         if (!Val) continue;
1305         // Add each leaf value from the operand to the Constants list
1306         // to form a flattened list of all the values.
1307         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1308           Constants.push_back(SDValue(Val, i));
1309       }
1310 
1311       return DAG.getMergeValues(Constants, getCurSDLoc());
1312     }
1313 
1314     if (const ConstantDataSequential *CDS =
1315           dyn_cast<ConstantDataSequential>(C)) {
1316       SmallVector<SDValue, 4> Ops;
1317       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1318         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1319         // Add each leaf value from the operand to the Constants list
1320         // to form a flattened list of all the values.
1321         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1322           Ops.push_back(SDValue(Val, i));
1323       }
1324 
1325       if (isa<ArrayType>(CDS->getType()))
1326         return DAG.getMergeValues(Ops, getCurSDLoc());
1327       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1328     }
1329 
1330     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1331       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1332              "Unknown struct or array constant!");
1333 
1334       SmallVector<EVT, 4> ValueVTs;
1335       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1336       unsigned NumElts = ValueVTs.size();
1337       if (NumElts == 0)
1338         return SDValue(); // empty struct
1339       SmallVector<SDValue, 4> Constants(NumElts);
1340       for (unsigned i = 0; i != NumElts; ++i) {
1341         EVT EltVT = ValueVTs[i];
1342         if (isa<UndefValue>(C))
1343           Constants[i] = DAG.getUNDEF(EltVT);
1344         else if (EltVT.isFloatingPoint())
1345           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1346         else
1347           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1348       }
1349 
1350       return DAG.getMergeValues(Constants, getCurSDLoc());
1351     }
1352 
1353     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1354       return DAG.getBlockAddress(BA, VT);
1355 
1356     VectorType *VecTy = cast<VectorType>(V->getType());
1357     unsigned NumElements = VecTy->getNumElements();
1358 
1359     // Now that we know the number and type of the elements, get that number of
1360     // elements into the Ops array based on what kind of constant it is.
1361     SmallVector<SDValue, 16> Ops;
1362     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1363       for (unsigned i = 0; i != NumElements; ++i)
1364         Ops.push_back(getValue(CV->getOperand(i)));
1365     } else {
1366       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1367       EVT EltVT =
1368           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1369 
1370       SDValue Op;
1371       if (EltVT.isFloatingPoint())
1372         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1373       else
1374         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1375       Ops.assign(NumElements, Op);
1376     }
1377 
1378     // Create a BUILD_VECTOR node.
1379     return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1380   }
1381 
1382   // If this is a static alloca, generate it as the frameindex instead of
1383   // computation.
1384   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1385     DenseMap<const AllocaInst*, int>::iterator SI =
1386       FuncInfo.StaticAllocaMap.find(AI);
1387     if (SI != FuncInfo.StaticAllocaMap.end())
1388       return DAG.getFrameIndex(SI->second,
1389                                TLI.getFrameIndexTy(DAG.getDataLayout()));
1390   }
1391 
1392   // If this is an instruction which fast-isel has deferred, select it now.
1393   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1394     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1395 
1396     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1397                      Inst->getType(), getABIRegCopyCC(V));
1398     SDValue Chain = DAG.getEntryNode();
1399     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1400   }
1401 
1402   llvm_unreachable("Can't get register for value!");
1403 }
1404 
1405 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1406   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1407   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1408   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1409   bool IsSEH = isAsynchronousEHPersonality(Pers);
1410   bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1411   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1412   if (!IsSEH)
1413     CatchPadMBB->setIsEHScopeEntry();
1414   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1415   if (IsMSVCCXX || IsCoreCLR)
1416     CatchPadMBB->setIsEHFuncletEntry();
1417   // Wasm does not need catchpads anymore
1418   if (!IsWasmCXX)
1419     DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1420                             getControlRoot()));
1421 }
1422 
1423 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1424   // Update machine-CFG edge.
1425   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1426   FuncInfo.MBB->addSuccessor(TargetMBB);
1427 
1428   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1429   bool IsSEH = isAsynchronousEHPersonality(Pers);
1430   if (IsSEH) {
1431     // If this is not a fall-through branch or optimizations are switched off,
1432     // emit the branch.
1433     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1434         TM.getOptLevel() == CodeGenOpt::None)
1435       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1436                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1437     return;
1438   }
1439 
1440   // Figure out the funclet membership for the catchret's successor.
1441   // This will be used by the FuncletLayout pass to determine how to order the
1442   // BB's.
1443   // A 'catchret' returns to the outer scope's color.
1444   Value *ParentPad = I.getCatchSwitchParentPad();
1445   const BasicBlock *SuccessorColor;
1446   if (isa<ConstantTokenNone>(ParentPad))
1447     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1448   else
1449     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1450   assert(SuccessorColor && "No parent funclet for catchret!");
1451   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1452   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1453 
1454   // Create the terminator node.
1455   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1456                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1457                             DAG.getBasicBlock(SuccessorColorMBB));
1458   DAG.setRoot(Ret);
1459 }
1460 
1461 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1462   // Don't emit any special code for the cleanuppad instruction. It just marks
1463   // the start of an EH scope/funclet.
1464   FuncInfo.MBB->setIsEHScopeEntry();
1465   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1466   if (Pers != EHPersonality::Wasm_CXX) {
1467     FuncInfo.MBB->setIsEHFuncletEntry();
1468     FuncInfo.MBB->setIsCleanupFuncletEntry();
1469   }
1470 }
1471 
1472 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1473 /// many places it could ultimately go. In the IR, we have a single unwind
1474 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1475 /// This function skips over imaginary basic blocks that hold catchswitch
1476 /// instructions, and finds all the "real" machine
1477 /// basic block destinations. As those destinations may not be successors of
1478 /// EHPadBB, here we also calculate the edge probability to those destinations.
1479 /// The passed-in Prob is the edge probability to EHPadBB.
1480 static void findUnwindDestinations(
1481     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1482     BranchProbability Prob,
1483     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1484         &UnwindDests) {
1485   EHPersonality Personality =
1486     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1487   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1488   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1489   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1490   bool IsSEH = isAsynchronousEHPersonality(Personality);
1491 
1492   while (EHPadBB) {
1493     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1494     BasicBlock *NewEHPadBB = nullptr;
1495     if (isa<LandingPadInst>(Pad)) {
1496       // Stop on landingpads. They are not funclets.
1497       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1498       break;
1499     } else if (isa<CleanupPadInst>(Pad)) {
1500       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1501       // personalities.
1502       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1503       UnwindDests.back().first->setIsEHScopeEntry();
1504       if (!IsWasmCXX)
1505         UnwindDests.back().first->setIsEHFuncletEntry();
1506       break;
1507     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1508       // Add the catchpad handlers to the possible destinations.
1509       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1510         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1511         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1512         if (IsMSVCCXX || IsCoreCLR)
1513           UnwindDests.back().first->setIsEHFuncletEntry();
1514         if (!IsSEH)
1515           UnwindDests.back().first->setIsEHScopeEntry();
1516       }
1517       NewEHPadBB = CatchSwitch->getUnwindDest();
1518     } else {
1519       continue;
1520     }
1521 
1522     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1523     if (BPI && NewEHPadBB)
1524       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1525     EHPadBB = NewEHPadBB;
1526   }
1527 }
1528 
1529 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1530   // Update successor info.
1531   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1532   auto UnwindDest = I.getUnwindDest();
1533   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1534   BranchProbability UnwindDestProb =
1535       (BPI && UnwindDest)
1536           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1537           : BranchProbability::getZero();
1538   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1539   for (auto &UnwindDest : UnwindDests) {
1540     UnwindDest.first->setIsEHPad();
1541     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1542   }
1543   FuncInfo.MBB->normalizeSuccProbs();
1544 
1545   // Create the terminator node.
1546   SDValue Ret =
1547       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1548   DAG.setRoot(Ret);
1549 }
1550 
1551 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1552   report_fatal_error("visitCatchSwitch not yet implemented!");
1553 }
1554 
1555 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1556   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1557   auto &DL = DAG.getDataLayout();
1558   SDValue Chain = getControlRoot();
1559   SmallVector<ISD::OutputArg, 8> Outs;
1560   SmallVector<SDValue, 8> OutVals;
1561 
1562   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1563   // lower
1564   //
1565   //   %val = call <ty> @llvm.experimental.deoptimize()
1566   //   ret <ty> %val
1567   //
1568   // differently.
1569   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1570     LowerDeoptimizingReturn();
1571     return;
1572   }
1573 
1574   if (!FuncInfo.CanLowerReturn) {
1575     unsigned DemoteReg = FuncInfo.DemoteRegister;
1576     const Function *F = I.getParent()->getParent();
1577 
1578     // Emit a store of the return value through the virtual register.
1579     // Leave Outs empty so that LowerReturn won't try to load return
1580     // registers the usual way.
1581     SmallVector<EVT, 1> PtrValueVTs;
1582     ComputeValueVTs(TLI, DL,
1583                     F->getReturnType()->getPointerTo(
1584                         DAG.getDataLayout().getAllocaAddrSpace()),
1585                     PtrValueVTs);
1586 
1587     SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1588                                         DemoteReg, PtrValueVTs[0]);
1589     SDValue RetOp = getValue(I.getOperand(0));
1590 
1591     SmallVector<EVT, 4> ValueVTs;
1592     SmallVector<uint64_t, 4> Offsets;
1593     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1594     unsigned NumValues = ValueVTs.size();
1595 
1596     SmallVector<SDValue, 4> Chains(NumValues);
1597     for (unsigned i = 0; i != NumValues; ++i) {
1598       // An aggregate return value cannot wrap around the address space, so
1599       // offsets to its parts don't wrap either.
1600       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1601       Chains[i] = DAG.getStore(
1602           Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1603           // FIXME: better loc info would be nice.
1604           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1605     }
1606 
1607     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1608                         MVT::Other, Chains);
1609   } else if (I.getNumOperands() != 0) {
1610     SmallVector<EVT, 4> ValueVTs;
1611     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1612     unsigned NumValues = ValueVTs.size();
1613     if (NumValues) {
1614       SDValue RetOp = getValue(I.getOperand(0));
1615 
1616       const Function *F = I.getParent()->getParent();
1617 
1618       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1619       if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1620                                           Attribute::SExt))
1621         ExtendKind = ISD::SIGN_EXTEND;
1622       else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1623                                                Attribute::ZExt))
1624         ExtendKind = ISD::ZERO_EXTEND;
1625 
1626       LLVMContext &Context = F->getContext();
1627       bool RetInReg = F->getAttributes().hasAttribute(
1628           AttributeList::ReturnIndex, Attribute::InReg);
1629 
1630       for (unsigned j = 0; j != NumValues; ++j) {
1631         EVT VT = ValueVTs[j];
1632 
1633         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1634           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1635 
1636         CallingConv::ID CC = F->getCallingConv();
1637 
1638         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1639         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1640         SmallVector<SDValue, 4> Parts(NumParts);
1641         getCopyToParts(DAG, getCurSDLoc(),
1642                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1643                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1644 
1645         // 'inreg' on function refers to return value
1646         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1647         if (RetInReg)
1648           Flags.setInReg();
1649 
1650         // Propagate extension type if any
1651         if (ExtendKind == ISD::SIGN_EXTEND)
1652           Flags.setSExt();
1653         else if (ExtendKind == ISD::ZERO_EXTEND)
1654           Flags.setZExt();
1655 
1656         for (unsigned i = 0; i < NumParts; ++i) {
1657           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1658                                         VT, /*isfixed=*/true, 0, 0));
1659           OutVals.push_back(Parts[i]);
1660         }
1661       }
1662     }
1663   }
1664 
1665   // Push in swifterror virtual register as the last element of Outs. This makes
1666   // sure swifterror virtual register will be returned in the swifterror
1667   // physical register.
1668   const Function *F = I.getParent()->getParent();
1669   if (TLI.supportSwiftError() &&
1670       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1671     assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1672     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1673     Flags.setSwiftError();
1674     Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1675                                   EVT(TLI.getPointerTy(DL)) /*argvt*/,
1676                                   true /*isfixed*/, 1 /*origidx*/,
1677                                   0 /*partOffs*/));
1678     // Create SDNode for the swifterror virtual register.
1679     OutVals.push_back(
1680         DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1681                             &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1682                         EVT(TLI.getPointerTy(DL))));
1683   }
1684 
1685   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1686   CallingConv::ID CallConv =
1687     DAG.getMachineFunction().getFunction().getCallingConv();
1688   Chain = DAG.getTargetLoweringInfo().LowerReturn(
1689       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1690 
1691   // Verify that the target's LowerReturn behaved as expected.
1692   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1693          "LowerReturn didn't return a valid chain!");
1694 
1695   // Update the DAG with the new chain value resulting from return lowering.
1696   DAG.setRoot(Chain);
1697 }
1698 
1699 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1700 /// created for it, emit nodes to copy the value into the virtual
1701 /// registers.
1702 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1703   // Skip empty types
1704   if (V->getType()->isEmptyTy())
1705     return;
1706 
1707   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1708   if (VMI != FuncInfo.ValueMap.end()) {
1709     assert(!V->use_empty() && "Unused value assigned virtual registers!");
1710     CopyValueToVirtualRegister(V, VMI->second);
1711   }
1712 }
1713 
1714 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1715 /// the current basic block, add it to ValueMap now so that we'll get a
1716 /// CopyTo/FromReg.
1717 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1718   // No need to export constants.
1719   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1720 
1721   // Already exported?
1722   if (FuncInfo.isExportedInst(V)) return;
1723 
1724   unsigned Reg = FuncInfo.InitializeRegForValue(V);
1725   CopyValueToVirtualRegister(V, Reg);
1726 }
1727 
1728 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1729                                                      const BasicBlock *FromBB) {
1730   // The operands of the setcc have to be in this block.  We don't know
1731   // how to export them from some other block.
1732   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1733     // Can export from current BB.
1734     if (VI->getParent() == FromBB)
1735       return true;
1736 
1737     // Is already exported, noop.
1738     return FuncInfo.isExportedInst(V);
1739   }
1740 
1741   // If this is an argument, we can export it if the BB is the entry block or
1742   // if it is already exported.
1743   if (isa<Argument>(V)) {
1744     if (FromBB == &FromBB->getParent()->getEntryBlock())
1745       return true;
1746 
1747     // Otherwise, can only export this if it is already exported.
1748     return FuncInfo.isExportedInst(V);
1749   }
1750 
1751   // Otherwise, constants can always be exported.
1752   return true;
1753 }
1754 
1755 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1756 BranchProbability
1757 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1758                                         const MachineBasicBlock *Dst) const {
1759   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1760   const BasicBlock *SrcBB = Src->getBasicBlock();
1761   const BasicBlock *DstBB = Dst->getBasicBlock();
1762   if (!BPI) {
1763     // If BPI is not available, set the default probability as 1 / N, where N is
1764     // the number of successors.
1765     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1766     return BranchProbability(1, SuccSize);
1767   }
1768   return BPI->getEdgeProbability(SrcBB, DstBB);
1769 }
1770 
1771 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1772                                                MachineBasicBlock *Dst,
1773                                                BranchProbability Prob) {
1774   if (!FuncInfo.BPI)
1775     Src->addSuccessorWithoutProb(Dst);
1776   else {
1777     if (Prob.isUnknown())
1778       Prob = getEdgeProbability(Src, Dst);
1779     Src->addSuccessor(Dst, Prob);
1780   }
1781 }
1782 
1783 static bool InBlock(const Value *V, const BasicBlock *BB) {
1784   if (const Instruction *I = dyn_cast<Instruction>(V))
1785     return I->getParent() == BB;
1786   return true;
1787 }
1788 
1789 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1790 /// This function emits a branch and is used at the leaves of an OR or an
1791 /// AND operator tree.
1792 void
1793 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1794                                                   MachineBasicBlock *TBB,
1795                                                   MachineBasicBlock *FBB,
1796                                                   MachineBasicBlock *CurBB,
1797                                                   MachineBasicBlock *SwitchBB,
1798                                                   BranchProbability TProb,
1799                                                   BranchProbability FProb,
1800                                                   bool InvertCond) {
1801   const BasicBlock *BB = CurBB->getBasicBlock();
1802 
1803   // If the leaf of the tree is a comparison, merge the condition into
1804   // the caseblock.
1805   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1806     // The operands of the cmp have to be in this block.  We don't know
1807     // how to export them from some other block.  If this is the first block
1808     // of the sequence, no exporting is needed.
1809     if (CurBB == SwitchBB ||
1810         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1811          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1812       ISD::CondCode Condition;
1813       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1814         ICmpInst::Predicate Pred =
1815             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1816         Condition = getICmpCondCode(Pred);
1817       } else {
1818         const FCmpInst *FC = cast<FCmpInst>(Cond);
1819         FCmpInst::Predicate Pred =
1820             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1821         Condition = getFCmpCondCode(Pred);
1822         if (TM.Options.NoNaNsFPMath)
1823           Condition = getFCmpCodeWithoutNaN(Condition);
1824       }
1825 
1826       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1827                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1828       SwitchCases.push_back(CB);
1829       return;
1830     }
1831   }
1832 
1833   // Create a CaseBlock record representing this branch.
1834   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1835   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1836                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1837   SwitchCases.push_back(CB);
1838 }
1839 
1840 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1841                                                MachineBasicBlock *TBB,
1842                                                MachineBasicBlock *FBB,
1843                                                MachineBasicBlock *CurBB,
1844                                                MachineBasicBlock *SwitchBB,
1845                                                Instruction::BinaryOps Opc,
1846                                                BranchProbability TProb,
1847                                                BranchProbability FProb,
1848                                                bool InvertCond) {
1849   // Skip over not part of the tree and remember to invert op and operands at
1850   // next level.
1851   Value *NotCond;
1852   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
1853       InBlock(NotCond, CurBB->getBasicBlock())) {
1854     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1855                          !InvertCond);
1856     return;
1857   }
1858 
1859   const Instruction *BOp = dyn_cast<Instruction>(Cond);
1860   // Compute the effective opcode for Cond, taking into account whether it needs
1861   // to be inverted, e.g.
1862   //   and (not (or A, B)), C
1863   // gets lowered as
1864   //   and (and (not A, not B), C)
1865   unsigned BOpc = 0;
1866   if (BOp) {
1867     BOpc = BOp->getOpcode();
1868     if (InvertCond) {
1869       if (BOpc == Instruction::And)
1870         BOpc = Instruction::Or;
1871       else if (BOpc == Instruction::Or)
1872         BOpc = Instruction::And;
1873     }
1874   }
1875 
1876   // If this node is not part of the or/and tree, emit it as a branch.
1877   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1878       BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1879       BOp->getParent() != CurBB->getBasicBlock() ||
1880       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1881       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1882     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1883                                  TProb, FProb, InvertCond);
1884     return;
1885   }
1886 
1887   //  Create TmpBB after CurBB.
1888   MachineFunction::iterator BBI(CurBB);
1889   MachineFunction &MF = DAG.getMachineFunction();
1890   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1891   CurBB->getParent()->insert(++BBI, TmpBB);
1892 
1893   if (Opc == Instruction::Or) {
1894     // Codegen X | Y as:
1895     // BB1:
1896     //   jmp_if_X TBB
1897     //   jmp TmpBB
1898     // TmpBB:
1899     //   jmp_if_Y TBB
1900     //   jmp FBB
1901     //
1902 
1903     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1904     // The requirement is that
1905     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1906     //     = TrueProb for original BB.
1907     // Assuming the original probabilities are A and B, one choice is to set
1908     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1909     // A/(1+B) and 2B/(1+B). This choice assumes that
1910     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1911     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1912     // TmpBB, but the math is more complicated.
1913 
1914     auto NewTrueProb = TProb / 2;
1915     auto NewFalseProb = TProb / 2 + FProb;
1916     // Emit the LHS condition.
1917     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1918                          NewTrueProb, NewFalseProb, InvertCond);
1919 
1920     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1921     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1922     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1923     // Emit the RHS condition into TmpBB.
1924     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1925                          Probs[0], Probs[1], InvertCond);
1926   } else {
1927     assert(Opc == Instruction::And && "Unknown merge op!");
1928     // Codegen X & Y as:
1929     // BB1:
1930     //   jmp_if_X TmpBB
1931     //   jmp FBB
1932     // TmpBB:
1933     //   jmp_if_Y TBB
1934     //   jmp FBB
1935     //
1936     //  This requires creation of TmpBB after CurBB.
1937 
1938     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1939     // The requirement is that
1940     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1941     //     = FalseProb for original BB.
1942     // Assuming the original probabilities are A and B, one choice is to set
1943     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1944     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1945     // TrueProb for BB1 * FalseProb for TmpBB.
1946 
1947     auto NewTrueProb = TProb + FProb / 2;
1948     auto NewFalseProb = FProb / 2;
1949     // Emit the LHS condition.
1950     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1951                          NewTrueProb, NewFalseProb, InvertCond);
1952 
1953     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1954     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1955     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1956     // Emit the RHS condition into TmpBB.
1957     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1958                          Probs[0], Probs[1], InvertCond);
1959   }
1960 }
1961 
1962 /// If the set of cases should be emitted as a series of branches, return true.
1963 /// If we should emit this as a bunch of and/or'd together conditions, return
1964 /// false.
1965 bool
1966 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1967   if (Cases.size() != 2) return true;
1968 
1969   // If this is two comparisons of the same values or'd or and'd together, they
1970   // will get folded into a single comparison, so don't emit two blocks.
1971   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1972        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1973       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1974        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1975     return false;
1976   }
1977 
1978   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1979   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1980   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1981       Cases[0].CC == Cases[1].CC &&
1982       isa<Constant>(Cases[0].CmpRHS) &&
1983       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1984     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1985       return false;
1986     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1987       return false;
1988   }
1989 
1990   return true;
1991 }
1992 
1993 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1994   MachineBasicBlock *BrMBB = FuncInfo.MBB;
1995 
1996   // Update machine-CFG edges.
1997   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1998 
1999   if (I.isUnconditional()) {
2000     // Update machine-CFG edges.
2001     BrMBB->addSuccessor(Succ0MBB);
2002 
2003     // If this is not a fall-through branch or optimizations are switched off,
2004     // emit the branch.
2005     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2006       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2007                               MVT::Other, getControlRoot(),
2008                               DAG.getBasicBlock(Succ0MBB)));
2009 
2010     return;
2011   }
2012 
2013   // If this condition is one of the special cases we handle, do special stuff
2014   // now.
2015   const Value *CondVal = I.getCondition();
2016   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2017 
2018   // If this is a series of conditions that are or'd or and'd together, emit
2019   // this as a sequence of branches instead of setcc's with and/or operations.
2020   // As long as jumps are not expensive, this should improve performance.
2021   // For example, instead of something like:
2022   //     cmp A, B
2023   //     C = seteq
2024   //     cmp D, E
2025   //     F = setle
2026   //     or C, F
2027   //     jnz foo
2028   // Emit:
2029   //     cmp A, B
2030   //     je foo
2031   //     cmp D, E
2032   //     jle foo
2033   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2034     Instruction::BinaryOps Opcode = BOp->getOpcode();
2035     if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2036         !I.getMetadata(LLVMContext::MD_unpredictable) &&
2037         (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2038       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2039                            Opcode,
2040                            getEdgeProbability(BrMBB, Succ0MBB),
2041                            getEdgeProbability(BrMBB, Succ1MBB),
2042                            /*InvertCond=*/false);
2043       // If the compares in later blocks need to use values not currently
2044       // exported from this block, export them now.  This block should always
2045       // be the first entry.
2046       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2047 
2048       // Allow some cases to be rejected.
2049       if (ShouldEmitAsBranches(SwitchCases)) {
2050         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2051           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2052           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2053         }
2054 
2055         // Emit the branch for this block.
2056         visitSwitchCase(SwitchCases[0], BrMBB);
2057         SwitchCases.erase(SwitchCases.begin());
2058         return;
2059       }
2060 
2061       // Okay, we decided not to do this, remove any inserted MBB's and clear
2062       // SwitchCases.
2063       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2064         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2065 
2066       SwitchCases.clear();
2067     }
2068   }
2069 
2070   // Create a CaseBlock record representing this branch.
2071   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2072                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2073 
2074   // Use visitSwitchCase to actually insert the fast branch sequence for this
2075   // cond branch.
2076   visitSwitchCase(CB, BrMBB);
2077 }
2078 
2079 /// visitSwitchCase - Emits the necessary code to represent a single node in
2080 /// the binary search tree resulting from lowering a switch instruction.
2081 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2082                                           MachineBasicBlock *SwitchBB) {
2083   SDValue Cond;
2084   SDValue CondLHS = getValue(CB.CmpLHS);
2085   SDLoc dl = CB.DL;
2086 
2087   // Build the setcc now.
2088   if (!CB.CmpMHS) {
2089     // Fold "(X == true)" to X and "(X == false)" to !X to
2090     // handle common cases produced by branch lowering.
2091     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2092         CB.CC == ISD::SETEQ)
2093       Cond = CondLHS;
2094     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2095              CB.CC == ISD::SETEQ) {
2096       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2097       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2098     } else
2099       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2100   } else {
2101     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2102 
2103     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2104     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2105 
2106     SDValue CmpOp = getValue(CB.CmpMHS);
2107     EVT VT = CmpOp.getValueType();
2108 
2109     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2110       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2111                           ISD::SETLE);
2112     } else {
2113       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2114                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2115       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2116                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2117     }
2118   }
2119 
2120   // Update successor info
2121   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2122   // TrueBB and FalseBB are always different unless the incoming IR is
2123   // degenerate. This only happens when running llc on weird IR.
2124   if (CB.TrueBB != CB.FalseBB)
2125     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2126   SwitchBB->normalizeSuccProbs();
2127 
2128   // If the lhs block is the next block, invert the condition so that we can
2129   // fall through to the lhs instead of the rhs block.
2130   if (CB.TrueBB == NextBlock(SwitchBB)) {
2131     std::swap(CB.TrueBB, CB.FalseBB);
2132     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2133     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2134   }
2135 
2136   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2137                                MVT::Other, getControlRoot(), Cond,
2138                                DAG.getBasicBlock(CB.TrueBB));
2139 
2140   // Insert the false branch. Do this even if it's a fall through branch,
2141   // this makes it easier to do DAG optimizations which require inverting
2142   // the branch condition.
2143   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2144                        DAG.getBasicBlock(CB.FalseBB));
2145 
2146   DAG.setRoot(BrCond);
2147 }
2148 
2149 /// visitJumpTable - Emit JumpTable node in the current MBB
2150 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
2151   // Emit the code for the jump table
2152   assert(JT.Reg != -1U && "Should lower JT Header first!");
2153   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2154   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2155                                      JT.Reg, PTy);
2156   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2157   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2158                                     MVT::Other, Index.getValue(1),
2159                                     Table, Index);
2160   DAG.setRoot(BrJumpTable);
2161 }
2162 
2163 /// visitJumpTableHeader - This function emits necessary code to produce index
2164 /// in the JumpTable from switch case.
2165 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
2166                                                JumpTableHeader &JTH,
2167                                                MachineBasicBlock *SwitchBB) {
2168   SDLoc dl = getCurSDLoc();
2169 
2170   // Subtract the lowest switch case value from the value being switched on and
2171   // conditional branch to default mbb if the result is greater than the
2172   // difference between smallest and largest cases.
2173   SDValue SwitchOp = getValue(JTH.SValue);
2174   EVT VT = SwitchOp.getValueType();
2175   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2176                             DAG.getConstant(JTH.First, dl, VT));
2177 
2178   // The SDNode we just created, which holds the value being switched on minus
2179   // the smallest case value, needs to be copied to a virtual register so it
2180   // can be used as an index into the jump table in a subsequent basic block.
2181   // This value may be smaller or larger than the target's pointer type, and
2182   // therefore require extension or truncating.
2183   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2184   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2185 
2186   unsigned JumpTableReg =
2187       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2188   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2189                                     JumpTableReg, SwitchOp);
2190   JT.Reg = JumpTableReg;
2191 
2192   // Emit the range check for the jump table, and branch to the default block
2193   // for the switch statement if the value being switched on exceeds the largest
2194   // case in the switch.
2195   SDValue CMP = DAG.getSetCC(
2196       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2197                                  Sub.getValueType()),
2198       Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2199 
2200   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2201                                MVT::Other, CopyTo, CMP,
2202                                DAG.getBasicBlock(JT.Default));
2203 
2204   // Avoid emitting unnecessary branches to the next block.
2205   if (JT.MBB != NextBlock(SwitchBB))
2206     BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2207                          DAG.getBasicBlock(JT.MBB));
2208 
2209   DAG.setRoot(BrCond);
2210 }
2211 
2212 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2213 /// variable if there exists one.
2214 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2215                                  SDValue &Chain) {
2216   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2217   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2218   MachineFunction &MF = DAG.getMachineFunction();
2219   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2220   MachineSDNode *Node =
2221       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2222   if (Global) {
2223     MachinePointerInfo MPInfo(Global);
2224     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2225                  MachineMemOperand::MODereferenceable;
2226     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2227         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2228     DAG.setNodeMemRefs(Node, {MemRef});
2229   }
2230   return SDValue(Node, 0);
2231 }
2232 
2233 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2234 /// tail spliced into a stack protector check success bb.
2235 ///
2236 /// For a high level explanation of how this fits into the stack protector
2237 /// generation see the comment on the declaration of class
2238 /// StackProtectorDescriptor.
2239 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2240                                                   MachineBasicBlock *ParentBB) {
2241 
2242   // First create the loads to the guard/stack slot for the comparison.
2243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2244   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2245 
2246   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2247   int FI = MFI.getStackProtectorIndex();
2248 
2249   SDValue Guard;
2250   SDLoc dl = getCurSDLoc();
2251   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2252   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2253   unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2254 
2255   // Generate code to load the content of the guard slot.
2256   SDValue GuardVal = DAG.getLoad(
2257       PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2258       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2259       MachineMemOperand::MOVolatile);
2260 
2261   if (TLI.useStackGuardXorFP())
2262     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2263 
2264   // Retrieve guard check function, nullptr if instrumentation is inlined.
2265   if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2266     // The target provides a guard check function to validate the guard value.
2267     // Generate a call to that function with the content of the guard slot as
2268     // argument.
2269     auto *Fn = cast<Function>(GuardCheck);
2270     FunctionType *FnTy = Fn->getFunctionType();
2271     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2272 
2273     TargetLowering::ArgListTy Args;
2274     TargetLowering::ArgListEntry Entry;
2275     Entry.Node = GuardVal;
2276     Entry.Ty = FnTy->getParamType(0);
2277     if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2278       Entry.IsInReg = true;
2279     Args.push_back(Entry);
2280 
2281     TargetLowering::CallLoweringInfo CLI(DAG);
2282     CLI.setDebugLoc(getCurSDLoc())
2283       .setChain(DAG.getEntryNode())
2284       .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2285                  getValue(GuardCheck), std::move(Args));
2286 
2287     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2288     DAG.setRoot(Result.second);
2289     return;
2290   }
2291 
2292   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2293   // Otherwise, emit a volatile load to retrieve the stack guard value.
2294   SDValue Chain = DAG.getEntryNode();
2295   if (TLI.useLoadStackGuardNode()) {
2296     Guard = getLoadStackGuard(DAG, dl, Chain);
2297   } else {
2298     const Value *IRGuard = TLI.getSDagStackGuard(M);
2299     SDValue GuardPtr = getValue(IRGuard);
2300 
2301     Guard =
2302         DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2303                     Align, MachineMemOperand::MOVolatile);
2304   }
2305 
2306   // Perform the comparison via a subtract/getsetcc.
2307   EVT VT = Guard.getValueType();
2308   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2309 
2310   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2311                                                         *DAG.getContext(),
2312                                                         Sub.getValueType()),
2313                              Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2314 
2315   // If the sub is not 0, then we know the guard/stackslot do not equal, so
2316   // branch to failure MBB.
2317   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2318                                MVT::Other, GuardVal.getOperand(0),
2319                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2320   // Otherwise branch to success MBB.
2321   SDValue Br = DAG.getNode(ISD::BR, dl,
2322                            MVT::Other, BrCond,
2323                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2324 
2325   DAG.setRoot(Br);
2326 }
2327 
2328 /// Codegen the failure basic block for a stack protector check.
2329 ///
2330 /// A failure stack protector machine basic block consists simply of a call to
2331 /// __stack_chk_fail().
2332 ///
2333 /// For a high level explanation of how this fits into the stack protector
2334 /// generation see the comment on the declaration of class
2335 /// StackProtectorDescriptor.
2336 void
2337 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2338   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2339   SDValue Chain =
2340       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2341                       None, false, getCurSDLoc(), false, false).second;
2342   DAG.setRoot(Chain);
2343 }
2344 
2345 /// visitBitTestHeader - This function emits necessary code to produce value
2346 /// suitable for "bit tests"
2347 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2348                                              MachineBasicBlock *SwitchBB) {
2349   SDLoc dl = getCurSDLoc();
2350 
2351   // Subtract the minimum value
2352   SDValue SwitchOp = getValue(B.SValue);
2353   EVT VT = SwitchOp.getValueType();
2354   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2355                             DAG.getConstant(B.First, dl, VT));
2356 
2357   // Check range
2358   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2359   SDValue RangeCmp = DAG.getSetCC(
2360       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2361                                  Sub.getValueType()),
2362       Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2363 
2364   // Determine the type of the test operands.
2365   bool UsePtrType = false;
2366   if (!TLI.isTypeLegal(VT))
2367     UsePtrType = true;
2368   else {
2369     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2370       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2371         // Switch table case range are encoded into series of masks.
2372         // Just use pointer type, it's guaranteed to fit.
2373         UsePtrType = true;
2374         break;
2375       }
2376   }
2377   if (UsePtrType) {
2378     VT = TLI.getPointerTy(DAG.getDataLayout());
2379     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2380   }
2381 
2382   B.RegVT = VT.getSimpleVT();
2383   B.Reg = FuncInfo.CreateReg(B.RegVT);
2384   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2385 
2386   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2387 
2388   addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2389   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2390   SwitchBB->normalizeSuccProbs();
2391 
2392   SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2393                                 MVT::Other, CopyTo, RangeCmp,
2394                                 DAG.getBasicBlock(B.Default));
2395 
2396   // Avoid emitting unnecessary branches to the next block.
2397   if (MBB != NextBlock(SwitchBB))
2398     BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2399                           DAG.getBasicBlock(MBB));
2400 
2401   DAG.setRoot(BrRange);
2402 }
2403 
2404 /// visitBitTestCase - this function produces one "bit test"
2405 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2406                                            MachineBasicBlock* NextMBB,
2407                                            BranchProbability BranchProbToNext,
2408                                            unsigned Reg,
2409                                            BitTestCase &B,
2410                                            MachineBasicBlock *SwitchBB) {
2411   SDLoc dl = getCurSDLoc();
2412   MVT VT = BB.RegVT;
2413   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2414   SDValue Cmp;
2415   unsigned PopCount = countPopulation(B.Mask);
2416   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2417   if (PopCount == 1) {
2418     // Testing for a single bit; just compare the shift count with what it
2419     // would need to be to shift a 1 bit in that position.
2420     Cmp = DAG.getSetCC(
2421         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2422         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2423         ISD::SETEQ);
2424   } else if (PopCount == BB.Range) {
2425     // There is only one zero bit in the range, test for it directly.
2426     Cmp = DAG.getSetCC(
2427         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2428         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2429         ISD::SETNE);
2430   } else {
2431     // Make desired shift
2432     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2433                                     DAG.getConstant(1, dl, VT), ShiftOp);
2434 
2435     // Emit bit tests and jumps
2436     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2437                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2438     Cmp = DAG.getSetCC(
2439         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2440         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2441   }
2442 
2443   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2444   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2445   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2446   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2447   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2448   // one as they are relative probabilities (and thus work more like weights),
2449   // and hence we need to normalize them to let the sum of them become one.
2450   SwitchBB->normalizeSuccProbs();
2451 
2452   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2453                               MVT::Other, getControlRoot(),
2454                               Cmp, DAG.getBasicBlock(B.TargetBB));
2455 
2456   // Avoid emitting unnecessary branches to the next block.
2457   if (NextMBB != NextBlock(SwitchBB))
2458     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2459                         DAG.getBasicBlock(NextMBB));
2460 
2461   DAG.setRoot(BrAnd);
2462 }
2463 
2464 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2465   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2466 
2467   // Retrieve successors. Look through artificial IR level blocks like
2468   // catchswitch for successors.
2469   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2470   const BasicBlock *EHPadBB = I.getSuccessor(1);
2471 
2472   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2473   // have to do anything here to lower funclet bundles.
2474   assert(!I.hasOperandBundlesOtherThan(
2475              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2476          "Cannot lower invokes with arbitrary operand bundles yet!");
2477 
2478   const Value *Callee(I.getCalledValue());
2479   const Function *Fn = dyn_cast<Function>(Callee);
2480   if (isa<InlineAsm>(Callee))
2481     visitInlineAsm(&I);
2482   else if (Fn && Fn->isIntrinsic()) {
2483     switch (Fn->getIntrinsicID()) {
2484     default:
2485       llvm_unreachable("Cannot invoke this intrinsic");
2486     case Intrinsic::donothing:
2487       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2488       break;
2489     case Intrinsic::experimental_patchpoint_void:
2490     case Intrinsic::experimental_patchpoint_i64:
2491       visitPatchpoint(&I, EHPadBB);
2492       break;
2493     case Intrinsic::experimental_gc_statepoint:
2494       LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2495       break;
2496     }
2497   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2498     // Currently we do not lower any intrinsic calls with deopt operand bundles.
2499     // Eventually we will support lowering the @llvm.experimental.deoptimize
2500     // intrinsic, and right now there are no plans to support other intrinsics
2501     // with deopt state.
2502     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2503   } else {
2504     LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2505   }
2506 
2507   // If the value of the invoke is used outside of its defining block, make it
2508   // available as a virtual register.
2509   // We already took care of the exported value for the statepoint instruction
2510   // during call to the LowerStatepoint.
2511   if (!isStatepoint(I)) {
2512     CopyToExportRegsIfNeeded(&I);
2513   }
2514 
2515   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2516   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2517   BranchProbability EHPadBBProb =
2518       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2519           : BranchProbability::getZero();
2520   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2521 
2522   // Update successor info.
2523   addSuccessorWithProb(InvokeMBB, Return);
2524   for (auto &UnwindDest : UnwindDests) {
2525     UnwindDest.first->setIsEHPad();
2526     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2527   }
2528   InvokeMBB->normalizeSuccProbs();
2529 
2530   // Drop into normal successor.
2531   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2532                           MVT::Other, getControlRoot(),
2533                           DAG.getBasicBlock(Return)));
2534 }
2535 
2536 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2537   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2538 }
2539 
2540 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2541   assert(FuncInfo.MBB->isEHPad() &&
2542          "Call to landingpad not in landing pad!");
2543 
2544   // If there aren't registers to copy the values into (e.g., during SjLj
2545   // exceptions), then don't bother to create these DAG nodes.
2546   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2547   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2548   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2549       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2550     return;
2551 
2552   // If landingpad's return type is token type, we don't create DAG nodes
2553   // for its exception pointer and selector value. The extraction of exception
2554   // pointer or selector value from token type landingpads is not currently
2555   // supported.
2556   if (LP.getType()->isTokenTy())
2557     return;
2558 
2559   SmallVector<EVT, 2> ValueVTs;
2560   SDLoc dl = getCurSDLoc();
2561   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2562   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2563 
2564   // Get the two live-in registers as SDValues. The physregs have already been
2565   // copied into virtual registers.
2566   SDValue Ops[2];
2567   if (FuncInfo.ExceptionPointerVirtReg) {
2568     Ops[0] = DAG.getZExtOrTrunc(
2569         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2570                            FuncInfo.ExceptionPointerVirtReg,
2571                            TLI.getPointerTy(DAG.getDataLayout())),
2572         dl, ValueVTs[0]);
2573   } else {
2574     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2575   }
2576   Ops[1] = DAG.getZExtOrTrunc(
2577       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2578                          FuncInfo.ExceptionSelectorVirtReg,
2579                          TLI.getPointerTy(DAG.getDataLayout())),
2580       dl, ValueVTs[1]);
2581 
2582   // Merge into one.
2583   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2584                             DAG.getVTList(ValueVTs), Ops);
2585   setValue(&LP, Res);
2586 }
2587 
2588 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2589 #ifndef NDEBUG
2590   for (const CaseCluster &CC : Clusters)
2591     assert(CC.Low == CC.High && "Input clusters must be single-case");
2592 #endif
2593 
2594   llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) {
2595     return a.Low->getValue().slt(b.Low->getValue());
2596   });
2597 
2598   // Merge adjacent clusters with the same destination.
2599   const unsigned N = Clusters.size();
2600   unsigned DstIndex = 0;
2601   for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2602     CaseCluster &CC = Clusters[SrcIndex];
2603     const ConstantInt *CaseVal = CC.Low;
2604     MachineBasicBlock *Succ = CC.MBB;
2605 
2606     if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2607         (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2608       // If this case has the same successor and is a neighbour, merge it into
2609       // the previous cluster.
2610       Clusters[DstIndex - 1].High = CaseVal;
2611       Clusters[DstIndex - 1].Prob += CC.Prob;
2612     } else {
2613       std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2614                    sizeof(Clusters[SrcIndex]));
2615     }
2616   }
2617   Clusters.resize(DstIndex);
2618 }
2619 
2620 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2621                                            MachineBasicBlock *Last) {
2622   // Update JTCases.
2623   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2624     if (JTCases[i].first.HeaderBB == First)
2625       JTCases[i].first.HeaderBB = Last;
2626 
2627   // Update BitTestCases.
2628   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2629     if (BitTestCases[i].Parent == First)
2630       BitTestCases[i].Parent = Last;
2631 }
2632 
2633 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2634   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2635 
2636   // Update machine-CFG edges with unique successors.
2637   SmallSet<BasicBlock*, 32> Done;
2638   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2639     BasicBlock *BB = I.getSuccessor(i);
2640     bool Inserted = Done.insert(BB).second;
2641     if (!Inserted)
2642         continue;
2643 
2644     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2645     addSuccessorWithProb(IndirectBrMBB, Succ);
2646   }
2647   IndirectBrMBB->normalizeSuccProbs();
2648 
2649   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2650                           MVT::Other, getControlRoot(),
2651                           getValue(I.getAddress())));
2652 }
2653 
2654 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2655   if (!DAG.getTarget().Options.TrapUnreachable)
2656     return;
2657 
2658   // We may be able to ignore unreachable behind a noreturn call.
2659   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
2660     const BasicBlock &BB = *I.getParent();
2661     if (&I != &BB.front()) {
2662       BasicBlock::const_iterator PredI =
2663         std::prev(BasicBlock::const_iterator(&I));
2664       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2665         if (Call->doesNotReturn())
2666           return;
2667       }
2668     }
2669   }
2670 
2671   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2672 }
2673 
2674 void SelectionDAGBuilder::visitFSub(const User &I) {
2675   // -0.0 - X --> fneg
2676   Type *Ty = I.getType();
2677   if (isa<Constant>(I.getOperand(0)) &&
2678       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2679     SDValue Op2 = getValue(I.getOperand(1));
2680     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2681                              Op2.getValueType(), Op2));
2682     return;
2683   }
2684 
2685   visitBinary(I, ISD::FSUB);
2686 }
2687 
2688 /// Checks if the given instruction performs a vector reduction, in which case
2689 /// we have the freedom to alter the elements in the result as long as the
2690 /// reduction of them stays unchanged.
2691 static bool isVectorReductionOp(const User *I) {
2692   const Instruction *Inst = dyn_cast<Instruction>(I);
2693   if (!Inst || !Inst->getType()->isVectorTy())
2694     return false;
2695 
2696   auto OpCode = Inst->getOpcode();
2697   switch (OpCode) {
2698   case Instruction::Add:
2699   case Instruction::Mul:
2700   case Instruction::And:
2701   case Instruction::Or:
2702   case Instruction::Xor:
2703     break;
2704   case Instruction::FAdd:
2705   case Instruction::FMul:
2706     if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2707       if (FPOp->getFastMathFlags().isFast())
2708         break;
2709     LLVM_FALLTHROUGH;
2710   default:
2711     return false;
2712   }
2713 
2714   unsigned ElemNum = Inst->getType()->getVectorNumElements();
2715   // Ensure the reduction size is a power of 2.
2716   if (!isPowerOf2_32(ElemNum))
2717     return false;
2718 
2719   unsigned ElemNumToReduce = ElemNum;
2720 
2721   // Do DFS search on the def-use chain from the given instruction. We only
2722   // allow four kinds of operations during the search until we reach the
2723   // instruction that extracts the first element from the vector:
2724   //
2725   //   1. The reduction operation of the same opcode as the given instruction.
2726   //
2727   //   2. PHI node.
2728   //
2729   //   3. ShuffleVector instruction together with a reduction operation that
2730   //      does a partial reduction.
2731   //
2732   //   4. ExtractElement that extracts the first element from the vector, and we
2733   //      stop searching the def-use chain here.
2734   //
2735   // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2736   // from 1-3 to the stack to continue the DFS. The given instruction is not
2737   // a reduction operation if we meet any other instructions other than those
2738   // listed above.
2739 
2740   SmallVector<const User *, 16> UsersToVisit{Inst};
2741   SmallPtrSet<const User *, 16> Visited;
2742   bool ReduxExtracted = false;
2743 
2744   while (!UsersToVisit.empty()) {
2745     auto User = UsersToVisit.back();
2746     UsersToVisit.pop_back();
2747     if (!Visited.insert(User).second)
2748       continue;
2749 
2750     for (const auto &U : User->users()) {
2751       auto Inst = dyn_cast<Instruction>(U);
2752       if (!Inst)
2753         return false;
2754 
2755       if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2756         if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2757           if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2758             return false;
2759         UsersToVisit.push_back(U);
2760       } else if (const ShuffleVectorInst *ShufInst =
2761                      dyn_cast<ShuffleVectorInst>(U)) {
2762         // Detect the following pattern: A ShuffleVector instruction together
2763         // with a reduction that do partial reduction on the first and second
2764         // ElemNumToReduce / 2 elements, and store the result in
2765         // ElemNumToReduce / 2 elements in another vector.
2766 
2767         unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2768         if (ResultElements < ElemNum)
2769           return false;
2770 
2771         if (ElemNumToReduce == 1)
2772           return false;
2773         if (!isa<UndefValue>(U->getOperand(1)))
2774           return false;
2775         for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2776           if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2777             return false;
2778         for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2779           if (ShufInst->getMaskValue(i) != -1)
2780             return false;
2781 
2782         // There is only one user of this ShuffleVector instruction, which
2783         // must be a reduction operation.
2784         if (!U->hasOneUse())
2785           return false;
2786 
2787         auto U2 = dyn_cast<Instruction>(*U->user_begin());
2788         if (!U2 || U2->getOpcode() != OpCode)
2789           return false;
2790 
2791         // Check operands of the reduction operation.
2792         if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2793             (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2794           UsersToVisit.push_back(U2);
2795           ElemNumToReduce /= 2;
2796         } else
2797           return false;
2798       } else if (isa<ExtractElementInst>(U)) {
2799         // At this moment we should have reduced all elements in the vector.
2800         if (ElemNumToReduce != 1)
2801           return false;
2802 
2803         const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2804         if (!Val || !Val->isZero())
2805           return false;
2806 
2807         ReduxExtracted = true;
2808       } else
2809         return false;
2810     }
2811   }
2812   return ReduxExtracted;
2813 }
2814 
2815 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
2816   SDNodeFlags Flags;
2817 
2818   SDValue Op = getValue(I.getOperand(0));
2819   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
2820                                     Op, Flags);
2821   setValue(&I, UnNodeValue);
2822 }
2823 
2824 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2825   SDNodeFlags Flags;
2826   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2827     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2828     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2829   }
2830   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2831     Flags.setExact(ExactOp->isExact());
2832   }
2833   if (isVectorReductionOp(&I)) {
2834     Flags.setVectorReduction(true);
2835     LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2836   }
2837 
2838   SDValue Op1 = getValue(I.getOperand(0));
2839   SDValue Op2 = getValue(I.getOperand(1));
2840   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2841                                      Op1, Op2, Flags);
2842   setValue(&I, BinNodeValue);
2843 }
2844 
2845 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2846   SDValue Op1 = getValue(I.getOperand(0));
2847   SDValue Op2 = getValue(I.getOperand(1));
2848 
2849   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2850       Op1.getValueType(), DAG.getDataLayout());
2851 
2852   // Coerce the shift amount to the right type if we can.
2853   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2854     unsigned ShiftSize = ShiftTy.getSizeInBits();
2855     unsigned Op2Size = Op2.getValueSizeInBits();
2856     SDLoc DL = getCurSDLoc();
2857 
2858     // If the operand is smaller than the shift count type, promote it.
2859     if (ShiftSize > Op2Size)
2860       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2861 
2862     // If the operand is larger than the shift count type but the shift
2863     // count type has enough bits to represent any shift value, truncate
2864     // it now. This is a common case and it exposes the truncate to
2865     // optimization early.
2866     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2867       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2868     // Otherwise we'll need to temporarily settle for some other convenient
2869     // type.  Type legalization will make adjustments once the shiftee is split.
2870     else
2871       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2872   }
2873 
2874   bool nuw = false;
2875   bool nsw = false;
2876   bool exact = false;
2877 
2878   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2879 
2880     if (const OverflowingBinaryOperator *OFBinOp =
2881             dyn_cast<const OverflowingBinaryOperator>(&I)) {
2882       nuw = OFBinOp->hasNoUnsignedWrap();
2883       nsw = OFBinOp->hasNoSignedWrap();
2884     }
2885     if (const PossiblyExactOperator *ExactOp =
2886             dyn_cast<const PossiblyExactOperator>(&I))
2887       exact = ExactOp->isExact();
2888   }
2889   SDNodeFlags Flags;
2890   Flags.setExact(exact);
2891   Flags.setNoSignedWrap(nsw);
2892   Flags.setNoUnsignedWrap(nuw);
2893   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2894                             Flags);
2895   setValue(&I, Res);
2896 }
2897 
2898 void SelectionDAGBuilder::visitSDiv(const User &I) {
2899   SDValue Op1 = getValue(I.getOperand(0));
2900   SDValue Op2 = getValue(I.getOperand(1));
2901 
2902   SDNodeFlags Flags;
2903   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2904                  cast<PossiblyExactOperator>(&I)->isExact());
2905   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2906                            Op2, Flags));
2907 }
2908 
2909 void SelectionDAGBuilder::visitICmp(const User &I) {
2910   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2911   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2912     predicate = IC->getPredicate();
2913   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2914     predicate = ICmpInst::Predicate(IC->getPredicate());
2915   SDValue Op1 = getValue(I.getOperand(0));
2916   SDValue Op2 = getValue(I.getOperand(1));
2917   ISD::CondCode Opcode = getICmpCondCode(predicate);
2918 
2919   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2920                                                         I.getType());
2921   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2922 }
2923 
2924 void SelectionDAGBuilder::visitFCmp(const User &I) {
2925   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2926   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2927     predicate = FC->getPredicate();
2928   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2929     predicate = FCmpInst::Predicate(FC->getPredicate());
2930   SDValue Op1 = getValue(I.getOperand(0));
2931   SDValue Op2 = getValue(I.getOperand(1));
2932 
2933   ISD::CondCode Condition = getFCmpCondCode(predicate);
2934   auto *FPMO = dyn_cast<FPMathOperator>(&I);
2935   if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2936     Condition = getFCmpCodeWithoutNaN(Condition);
2937 
2938   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2939                                                         I.getType());
2940   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2941 }
2942 
2943 // Check if the condition of the select has one use or two users that are both
2944 // selects with the same condition.
2945 static bool hasOnlySelectUsers(const Value *Cond) {
2946   return llvm::all_of(Cond->users(), [](const Value *V) {
2947     return isa<SelectInst>(V);
2948   });
2949 }
2950 
2951 void SelectionDAGBuilder::visitSelect(const User &I) {
2952   SmallVector<EVT, 4> ValueVTs;
2953   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2954                   ValueVTs);
2955   unsigned NumValues = ValueVTs.size();
2956   if (NumValues == 0) return;
2957 
2958   SmallVector<SDValue, 4> Values(NumValues);
2959   SDValue Cond     = getValue(I.getOperand(0));
2960   SDValue LHSVal   = getValue(I.getOperand(1));
2961   SDValue RHSVal   = getValue(I.getOperand(2));
2962   auto BaseOps = {Cond};
2963   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2964     ISD::VSELECT : ISD::SELECT;
2965 
2966   // Min/max matching is only viable if all output VTs are the same.
2967   if (is_splat(ValueVTs)) {
2968     EVT VT = ValueVTs[0];
2969     LLVMContext &Ctx = *DAG.getContext();
2970     auto &TLI = DAG.getTargetLoweringInfo();
2971 
2972     // We care about the legality of the operation after it has been type
2973     // legalized.
2974     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2975            VT != TLI.getTypeToTransformTo(Ctx, VT))
2976       VT = TLI.getTypeToTransformTo(Ctx, VT);
2977 
2978     // If the vselect is legal, assume we want to leave this as a vector setcc +
2979     // vselect. Otherwise, if this is going to be scalarized, we want to see if
2980     // min/max is legal on the scalar type.
2981     bool UseScalarMinMax = VT.isVector() &&
2982       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2983 
2984     Value *LHS, *RHS;
2985     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2986     ISD::NodeType Opc = ISD::DELETED_NODE;
2987     switch (SPR.Flavor) {
2988     case SPF_UMAX:    Opc = ISD::UMAX; break;
2989     case SPF_UMIN:    Opc = ISD::UMIN; break;
2990     case SPF_SMAX:    Opc = ISD::SMAX; break;
2991     case SPF_SMIN:    Opc = ISD::SMIN; break;
2992     case SPF_FMINNUM:
2993       switch (SPR.NaNBehavior) {
2994       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2995       case SPNB_RETURNS_NAN:   Opc = ISD::FMINIMUM; break;
2996       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2997       case SPNB_RETURNS_ANY: {
2998         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2999           Opc = ISD::FMINNUM;
3000         else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3001           Opc = ISD::FMINIMUM;
3002         else if (UseScalarMinMax)
3003           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3004             ISD::FMINNUM : ISD::FMINIMUM;
3005         break;
3006       }
3007       }
3008       break;
3009     case SPF_FMAXNUM:
3010       switch (SPR.NaNBehavior) {
3011       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3012       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXIMUM; break;
3013       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3014       case SPNB_RETURNS_ANY:
3015 
3016         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3017           Opc = ISD::FMAXNUM;
3018         else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3019           Opc = ISD::FMAXIMUM;
3020         else if (UseScalarMinMax)
3021           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3022             ISD::FMAXNUM : ISD::FMAXIMUM;
3023         break;
3024       }
3025       break;
3026     default: break;
3027     }
3028 
3029     if (Opc != ISD::DELETED_NODE &&
3030         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3031          (UseScalarMinMax &&
3032           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3033         // If the underlying comparison instruction is used by any other
3034         // instruction, the consumed instructions won't be destroyed, so it is
3035         // not profitable to convert to a min/max.
3036         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3037       OpCode = Opc;
3038       LHSVal = getValue(LHS);
3039       RHSVal = getValue(RHS);
3040       BaseOps = {};
3041     }
3042   }
3043 
3044   for (unsigned i = 0; i != NumValues; ++i) {
3045     SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3046     Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3047     Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3048     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3049                             LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3050                             Ops);
3051   }
3052 
3053   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3054                            DAG.getVTList(ValueVTs), Values));
3055 }
3056 
3057 void SelectionDAGBuilder::visitTrunc(const User &I) {
3058   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3059   SDValue N = getValue(I.getOperand(0));
3060   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3061                                                         I.getType());
3062   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3063 }
3064 
3065 void SelectionDAGBuilder::visitZExt(const User &I) {
3066   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3067   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3068   SDValue N = getValue(I.getOperand(0));
3069   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3070                                                         I.getType());
3071   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3072 }
3073 
3074 void SelectionDAGBuilder::visitSExt(const User &I) {
3075   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3076   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3077   SDValue N = getValue(I.getOperand(0));
3078   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3079                                                         I.getType());
3080   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3081 }
3082 
3083 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3084   // FPTrunc is never a no-op cast, no need to check
3085   SDValue N = getValue(I.getOperand(0));
3086   SDLoc dl = getCurSDLoc();
3087   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3088   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3089   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3090                            DAG.getTargetConstant(
3091                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3092 }
3093 
3094 void SelectionDAGBuilder::visitFPExt(const User &I) {
3095   // FPExt is never a no-op cast, no need to check
3096   SDValue N = getValue(I.getOperand(0));
3097   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3098                                                         I.getType());
3099   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3100 }
3101 
3102 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3103   // FPToUI is never a no-op cast, no need to check
3104   SDValue N = getValue(I.getOperand(0));
3105   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3106                                                         I.getType());
3107   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3108 }
3109 
3110 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3111   // FPToSI is never a no-op cast, no need to check
3112   SDValue N = getValue(I.getOperand(0));
3113   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3114                                                         I.getType());
3115   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3116 }
3117 
3118 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3119   // UIToFP is never a no-op cast, no need to check
3120   SDValue N = getValue(I.getOperand(0));
3121   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3122                                                         I.getType());
3123   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3124 }
3125 
3126 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3127   // SIToFP is never a no-op cast, no need to check
3128   SDValue N = getValue(I.getOperand(0));
3129   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3130                                                         I.getType());
3131   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3132 }
3133 
3134 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3135   // What to do depends on the size of the integer and the size of the pointer.
3136   // We can either truncate, zero extend, or no-op, accordingly.
3137   SDValue N = getValue(I.getOperand(0));
3138   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3139                                                         I.getType());
3140   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3141 }
3142 
3143 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3144   // What to do depends on the size of the integer and the size of the pointer.
3145   // We can either truncate, zero extend, or no-op, accordingly.
3146   SDValue N = getValue(I.getOperand(0));
3147   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3148                                                         I.getType());
3149   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3150 }
3151 
3152 void SelectionDAGBuilder::visitBitCast(const User &I) {
3153   SDValue N = getValue(I.getOperand(0));
3154   SDLoc dl = getCurSDLoc();
3155   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3156                                                         I.getType());
3157 
3158   // BitCast assures us that source and destination are the same size so this is
3159   // either a BITCAST or a no-op.
3160   if (DestVT != N.getValueType())
3161     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3162                              DestVT, N)); // convert types.
3163   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3164   // might fold any kind of constant expression to an integer constant and that
3165   // is not what we are looking for. Only recognize a bitcast of a genuine
3166   // constant integer as an opaque constant.
3167   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3168     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3169                                  /*isOpaque*/true));
3170   else
3171     setValue(&I, N);            // noop cast.
3172 }
3173 
3174 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3175   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3176   const Value *SV = I.getOperand(0);
3177   SDValue N = getValue(SV);
3178   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3179 
3180   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3181   unsigned DestAS = I.getType()->getPointerAddressSpace();
3182 
3183   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3184     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3185 
3186   setValue(&I, N);
3187 }
3188 
3189 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3190   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3191   SDValue InVec = getValue(I.getOperand(0));
3192   SDValue InVal = getValue(I.getOperand(1));
3193   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3194                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3195   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3196                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3197                            InVec, InVal, InIdx));
3198 }
3199 
3200 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3201   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3202   SDValue InVec = getValue(I.getOperand(0));
3203   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3204                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3205   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3206                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3207                            InVec, InIdx));
3208 }
3209 
3210 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3211   SDValue Src1 = getValue(I.getOperand(0));
3212   SDValue Src2 = getValue(I.getOperand(1));
3213   SDLoc DL = getCurSDLoc();
3214 
3215   SmallVector<int, 8> Mask;
3216   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3217   unsigned MaskNumElts = Mask.size();
3218 
3219   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3220   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3221   EVT SrcVT = Src1.getValueType();
3222   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3223 
3224   if (SrcNumElts == MaskNumElts) {
3225     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3226     return;
3227   }
3228 
3229   // Normalize the shuffle vector since mask and vector length don't match.
3230   if (SrcNumElts < MaskNumElts) {
3231     // Mask is longer than the source vectors. We can use concatenate vector to
3232     // make the mask and vectors lengths match.
3233 
3234     if (MaskNumElts % SrcNumElts == 0) {
3235       // Mask length is a multiple of the source vector length.
3236       // Check if the shuffle is some kind of concatenation of the input
3237       // vectors.
3238       unsigned NumConcat = MaskNumElts / SrcNumElts;
3239       bool IsConcat = true;
3240       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3241       for (unsigned i = 0; i != MaskNumElts; ++i) {
3242         int Idx = Mask[i];
3243         if (Idx < 0)
3244           continue;
3245         // Ensure the indices in each SrcVT sized piece are sequential and that
3246         // the same source is used for the whole piece.
3247         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3248             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3249              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3250           IsConcat = false;
3251           break;
3252         }
3253         // Remember which source this index came from.
3254         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3255       }
3256 
3257       // The shuffle is concatenating multiple vectors together. Just emit
3258       // a CONCAT_VECTORS operation.
3259       if (IsConcat) {
3260         SmallVector<SDValue, 8> ConcatOps;
3261         for (auto Src : ConcatSrcs) {
3262           if (Src < 0)
3263             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3264           else if (Src == 0)
3265             ConcatOps.push_back(Src1);
3266           else
3267             ConcatOps.push_back(Src2);
3268         }
3269         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3270         return;
3271       }
3272     }
3273 
3274     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3275     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3276     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3277                                     PaddedMaskNumElts);
3278 
3279     // Pad both vectors with undefs to make them the same length as the mask.
3280     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3281 
3282     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3283     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3284     MOps1[0] = Src1;
3285     MOps2[0] = Src2;
3286 
3287     Src1 = Src1.isUndef()
3288                ? DAG.getUNDEF(PaddedVT)
3289                : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3290     Src2 = Src2.isUndef()
3291                ? DAG.getUNDEF(PaddedVT)
3292                : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3293 
3294     // Readjust mask for new input vector length.
3295     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3296     for (unsigned i = 0; i != MaskNumElts; ++i) {
3297       int Idx = Mask[i];
3298       if (Idx >= (int)SrcNumElts)
3299         Idx -= SrcNumElts - PaddedMaskNumElts;
3300       MappedOps[i] = Idx;
3301     }
3302 
3303     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3304 
3305     // If the concatenated vector was padded, extract a subvector with the
3306     // correct number of elements.
3307     if (MaskNumElts != PaddedMaskNumElts)
3308       Result = DAG.getNode(
3309           ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3310           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3311 
3312     setValue(&I, Result);
3313     return;
3314   }
3315 
3316   if (SrcNumElts > MaskNumElts) {
3317     // Analyze the access pattern of the vector to see if we can extract
3318     // two subvectors and do the shuffle.
3319     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3320     bool CanExtract = true;
3321     for (int Idx : Mask) {
3322       unsigned Input = 0;
3323       if (Idx < 0)
3324         continue;
3325 
3326       if (Idx >= (int)SrcNumElts) {
3327         Input = 1;
3328         Idx -= SrcNumElts;
3329       }
3330 
3331       // If all the indices come from the same MaskNumElts sized portion of
3332       // the sources we can use extract. Also make sure the extract wouldn't
3333       // extract past the end of the source.
3334       int NewStartIdx = alignDown(Idx, MaskNumElts);
3335       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3336           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3337         CanExtract = false;
3338       // Make sure we always update StartIdx as we use it to track if all
3339       // elements are undef.
3340       StartIdx[Input] = NewStartIdx;
3341     }
3342 
3343     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3344       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3345       return;
3346     }
3347     if (CanExtract) {
3348       // Extract appropriate subvector and generate a vector shuffle
3349       for (unsigned Input = 0; Input < 2; ++Input) {
3350         SDValue &Src = Input == 0 ? Src1 : Src2;
3351         if (StartIdx[Input] < 0)
3352           Src = DAG.getUNDEF(VT);
3353         else {
3354           Src = DAG.getNode(
3355               ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3356               DAG.getConstant(StartIdx[Input], DL,
3357                               TLI.getVectorIdxTy(DAG.getDataLayout())));
3358         }
3359       }
3360 
3361       // Calculate new mask.
3362       SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3363       for (int &Idx : MappedOps) {
3364         if (Idx >= (int)SrcNumElts)
3365           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3366         else if (Idx >= 0)
3367           Idx -= StartIdx[0];
3368       }
3369 
3370       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3371       return;
3372     }
3373   }
3374 
3375   // We can't use either concat vectors or extract subvectors so fall back to
3376   // replacing the shuffle with extract and build vector.
3377   // to insert and build vector.
3378   EVT EltVT = VT.getVectorElementType();
3379   EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3380   SmallVector<SDValue,8> Ops;
3381   for (int Idx : Mask) {
3382     SDValue Res;
3383 
3384     if (Idx < 0) {
3385       Res = DAG.getUNDEF(EltVT);
3386     } else {
3387       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3388       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3389 
3390       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3391                         EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3392     }
3393 
3394     Ops.push_back(Res);
3395   }
3396 
3397   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3398 }
3399 
3400 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3401   ArrayRef<unsigned> Indices;
3402   if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3403     Indices = IV->getIndices();
3404   else
3405     Indices = cast<ConstantExpr>(&I)->getIndices();
3406 
3407   const Value *Op0 = I.getOperand(0);
3408   const Value *Op1 = I.getOperand(1);
3409   Type *AggTy = I.getType();
3410   Type *ValTy = Op1->getType();
3411   bool IntoUndef = isa<UndefValue>(Op0);
3412   bool FromUndef = isa<UndefValue>(Op1);
3413 
3414   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3415 
3416   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3417   SmallVector<EVT, 4> AggValueVTs;
3418   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3419   SmallVector<EVT, 4> ValValueVTs;
3420   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3421 
3422   unsigned NumAggValues = AggValueVTs.size();
3423   unsigned NumValValues = ValValueVTs.size();
3424   SmallVector<SDValue, 4> Values(NumAggValues);
3425 
3426   // Ignore an insertvalue that produces an empty object
3427   if (!NumAggValues) {
3428     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3429     return;
3430   }
3431 
3432   SDValue Agg = getValue(Op0);
3433   unsigned i = 0;
3434   // Copy the beginning value(s) from the original aggregate.
3435   for (; i != LinearIndex; ++i)
3436     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3437                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3438   // Copy values from the inserted value(s).
3439   if (NumValValues) {
3440     SDValue Val = getValue(Op1);
3441     for (; i != LinearIndex + NumValValues; ++i)
3442       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3443                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3444   }
3445   // Copy remaining value(s) from the original aggregate.
3446   for (; i != NumAggValues; ++i)
3447     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3448                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3449 
3450   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3451                            DAG.getVTList(AggValueVTs), Values));
3452 }
3453 
3454 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3455   ArrayRef<unsigned> Indices;
3456   if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3457     Indices = EV->getIndices();
3458   else
3459     Indices = cast<ConstantExpr>(&I)->getIndices();
3460 
3461   const Value *Op0 = I.getOperand(0);
3462   Type *AggTy = Op0->getType();
3463   Type *ValTy = I.getType();
3464   bool OutOfUndef = isa<UndefValue>(Op0);
3465 
3466   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3467 
3468   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3469   SmallVector<EVT, 4> ValValueVTs;
3470   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3471 
3472   unsigned NumValValues = ValValueVTs.size();
3473 
3474   // Ignore a extractvalue that produces an empty object
3475   if (!NumValValues) {
3476     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3477     return;
3478   }
3479 
3480   SmallVector<SDValue, 4> Values(NumValValues);
3481 
3482   SDValue Agg = getValue(Op0);
3483   // Copy out the selected value(s).
3484   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3485     Values[i - LinearIndex] =
3486       OutOfUndef ?
3487         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3488         SDValue(Agg.getNode(), Agg.getResNo() + i);
3489 
3490   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3491                            DAG.getVTList(ValValueVTs), Values));
3492 }
3493 
3494 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3495   Value *Op0 = I.getOperand(0);
3496   // Note that the pointer operand may be a vector of pointers. Take the scalar
3497   // element which holds a pointer.
3498   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3499   SDValue N = getValue(Op0);
3500   SDLoc dl = getCurSDLoc();
3501 
3502   // Normalize Vector GEP - all scalar operands should be converted to the
3503   // splat vector.
3504   unsigned VectorWidth = I.getType()->isVectorTy() ?
3505     cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3506 
3507   if (VectorWidth && !N.getValueType().isVector()) {
3508     LLVMContext &Context = *DAG.getContext();
3509     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3510     N = DAG.getSplatBuildVector(VT, dl, N);
3511   }
3512 
3513   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3514        GTI != E; ++GTI) {
3515     const Value *Idx = GTI.getOperand();
3516     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3517       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3518       if (Field) {
3519         // N = N + Offset
3520         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3521 
3522         // In an inbounds GEP with an offset that is nonnegative even when
3523         // interpreted as signed, assume there is no unsigned overflow.
3524         SDNodeFlags Flags;
3525         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3526           Flags.setNoUnsignedWrap(true);
3527 
3528         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3529                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3530       }
3531     } else {
3532       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3533       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3534       APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3535 
3536       // If this is a scalar constant or a splat vector of constants,
3537       // handle it quickly.
3538       const auto *CI = dyn_cast<ConstantInt>(Idx);
3539       if (!CI && isa<ConstantDataVector>(Idx) &&
3540           cast<ConstantDataVector>(Idx)->getSplatValue())
3541         CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3542 
3543       if (CI) {
3544         if (CI->isZero())
3545           continue;
3546         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3547         LLVMContext &Context = *DAG.getContext();
3548         SDValue OffsVal = VectorWidth ?
3549           DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3550           DAG.getConstant(Offs, dl, IdxTy);
3551 
3552         // In an inbouds GEP with an offset that is nonnegative even when
3553         // interpreted as signed, assume there is no unsigned overflow.
3554         SDNodeFlags Flags;
3555         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3556           Flags.setNoUnsignedWrap(true);
3557 
3558         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3559         continue;
3560       }
3561 
3562       // N = N + Idx * ElementSize;
3563       SDValue IdxN = getValue(Idx);
3564 
3565       if (!IdxN.getValueType().isVector() && VectorWidth) {
3566         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3567         IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3568       }
3569 
3570       // If the index is smaller or larger than intptr_t, truncate or extend
3571       // it.
3572       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3573 
3574       // If this is a multiply by a power of two, turn it into a shl
3575       // immediately.  This is a very common case.
3576       if (ElementSize != 1) {
3577         if (ElementSize.isPowerOf2()) {
3578           unsigned Amt = ElementSize.logBase2();
3579           IdxN = DAG.getNode(ISD::SHL, dl,
3580                              N.getValueType(), IdxN,
3581                              DAG.getConstant(Amt, dl, IdxN.getValueType()));
3582         } else {
3583           SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3584           IdxN = DAG.getNode(ISD::MUL, dl,
3585                              N.getValueType(), IdxN, Scale);
3586         }
3587       }
3588 
3589       N = DAG.getNode(ISD::ADD, dl,
3590                       N.getValueType(), N, IdxN);
3591     }
3592   }
3593 
3594   setValue(&I, N);
3595 }
3596 
3597 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3598   // If this is a fixed sized alloca in the entry block of the function,
3599   // allocate it statically on the stack.
3600   if (FuncInfo.StaticAllocaMap.count(&I))
3601     return;   // getValue will auto-populate this.
3602 
3603   SDLoc dl = getCurSDLoc();
3604   Type *Ty = I.getAllocatedType();
3605   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3606   auto &DL = DAG.getDataLayout();
3607   uint64_t TySize = DL.getTypeAllocSize(Ty);
3608   unsigned Align =
3609       std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3610 
3611   SDValue AllocSize = getValue(I.getArraySize());
3612 
3613   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3614   if (AllocSize.getValueType() != IntPtr)
3615     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3616 
3617   AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3618                           AllocSize,
3619                           DAG.getConstant(TySize, dl, IntPtr));
3620 
3621   // Handle alignment.  If the requested alignment is less than or equal to
3622   // the stack alignment, ignore it.  If the size is greater than or equal to
3623   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3624   unsigned StackAlign =
3625       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3626   if (Align <= StackAlign)
3627     Align = 0;
3628 
3629   // Round the size of the allocation up to the stack alignment size
3630   // by add SA-1 to the size. This doesn't overflow because we're computing
3631   // an address inside an alloca.
3632   SDNodeFlags Flags;
3633   Flags.setNoUnsignedWrap(true);
3634   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3635                           DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3636 
3637   // Mask out the low bits for alignment purposes.
3638   AllocSize =
3639       DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3640                   DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3641 
3642   SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3643   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3644   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3645   setValue(&I, DSA);
3646   DAG.setRoot(DSA.getValue(1));
3647 
3648   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3649 }
3650 
3651 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3652   if (I.isAtomic())
3653     return visitAtomicLoad(I);
3654 
3655   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3656   const Value *SV = I.getOperand(0);
3657   if (TLI.supportSwiftError()) {
3658     // Swifterror values can come from either a function parameter with
3659     // swifterror attribute or an alloca with swifterror attribute.
3660     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3661       if (Arg->hasSwiftErrorAttr())
3662         return visitLoadFromSwiftError(I);
3663     }
3664 
3665     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3666       if (Alloca->isSwiftError())
3667         return visitLoadFromSwiftError(I);
3668     }
3669   }
3670 
3671   SDValue Ptr = getValue(SV);
3672 
3673   Type *Ty = I.getType();
3674 
3675   bool isVolatile = I.isVolatile();
3676   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3677   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3678   bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3679   unsigned Alignment = I.getAlignment();
3680 
3681   AAMDNodes AAInfo;
3682   I.getAAMetadata(AAInfo);
3683   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3684 
3685   SmallVector<EVT, 4> ValueVTs;
3686   SmallVector<uint64_t, 4> Offsets;
3687   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3688   unsigned NumValues = ValueVTs.size();
3689   if (NumValues == 0)
3690     return;
3691 
3692   SDValue Root;
3693   bool ConstantMemory = false;
3694   if (isVolatile || NumValues > MaxParallelChains)
3695     // Serialize volatile loads with other side effects.
3696     Root = getRoot();
3697   else if (AA &&
3698            AA->pointsToConstantMemory(MemoryLocation(
3699                SV,
3700                LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
3701                AAInfo))) {
3702     // Do not serialize (non-volatile) loads of constant memory with anything.
3703     Root = DAG.getEntryNode();
3704     ConstantMemory = true;
3705   } else {
3706     // Do not serialize non-volatile loads against each other.
3707     Root = DAG.getRoot();
3708   }
3709 
3710   SDLoc dl = getCurSDLoc();
3711 
3712   if (isVolatile)
3713     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3714 
3715   // An aggregate load cannot wrap around the address space, so offsets to its
3716   // parts don't wrap either.
3717   SDNodeFlags Flags;
3718   Flags.setNoUnsignedWrap(true);
3719 
3720   SmallVector<SDValue, 4> Values(NumValues);
3721   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3722   EVT PtrVT = Ptr.getValueType();
3723   unsigned ChainI = 0;
3724   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3725     // Serializing loads here may result in excessive register pressure, and
3726     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3727     // could recover a bit by hoisting nodes upward in the chain by recognizing
3728     // they are side-effect free or do not alias. The optimizer should really
3729     // avoid this case by converting large object/array copies to llvm.memcpy
3730     // (MaxParallelChains should always remain as failsafe).
3731     if (ChainI == MaxParallelChains) {
3732       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3733       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3734                                   makeArrayRef(Chains.data(), ChainI));
3735       Root = Chain;
3736       ChainI = 0;
3737     }
3738     SDValue A = DAG.getNode(ISD::ADD, dl,
3739                             PtrVT, Ptr,
3740                             DAG.getConstant(Offsets[i], dl, PtrVT),
3741                             Flags);
3742     auto MMOFlags = MachineMemOperand::MONone;
3743     if (isVolatile)
3744       MMOFlags |= MachineMemOperand::MOVolatile;
3745     if (isNonTemporal)
3746       MMOFlags |= MachineMemOperand::MONonTemporal;
3747     if (isInvariant)
3748       MMOFlags |= MachineMemOperand::MOInvariant;
3749     if (isDereferenceable)
3750       MMOFlags |= MachineMemOperand::MODereferenceable;
3751     MMOFlags |= TLI.getMMOFlags(I);
3752 
3753     SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3754                             MachinePointerInfo(SV, Offsets[i]), Alignment,
3755                             MMOFlags, AAInfo, Ranges);
3756 
3757     Values[i] = L;
3758     Chains[ChainI] = L.getValue(1);
3759   }
3760 
3761   if (!ConstantMemory) {
3762     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3763                                 makeArrayRef(Chains.data(), ChainI));
3764     if (isVolatile)
3765       DAG.setRoot(Chain);
3766     else
3767       PendingLoads.push_back(Chain);
3768   }
3769 
3770   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3771                            DAG.getVTList(ValueVTs), Values));
3772 }
3773 
3774 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3775   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3776          "call visitStoreToSwiftError when backend supports swifterror");
3777 
3778   SmallVector<EVT, 4> ValueVTs;
3779   SmallVector<uint64_t, 4> Offsets;
3780   const Value *SrcV = I.getOperand(0);
3781   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3782                   SrcV->getType(), ValueVTs, &Offsets);
3783   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3784          "expect a single EVT for swifterror");
3785 
3786   SDValue Src = getValue(SrcV);
3787   // Create a virtual register, then update the virtual register.
3788   unsigned VReg; bool CreatedVReg;
3789   std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3790   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3791   // Chain can be getRoot or getControlRoot.
3792   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3793                                       SDValue(Src.getNode(), Src.getResNo()));
3794   DAG.setRoot(CopyNode);
3795   if (CreatedVReg)
3796     FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3797 }
3798 
3799 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3800   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3801          "call visitLoadFromSwiftError when backend supports swifterror");
3802 
3803   assert(!I.isVolatile() &&
3804          I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3805          I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
3806          "Support volatile, non temporal, invariant for load_from_swift_error");
3807 
3808   const Value *SV = I.getOperand(0);
3809   Type *Ty = I.getType();
3810   AAMDNodes AAInfo;
3811   I.getAAMetadata(AAInfo);
3812   assert(
3813       (!AA ||
3814        !AA->pointsToConstantMemory(MemoryLocation(
3815            SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
3816            AAInfo))) &&
3817       "load_from_swift_error should not be constant memory");
3818 
3819   SmallVector<EVT, 4> ValueVTs;
3820   SmallVector<uint64_t, 4> Offsets;
3821   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3822                   ValueVTs, &Offsets);
3823   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3824          "expect a single EVT for swifterror");
3825 
3826   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3827   SDValue L = DAG.getCopyFromReg(
3828       getRoot(), getCurSDLoc(),
3829       FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3830       ValueVTs[0]);
3831 
3832   setValue(&I, L);
3833 }
3834 
3835 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3836   if (I.isAtomic())
3837     return visitAtomicStore(I);
3838 
3839   const Value *SrcV = I.getOperand(0);
3840   const Value *PtrV = I.getOperand(1);
3841 
3842   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3843   if (TLI.supportSwiftError()) {
3844     // Swifterror values can come from either a function parameter with
3845     // swifterror attribute or an alloca with swifterror attribute.
3846     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3847       if (Arg->hasSwiftErrorAttr())
3848         return visitStoreToSwiftError(I);
3849     }
3850 
3851     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3852       if (Alloca->isSwiftError())
3853         return visitStoreToSwiftError(I);
3854     }
3855   }
3856 
3857   SmallVector<EVT, 4> ValueVTs;
3858   SmallVector<uint64_t, 4> Offsets;
3859   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3860                   SrcV->getType(), ValueVTs, &Offsets);
3861   unsigned NumValues = ValueVTs.size();
3862   if (NumValues == 0)
3863     return;
3864 
3865   // Get the lowered operands. Note that we do this after
3866   // checking if NumResults is zero, because with zero results
3867   // the operands won't have values in the map.
3868   SDValue Src = getValue(SrcV);
3869   SDValue Ptr = getValue(PtrV);
3870 
3871   SDValue Root = getRoot();
3872   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3873   SDLoc dl = getCurSDLoc();
3874   EVT PtrVT = Ptr.getValueType();
3875   unsigned Alignment = I.getAlignment();
3876   AAMDNodes AAInfo;
3877   I.getAAMetadata(AAInfo);
3878 
3879   auto MMOFlags = MachineMemOperand::MONone;
3880   if (I.isVolatile())
3881     MMOFlags |= MachineMemOperand::MOVolatile;
3882   if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3883     MMOFlags |= MachineMemOperand::MONonTemporal;
3884   MMOFlags |= TLI.getMMOFlags(I);
3885 
3886   // An aggregate load cannot wrap around the address space, so offsets to its
3887   // parts don't wrap either.
3888   SDNodeFlags Flags;
3889   Flags.setNoUnsignedWrap(true);
3890 
3891   unsigned ChainI = 0;
3892   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3893     // See visitLoad comments.
3894     if (ChainI == MaxParallelChains) {
3895       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3896                                   makeArrayRef(Chains.data(), ChainI));
3897       Root = Chain;
3898       ChainI = 0;
3899     }
3900     SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3901                               DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3902     SDValue St = DAG.getStore(
3903         Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3904         MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3905     Chains[ChainI] = St;
3906   }
3907 
3908   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3909                                   makeArrayRef(Chains.data(), ChainI));
3910   DAG.setRoot(StoreNode);
3911 }
3912 
3913 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3914                                            bool IsCompressing) {
3915   SDLoc sdl = getCurSDLoc();
3916 
3917   auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3918                            unsigned& Alignment) {
3919     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3920     Src0 = I.getArgOperand(0);
3921     Ptr = I.getArgOperand(1);
3922     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3923     Mask = I.getArgOperand(3);
3924   };
3925   auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3926                            unsigned& Alignment) {
3927     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3928     Src0 = I.getArgOperand(0);
3929     Ptr = I.getArgOperand(1);
3930     Mask = I.getArgOperand(2);
3931     Alignment = 0;
3932   };
3933 
3934   Value  *PtrOperand, *MaskOperand, *Src0Operand;
3935   unsigned Alignment;
3936   if (IsCompressing)
3937     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3938   else
3939     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3940 
3941   SDValue Ptr = getValue(PtrOperand);
3942   SDValue Src0 = getValue(Src0Operand);
3943   SDValue Mask = getValue(MaskOperand);
3944 
3945   EVT VT = Src0.getValueType();
3946   if (!Alignment)
3947     Alignment = DAG.getEVTAlignment(VT);
3948 
3949   AAMDNodes AAInfo;
3950   I.getAAMetadata(AAInfo);
3951 
3952   MachineMemOperand *MMO =
3953     DAG.getMachineFunction().
3954     getMachineMemOperand(MachinePointerInfo(PtrOperand),
3955                           MachineMemOperand::MOStore,  VT.getStoreSize(),
3956                           Alignment, AAInfo);
3957   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3958                                          MMO, false /* Truncating */,
3959                                          IsCompressing);
3960   DAG.setRoot(StoreNode);
3961   setValue(&I, StoreNode);
3962 }
3963 
3964 // Get a uniform base for the Gather/Scatter intrinsic.
3965 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3966 // We try to represent it as a base pointer + vector of indices.
3967 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3968 // The first operand of the GEP may be a single pointer or a vector of pointers
3969 // Example:
3970 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3971 //  or
3972 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
3973 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3974 //
3975 // When the first GEP operand is a single pointer - it is the uniform base we
3976 // are looking for. If first operand of the GEP is a splat vector - we
3977 // extract the splat value and use it as a uniform base.
3978 // In all other cases the function returns 'false'.
3979 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3980                            SDValue &Scale, SelectionDAGBuilder* SDB) {
3981   SelectionDAG& DAG = SDB->DAG;
3982   LLVMContext &Context = *DAG.getContext();
3983 
3984   assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3985   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3986   if (!GEP)
3987     return false;
3988 
3989   const Value *GEPPtr = GEP->getPointerOperand();
3990   if (!GEPPtr->getType()->isVectorTy())
3991     Ptr = GEPPtr;
3992   else if (!(Ptr = getSplatValue(GEPPtr)))
3993     return false;
3994 
3995   unsigned FinalIndex = GEP->getNumOperands() - 1;
3996   Value *IndexVal = GEP->getOperand(FinalIndex);
3997 
3998   // Ensure all the other indices are 0.
3999   for (unsigned i = 1; i < FinalIndex; ++i) {
4000     auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
4001     if (!C || !C->isZero())
4002       return false;
4003   }
4004 
4005   // The operands of the GEP may be defined in another basic block.
4006   // In this case we'll not find nodes for the operands.
4007   if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
4008     return false;
4009 
4010   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4011   const DataLayout &DL = DAG.getDataLayout();
4012   Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4013                                 SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4014   Base = SDB->getValue(Ptr);
4015   Index = SDB->getValue(IndexVal);
4016 
4017   if (!Index.getValueType().isVector()) {
4018     unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4019     EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4020     Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4021   }
4022   return true;
4023 }
4024 
4025 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4026   SDLoc sdl = getCurSDLoc();
4027 
4028   // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4029   const Value *Ptr = I.getArgOperand(1);
4030   SDValue Src0 = getValue(I.getArgOperand(0));
4031   SDValue Mask = getValue(I.getArgOperand(3));
4032   EVT VT = Src0.getValueType();
4033   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4034   if (!Alignment)
4035     Alignment = DAG.getEVTAlignment(VT);
4036   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4037 
4038   AAMDNodes AAInfo;
4039   I.getAAMetadata(AAInfo);
4040 
4041   SDValue Base;
4042   SDValue Index;
4043   SDValue Scale;
4044   const Value *BasePtr = Ptr;
4045   bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4046 
4047   const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4048   MachineMemOperand *MMO = DAG.getMachineFunction().
4049     getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4050                          MachineMemOperand::MOStore,  VT.getStoreSize(),
4051                          Alignment, AAInfo);
4052   if (!UniformBase) {
4053     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4054     Index = getValue(Ptr);
4055     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4056   }
4057   SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4058   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4059                                          Ops, MMO);
4060   DAG.setRoot(Scatter);
4061   setValue(&I, Scatter);
4062 }
4063 
4064 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4065   SDLoc sdl = getCurSDLoc();
4066 
4067   auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4068                            unsigned& Alignment) {
4069     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4070     Ptr = I.getArgOperand(0);
4071     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4072     Mask = I.getArgOperand(2);
4073     Src0 = I.getArgOperand(3);
4074   };
4075   auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4076                            unsigned& Alignment) {
4077     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4078     Ptr = I.getArgOperand(0);
4079     Alignment = 0;
4080     Mask = I.getArgOperand(1);
4081     Src0 = I.getArgOperand(2);
4082   };
4083 
4084   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4085   unsigned Alignment;
4086   if (IsExpanding)
4087     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4088   else
4089     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4090 
4091   SDValue Ptr = getValue(PtrOperand);
4092   SDValue Src0 = getValue(Src0Operand);
4093   SDValue Mask = getValue(MaskOperand);
4094 
4095   EVT VT = Src0.getValueType();
4096   if (!Alignment)
4097     Alignment = DAG.getEVTAlignment(VT);
4098 
4099   AAMDNodes AAInfo;
4100   I.getAAMetadata(AAInfo);
4101   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4102 
4103   // Do not serialize masked loads of constant memory with anything.
4104   bool AddToChain =
4105       !AA || !AA->pointsToConstantMemory(MemoryLocation(
4106                  PtrOperand,
4107                  LocationSize::precise(
4108                      DAG.getDataLayout().getTypeStoreSize(I.getType())),
4109                  AAInfo));
4110   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4111 
4112   MachineMemOperand *MMO =
4113     DAG.getMachineFunction().
4114     getMachineMemOperand(MachinePointerInfo(PtrOperand),
4115                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
4116                           Alignment, AAInfo, Ranges);
4117 
4118   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4119                                    ISD::NON_EXTLOAD, IsExpanding);
4120   if (AddToChain)
4121     PendingLoads.push_back(Load.getValue(1));
4122   setValue(&I, Load);
4123 }
4124 
4125 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4126   SDLoc sdl = getCurSDLoc();
4127 
4128   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4129   const Value *Ptr = I.getArgOperand(0);
4130   SDValue Src0 = getValue(I.getArgOperand(3));
4131   SDValue Mask = getValue(I.getArgOperand(2));
4132 
4133   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4134   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4135   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4136   if (!Alignment)
4137     Alignment = DAG.getEVTAlignment(VT);
4138 
4139   AAMDNodes AAInfo;
4140   I.getAAMetadata(AAInfo);
4141   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4142 
4143   SDValue Root = DAG.getRoot();
4144   SDValue Base;
4145   SDValue Index;
4146   SDValue Scale;
4147   const Value *BasePtr = Ptr;
4148   bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4149   bool ConstantMemory = false;
4150   if (UniformBase && AA &&
4151       AA->pointsToConstantMemory(
4152           MemoryLocation(BasePtr,
4153                          LocationSize::precise(
4154                              DAG.getDataLayout().getTypeStoreSize(I.getType())),
4155                          AAInfo))) {
4156     // Do not serialize (non-volatile) loads of constant memory with anything.
4157     Root = DAG.getEntryNode();
4158     ConstantMemory = true;
4159   }
4160 
4161   MachineMemOperand *MMO =
4162     DAG.getMachineFunction().
4163     getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4164                          MachineMemOperand::MOLoad,  VT.getStoreSize(),
4165                          Alignment, AAInfo, Ranges);
4166 
4167   if (!UniformBase) {
4168     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4169     Index = getValue(Ptr);
4170     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4171   }
4172   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4173   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4174                                        Ops, MMO);
4175 
4176   SDValue OutChain = Gather.getValue(1);
4177   if (!ConstantMemory)
4178     PendingLoads.push_back(OutChain);
4179   setValue(&I, Gather);
4180 }
4181 
4182 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4183   SDLoc dl = getCurSDLoc();
4184   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4185   AtomicOrdering FailureOrder = I.getFailureOrdering();
4186   SyncScope::ID SSID = I.getSyncScopeID();
4187 
4188   SDValue InChain = getRoot();
4189 
4190   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4191   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4192   SDValue L = DAG.getAtomicCmpSwap(
4193       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4194       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4195       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
4196       /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4197 
4198   SDValue OutChain = L.getValue(2);
4199 
4200   setValue(&I, L);
4201   DAG.setRoot(OutChain);
4202 }
4203 
4204 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4205   SDLoc dl = getCurSDLoc();
4206   ISD::NodeType NT;
4207   switch (I.getOperation()) {
4208   default: llvm_unreachable("Unknown atomicrmw operation");
4209   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4210   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4211   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4212   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4213   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4214   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4215   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4216   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4217   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4218   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4219   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4220   }
4221   AtomicOrdering Order = I.getOrdering();
4222   SyncScope::ID SSID = I.getSyncScopeID();
4223 
4224   SDValue InChain = getRoot();
4225 
4226   SDValue L =
4227     DAG.getAtomic(NT, dl,
4228                   getValue(I.getValOperand()).getSimpleValueType(),
4229                   InChain,
4230                   getValue(I.getPointerOperand()),
4231                   getValue(I.getValOperand()),
4232                   I.getPointerOperand(),
4233                   /* Alignment=*/ 0, Order, SSID);
4234 
4235   SDValue OutChain = L.getValue(1);
4236 
4237   setValue(&I, L);
4238   DAG.setRoot(OutChain);
4239 }
4240 
4241 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4242   SDLoc dl = getCurSDLoc();
4243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4244   SDValue Ops[3];
4245   Ops[0] = getRoot();
4246   Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4247                            TLI.getFenceOperandTy(DAG.getDataLayout()));
4248   Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4249                            TLI.getFenceOperandTy(DAG.getDataLayout()));
4250   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4251 }
4252 
4253 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4254   SDLoc dl = getCurSDLoc();
4255   AtomicOrdering Order = I.getOrdering();
4256   SyncScope::ID SSID = I.getSyncScopeID();
4257 
4258   SDValue InChain = getRoot();
4259 
4260   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4261   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4262 
4263   if (!TLI.supportsUnalignedAtomics() &&
4264       I.getAlignment() < VT.getStoreSize())
4265     report_fatal_error("Cannot generate unaligned atomic load");
4266 
4267   MachineMemOperand *MMO =
4268       DAG.getMachineFunction().
4269       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4270                            MachineMemOperand::MOVolatile |
4271                            MachineMemOperand::MOLoad,
4272                            VT.getStoreSize(),
4273                            I.getAlignment() ? I.getAlignment() :
4274                                               DAG.getEVTAlignment(VT),
4275                            AAMDNodes(), nullptr, SSID, Order);
4276 
4277   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4278   SDValue L =
4279       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4280                     getValue(I.getPointerOperand()), MMO);
4281 
4282   SDValue OutChain = L.getValue(1);
4283 
4284   setValue(&I, L);
4285   DAG.setRoot(OutChain);
4286 }
4287 
4288 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4289   SDLoc dl = getCurSDLoc();
4290 
4291   AtomicOrdering Order = I.getOrdering();
4292   SyncScope::ID SSID = I.getSyncScopeID();
4293 
4294   SDValue InChain = getRoot();
4295 
4296   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4297   EVT VT =
4298       TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4299 
4300   if (I.getAlignment() < VT.getStoreSize())
4301     report_fatal_error("Cannot generate unaligned atomic store");
4302 
4303   SDValue OutChain =
4304     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4305                   InChain,
4306                   getValue(I.getPointerOperand()),
4307                   getValue(I.getValueOperand()),
4308                   I.getPointerOperand(), I.getAlignment(),
4309                   Order, SSID);
4310 
4311   DAG.setRoot(OutChain);
4312 }
4313 
4314 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4315 /// node.
4316 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4317                                                unsigned Intrinsic) {
4318   // Ignore the callsite's attributes. A specific call site may be marked with
4319   // readnone, but the lowering code will expect the chain based on the
4320   // definition.
4321   const Function *F = I.getCalledFunction();
4322   bool HasChain = !F->doesNotAccessMemory();
4323   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4324 
4325   // Build the operand list.
4326   SmallVector<SDValue, 8> Ops;
4327   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4328     if (OnlyLoad) {
4329       // We don't need to serialize loads against other loads.
4330       Ops.push_back(DAG.getRoot());
4331     } else {
4332       Ops.push_back(getRoot());
4333     }
4334   }
4335 
4336   // Info is set by getTgtMemInstrinsic
4337   TargetLowering::IntrinsicInfo Info;
4338   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4339   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4340                                                DAG.getMachineFunction(),
4341                                                Intrinsic);
4342 
4343   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4344   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4345       Info.opc == ISD::INTRINSIC_W_CHAIN)
4346     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4347                                         TLI.getPointerTy(DAG.getDataLayout())));
4348 
4349   // Add all operands of the call to the operand list.
4350   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4351     SDValue Op = getValue(I.getArgOperand(i));
4352     Ops.push_back(Op);
4353   }
4354 
4355   SmallVector<EVT, 4> ValueVTs;
4356   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4357 
4358   if (HasChain)
4359     ValueVTs.push_back(MVT::Other);
4360 
4361   SDVTList VTs = DAG.getVTList(ValueVTs);
4362 
4363   // Create the node.
4364   SDValue Result;
4365   if (IsTgtIntrinsic) {
4366     // This is target intrinsic that touches memory
4367     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4368       Ops, Info.memVT,
4369       MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4370       Info.flags, Info.size);
4371   } else if (!HasChain) {
4372     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4373   } else if (!I.getType()->isVoidTy()) {
4374     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4375   } else {
4376     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4377   }
4378 
4379   if (HasChain) {
4380     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4381     if (OnlyLoad)
4382       PendingLoads.push_back(Chain);
4383     else
4384       DAG.setRoot(Chain);
4385   }
4386 
4387   if (!I.getType()->isVoidTy()) {
4388     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4389       EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4390       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4391     } else
4392       Result = lowerRangeToAssertZExt(DAG, I, Result);
4393 
4394     setValue(&I, Result);
4395   }
4396 }
4397 
4398 /// GetSignificand - Get the significand and build it into a floating-point
4399 /// number with exponent of 1:
4400 ///
4401 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4402 ///
4403 /// where Op is the hexadecimal representation of floating point value.
4404 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4405   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4406                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4407   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4408                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4409   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4410 }
4411 
4412 /// GetExponent - Get the exponent:
4413 ///
4414 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4415 ///
4416 /// where Op is the hexadecimal representation of floating point value.
4417 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4418                            const TargetLowering &TLI, const SDLoc &dl) {
4419   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4420                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4421   SDValue t1 = DAG.getNode(
4422       ISD::SRL, dl, MVT::i32, t0,
4423       DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4424   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4425                            DAG.getConstant(127, dl, MVT::i32));
4426   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4427 }
4428 
4429 /// getF32Constant - Get 32-bit floating point constant.
4430 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4431                               const SDLoc &dl) {
4432   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4433                            MVT::f32);
4434 }
4435 
4436 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4437                                        SelectionDAG &DAG) {
4438   // TODO: What fast-math-flags should be set on the floating-point nodes?
4439 
4440   //   IntegerPartOfX = ((int32_t)(t0);
4441   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4442 
4443   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4444   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4445   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4446 
4447   //   IntegerPartOfX <<= 23;
4448   IntegerPartOfX = DAG.getNode(
4449       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4450       DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4451                                   DAG.getDataLayout())));
4452 
4453   SDValue TwoToFractionalPartOfX;
4454   if (LimitFloatPrecision <= 6) {
4455     // For floating-point precision of 6:
4456     //
4457     //   TwoToFractionalPartOfX =
4458     //     0.997535578f +
4459     //       (0.735607626f + 0.252464424f * x) * x;
4460     //
4461     // error 0.0144103317, which is 6 bits
4462     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4463                              getF32Constant(DAG, 0x3e814304, dl));
4464     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4465                              getF32Constant(DAG, 0x3f3c50c8, dl));
4466     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4467     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4468                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
4469   } else if (LimitFloatPrecision <= 12) {
4470     // For floating-point precision of 12:
4471     //
4472     //   TwoToFractionalPartOfX =
4473     //     0.999892986f +
4474     //       (0.696457318f +
4475     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4476     //
4477     // error 0.000107046256, which is 13 to 14 bits
4478     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4479                              getF32Constant(DAG, 0x3da235e3, dl));
4480     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4481                              getF32Constant(DAG, 0x3e65b8f3, dl));
4482     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4483     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4484                              getF32Constant(DAG, 0x3f324b07, dl));
4485     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4486     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4487                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
4488   } else { // LimitFloatPrecision <= 18
4489     // For floating-point precision of 18:
4490     //
4491     //   TwoToFractionalPartOfX =
4492     //     0.999999982f +
4493     //       (0.693148872f +
4494     //         (0.240227044f +
4495     //           (0.554906021e-1f +
4496     //             (0.961591928e-2f +
4497     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4498     // error 2.47208000*10^(-7), which is better than 18 bits
4499     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4500                              getF32Constant(DAG, 0x3924b03e, dl));
4501     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4502                              getF32Constant(DAG, 0x3ab24b87, dl));
4503     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4504     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4505                              getF32Constant(DAG, 0x3c1d8c17, dl));
4506     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4507     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4508                              getF32Constant(DAG, 0x3d634a1d, dl));
4509     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4510     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4511                              getF32Constant(DAG, 0x3e75fe14, dl));
4512     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4513     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4514                               getF32Constant(DAG, 0x3f317234, dl));
4515     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4516     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4517                                          getF32Constant(DAG, 0x3f800000, dl));
4518   }
4519 
4520   // Add the exponent into the result in integer domain.
4521   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4522   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4523                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4524 }
4525 
4526 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4527 /// limited-precision mode.
4528 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4529                          const TargetLowering &TLI) {
4530   if (Op.getValueType() == MVT::f32 &&
4531       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4532 
4533     // Put the exponent in the right bit position for later addition to the
4534     // final result:
4535     //
4536     //   #define LOG2OFe 1.4426950f
4537     //   t0 = Op * LOG2OFe
4538 
4539     // TODO: What fast-math-flags should be set here?
4540     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4541                              getF32Constant(DAG, 0x3fb8aa3b, dl));
4542     return getLimitedPrecisionExp2(t0, dl, DAG);
4543   }
4544 
4545   // No special expansion.
4546   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4547 }
4548 
4549 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4550 /// limited-precision mode.
4551 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4552                          const TargetLowering &TLI) {
4553   // TODO: What fast-math-flags should be set on the floating-point nodes?
4554 
4555   if (Op.getValueType() == MVT::f32 &&
4556       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4557     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4558 
4559     // Scale the exponent by log(2) [0.69314718f].
4560     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4561     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4562                                         getF32Constant(DAG, 0x3f317218, dl));
4563 
4564     // Get the significand and build it into a floating-point number with
4565     // exponent of 1.
4566     SDValue X = GetSignificand(DAG, Op1, dl);
4567 
4568     SDValue LogOfMantissa;
4569     if (LimitFloatPrecision <= 6) {
4570       // For floating-point precision of 6:
4571       //
4572       //   LogofMantissa =
4573       //     -1.1609546f +
4574       //       (1.4034025f - 0.23903021f * x) * x;
4575       //
4576       // error 0.0034276066, which is better than 8 bits
4577       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4578                                getF32Constant(DAG, 0xbe74c456, dl));
4579       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4580                                getF32Constant(DAG, 0x3fb3a2b1, dl));
4581       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4582       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4583                                   getF32Constant(DAG, 0x3f949a29, dl));
4584     } else if (LimitFloatPrecision <= 12) {
4585       // For floating-point precision of 12:
4586       //
4587       //   LogOfMantissa =
4588       //     -1.7417939f +
4589       //       (2.8212026f +
4590       //         (-1.4699568f +
4591       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4592       //
4593       // error 0.000061011436, which is 14 bits
4594       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4595                                getF32Constant(DAG, 0xbd67b6d6, dl));
4596       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4597                                getF32Constant(DAG, 0x3ee4f4b8, dl));
4598       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4599       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4600                                getF32Constant(DAG, 0x3fbc278b, dl));
4601       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4602       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4603                                getF32Constant(DAG, 0x40348e95, dl));
4604       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4605       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4606                                   getF32Constant(DAG, 0x3fdef31a, dl));
4607     } else { // LimitFloatPrecision <= 18
4608       // For floating-point precision of 18:
4609       //
4610       //   LogOfMantissa =
4611       //     -2.1072184f +
4612       //       (4.2372794f +
4613       //         (-3.7029485f +
4614       //           (2.2781945f +
4615       //             (-0.87823314f +
4616       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4617       //
4618       // error 0.0000023660568, which is better than 18 bits
4619       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4620                                getF32Constant(DAG, 0xbc91e5ac, dl));
4621       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4622                                getF32Constant(DAG, 0x3e4350aa, dl));
4623       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4624       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4625                                getF32Constant(DAG, 0x3f60d3e3, dl));
4626       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4627       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4628                                getF32Constant(DAG, 0x4011cdf0, dl));
4629       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4630       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4631                                getF32Constant(DAG, 0x406cfd1c, dl));
4632       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4633       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4634                                getF32Constant(DAG, 0x408797cb, dl));
4635       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4636       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4637                                   getF32Constant(DAG, 0x4006dcab, dl));
4638     }
4639 
4640     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4641   }
4642 
4643   // No special expansion.
4644   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4645 }
4646 
4647 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4648 /// limited-precision mode.
4649 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4650                           const TargetLowering &TLI) {
4651   // TODO: What fast-math-flags should be set on the floating-point nodes?
4652 
4653   if (Op.getValueType() == MVT::f32 &&
4654       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4655     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4656 
4657     // Get the exponent.
4658     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4659 
4660     // Get the significand and build it into a floating-point number with
4661     // exponent of 1.
4662     SDValue X = GetSignificand(DAG, Op1, dl);
4663 
4664     // Different possible minimax approximations of significand in
4665     // floating-point for various degrees of accuracy over [1,2].
4666     SDValue Log2ofMantissa;
4667     if (LimitFloatPrecision <= 6) {
4668       // For floating-point precision of 6:
4669       //
4670       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4671       //
4672       // error 0.0049451742, which is more than 7 bits
4673       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4674                                getF32Constant(DAG, 0xbeb08fe0, dl));
4675       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4676                                getF32Constant(DAG, 0x40019463, dl));
4677       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4678       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4679                                    getF32Constant(DAG, 0x3fd6633d, dl));
4680     } else if (LimitFloatPrecision <= 12) {
4681       // For floating-point precision of 12:
4682       //
4683       //   Log2ofMantissa =
4684       //     -2.51285454f +
4685       //       (4.07009056f +
4686       //         (-2.12067489f +
4687       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4688       //
4689       // error 0.0000876136000, which is better than 13 bits
4690       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4691                                getF32Constant(DAG, 0xbda7262e, dl));
4692       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4693                                getF32Constant(DAG, 0x3f25280b, dl));
4694       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4695       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4696                                getF32Constant(DAG, 0x4007b923, dl));
4697       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4698       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4699                                getF32Constant(DAG, 0x40823e2f, dl));
4700       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4701       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4702                                    getF32Constant(DAG, 0x4020d29c, dl));
4703     } else { // LimitFloatPrecision <= 18
4704       // For floating-point precision of 18:
4705       //
4706       //   Log2ofMantissa =
4707       //     -3.0400495f +
4708       //       (6.1129976f +
4709       //         (-5.3420409f +
4710       //           (3.2865683f +
4711       //             (-1.2669343f +
4712       //               (0.27515199f -
4713       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4714       //
4715       // error 0.0000018516, which is better than 18 bits
4716       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4717                                getF32Constant(DAG, 0xbcd2769e, dl));
4718       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4719                                getF32Constant(DAG, 0x3e8ce0b9, dl));
4720       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4721       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4722                                getF32Constant(DAG, 0x3fa22ae7, dl));
4723       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4724       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4725                                getF32Constant(DAG, 0x40525723, dl));
4726       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4727       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4728                                getF32Constant(DAG, 0x40aaf200, dl));
4729       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4730       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4731                                getF32Constant(DAG, 0x40c39dad, dl));
4732       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4733       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4734                                    getF32Constant(DAG, 0x4042902c, dl));
4735     }
4736 
4737     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4738   }
4739 
4740   // No special expansion.
4741   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4742 }
4743 
4744 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4745 /// limited-precision mode.
4746 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4747                            const TargetLowering &TLI) {
4748   // TODO: What fast-math-flags should be set on the floating-point nodes?
4749 
4750   if (Op.getValueType() == MVT::f32 &&
4751       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4752     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4753 
4754     // Scale the exponent by log10(2) [0.30102999f].
4755     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4756     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4757                                         getF32Constant(DAG, 0x3e9a209a, dl));
4758 
4759     // Get the significand and build it into a floating-point number with
4760     // exponent of 1.
4761     SDValue X = GetSignificand(DAG, Op1, dl);
4762 
4763     SDValue Log10ofMantissa;
4764     if (LimitFloatPrecision <= 6) {
4765       // For floating-point precision of 6:
4766       //
4767       //   Log10ofMantissa =
4768       //     -0.50419619f +
4769       //       (0.60948995f - 0.10380950f * x) * x;
4770       //
4771       // error 0.0014886165, which is 6 bits
4772       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4773                                getF32Constant(DAG, 0xbdd49a13, dl));
4774       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4775                                getF32Constant(DAG, 0x3f1c0789, dl));
4776       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4777       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4778                                     getF32Constant(DAG, 0x3f011300, dl));
4779     } else if (LimitFloatPrecision <= 12) {
4780       // For floating-point precision of 12:
4781       //
4782       //   Log10ofMantissa =
4783       //     -0.64831180f +
4784       //       (0.91751397f +
4785       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4786       //
4787       // error 0.00019228036, which is better than 12 bits
4788       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4789                                getF32Constant(DAG, 0x3d431f31, dl));
4790       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4791                                getF32Constant(DAG, 0x3ea21fb2, dl));
4792       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4793       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4794                                getF32Constant(DAG, 0x3f6ae232, dl));
4795       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4796       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4797                                     getF32Constant(DAG, 0x3f25f7c3, dl));
4798     } else { // LimitFloatPrecision <= 18
4799       // For floating-point precision of 18:
4800       //
4801       //   Log10ofMantissa =
4802       //     -0.84299375f +
4803       //       (1.5327582f +
4804       //         (-1.0688956f +
4805       //           (0.49102474f +
4806       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4807       //
4808       // error 0.0000037995730, which is better than 18 bits
4809       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4810                                getF32Constant(DAG, 0x3c5d51ce, dl));
4811       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4812                                getF32Constant(DAG, 0x3e00685a, dl));
4813       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4814       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4815                                getF32Constant(DAG, 0x3efb6798, dl));
4816       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4817       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4818                                getF32Constant(DAG, 0x3f88d192, dl));
4819       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4820       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4821                                getF32Constant(DAG, 0x3fc4316c, dl));
4822       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4823       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4824                                     getF32Constant(DAG, 0x3f57ce70, dl));
4825     }
4826 
4827     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4828   }
4829 
4830   // No special expansion.
4831   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4832 }
4833 
4834 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4835 /// limited-precision mode.
4836 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4837                           const TargetLowering &TLI) {
4838   if (Op.getValueType() == MVT::f32 &&
4839       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4840     return getLimitedPrecisionExp2(Op, dl, DAG);
4841 
4842   // No special expansion.
4843   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4844 }
4845 
4846 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4847 /// limited-precision mode with x == 10.0f.
4848 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4849                          SelectionDAG &DAG, const TargetLowering &TLI) {
4850   bool IsExp10 = false;
4851   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4852       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4853     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4854       APFloat Ten(10.0f);
4855       IsExp10 = LHSC->isExactlyValue(Ten);
4856     }
4857   }
4858 
4859   // TODO: What fast-math-flags should be set on the FMUL node?
4860   if (IsExp10) {
4861     // Put the exponent in the right bit position for later addition to the
4862     // final result:
4863     //
4864     //   #define LOG2OF10 3.3219281f
4865     //   t0 = Op * LOG2OF10;
4866     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4867                              getF32Constant(DAG, 0x40549a78, dl));
4868     return getLimitedPrecisionExp2(t0, dl, DAG);
4869   }
4870 
4871   // No special expansion.
4872   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4873 }
4874 
4875 /// ExpandPowI - Expand a llvm.powi intrinsic.
4876 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4877                           SelectionDAG &DAG) {
4878   // If RHS is a constant, we can expand this out to a multiplication tree,
4879   // otherwise we end up lowering to a call to __powidf2 (for example).  When
4880   // optimizing for size, we only want to do this if the expansion would produce
4881   // a small number of multiplies, otherwise we do the full expansion.
4882   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4883     // Get the exponent as a positive value.
4884     unsigned Val = RHSC->getSExtValue();
4885     if ((int)Val < 0) Val = -Val;
4886 
4887     // powi(x, 0) -> 1.0
4888     if (Val == 0)
4889       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4890 
4891     const Function &F = DAG.getMachineFunction().getFunction();
4892     if (!F.optForSize() ||
4893         // If optimizing for size, don't insert too many multiplies.
4894         // This inserts up to 5 multiplies.
4895         countPopulation(Val) + Log2_32(Val) < 7) {
4896       // We use the simple binary decomposition method to generate the multiply
4897       // sequence.  There are more optimal ways to do this (for example,
4898       // powi(x,15) generates one more multiply than it should), but this has
4899       // the benefit of being both really simple and much better than a libcall.
4900       SDValue Res;  // Logically starts equal to 1.0
4901       SDValue CurSquare = LHS;
4902       // TODO: Intrinsics should have fast-math-flags that propagate to these
4903       // nodes.
4904       while (Val) {
4905         if (Val & 1) {
4906           if (Res.getNode())
4907             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4908           else
4909             Res = CurSquare;  // 1.0*CurSquare.
4910         }
4911 
4912         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4913                                 CurSquare, CurSquare);
4914         Val >>= 1;
4915       }
4916 
4917       // If the original was negative, invert the result, producing 1/(x*x*x).
4918       if (RHSC->getSExtValue() < 0)
4919         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4920                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4921       return Res;
4922     }
4923   }
4924 
4925   // Otherwise, expand to a libcall.
4926   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4927 }
4928 
4929 // getUnderlyingArgReg - Find underlying register used for a truncated or
4930 // bitcasted argument.
4931 static unsigned getUnderlyingArgReg(const SDValue &N) {
4932   switch (N.getOpcode()) {
4933   case ISD::CopyFromReg:
4934     return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4935   case ISD::BITCAST:
4936   case ISD::AssertZext:
4937   case ISD::AssertSext:
4938   case ISD::TRUNCATE:
4939     return getUnderlyingArgReg(N.getOperand(0));
4940   default:
4941     return 0;
4942   }
4943 }
4944 
4945 /// If the DbgValueInst is a dbg_value of a function argument, create the
4946 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
4947 /// instruction selection, they will be inserted to the entry BB.
4948 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4949     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4950     DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4951   const Argument *Arg = dyn_cast<Argument>(V);
4952   if (!Arg)
4953     return false;
4954 
4955   MachineFunction &MF = DAG.getMachineFunction();
4956   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4957 
4958   bool IsIndirect = false;
4959   Optional<MachineOperand> Op;
4960   // Some arguments' frame index is recorded during argument lowering.
4961   int FI = FuncInfo.getArgumentFrameIndex(Arg);
4962   if (FI != std::numeric_limits<int>::max())
4963     Op = MachineOperand::CreateFI(FI);
4964 
4965   if (!Op && N.getNode()) {
4966     unsigned Reg = getUnderlyingArgReg(N);
4967     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4968       MachineRegisterInfo &RegInfo = MF.getRegInfo();
4969       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4970       if (PR)
4971         Reg = PR;
4972     }
4973     if (Reg) {
4974       Op = MachineOperand::CreateReg(Reg, false);
4975       IsIndirect = IsDbgDeclare;
4976     }
4977   }
4978 
4979   if (!Op && N.getNode())
4980     // Check if frame index is available.
4981     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4982       if (FrameIndexSDNode *FINode =
4983           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4984         Op = MachineOperand::CreateFI(FINode->getIndex());
4985 
4986   if (!Op) {
4987     // Check if ValueMap has reg number.
4988     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4989     if (VMI != FuncInfo.ValueMap.end()) {
4990       const auto &TLI = DAG.getTargetLoweringInfo();
4991       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4992                        V->getType(), getABIRegCopyCC(V));
4993       if (RFV.occupiesMultipleRegs()) {
4994         unsigned Offset = 0;
4995         for (auto RegAndSize : RFV.getRegsAndSizes()) {
4996           Op = MachineOperand::CreateReg(RegAndSize.first, false);
4997           auto FragmentExpr = DIExpression::createFragmentExpression(
4998               Expr, Offset, RegAndSize.second);
4999           if (!FragmentExpr)
5000             continue;
5001           FuncInfo.ArgDbgValues.push_back(
5002               BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
5003                       Op->getReg(), Variable, *FragmentExpr));
5004           Offset += RegAndSize.second;
5005         }
5006         return true;
5007       }
5008       Op = MachineOperand::CreateReg(VMI->second, false);
5009       IsIndirect = IsDbgDeclare;
5010     }
5011   }
5012 
5013   if (!Op)
5014     return false;
5015 
5016   assert(Variable->isValidLocationForIntrinsic(DL) &&
5017          "Expected inlined-at fields to agree");
5018   IsIndirect = (Op->isReg()) ? IsIndirect : true;
5019   FuncInfo.ArgDbgValues.push_back(
5020       BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
5021               *Op, Variable, Expr));
5022 
5023   return true;
5024 }
5025 
5026 /// Return the appropriate SDDbgValue based on N.
5027 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5028                                              DILocalVariable *Variable,
5029                                              DIExpression *Expr,
5030                                              const DebugLoc &dl,
5031                                              unsigned DbgSDNodeOrder) {
5032   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5033     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5034     // stack slot locations.
5035     //
5036     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5037     // debug values here after optimization:
5038     //
5039     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5040     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5041     //
5042     // Both describe the direct values of their associated variables.
5043     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5044                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5045   }
5046   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5047                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5048 }
5049 
5050 // VisualStudio defines setjmp as _setjmp
5051 #if defined(_MSC_VER) && defined(setjmp) && \
5052                          !defined(setjmp_undefined_for_msvc)
5053 #  pragma push_macro("setjmp")
5054 #  undef setjmp
5055 #  define setjmp_undefined_for_msvc
5056 #endif
5057 
5058 /// Lower the call to the specified intrinsic function. If we want to emit this
5059 /// as a call to a named external function, return the name. Otherwise, lower it
5060 /// and return null.
5061 const char *
5062 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
5063   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5064   SDLoc sdl = getCurSDLoc();
5065   DebugLoc dl = getCurDebugLoc();
5066   SDValue Res;
5067 
5068   switch (Intrinsic) {
5069   default:
5070     // By default, turn this into a target intrinsic node.
5071     visitTargetIntrinsic(I, Intrinsic);
5072     return nullptr;
5073   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
5074   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
5075   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
5076   case Intrinsic::returnaddress:
5077     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5078                              TLI.getPointerTy(DAG.getDataLayout()),
5079                              getValue(I.getArgOperand(0))));
5080     return nullptr;
5081   case Intrinsic::addressofreturnaddress:
5082     setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5083                              TLI.getPointerTy(DAG.getDataLayout())));
5084     return nullptr;
5085   case Intrinsic::sponentry:
5086     setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl,
5087                              TLI.getPointerTy(DAG.getDataLayout())));
5088     return nullptr;
5089   case Intrinsic::frameaddress:
5090     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5091                              TLI.getPointerTy(DAG.getDataLayout()),
5092                              getValue(I.getArgOperand(0))));
5093     return nullptr;
5094   case Intrinsic::read_register: {
5095     Value *Reg = I.getArgOperand(0);
5096     SDValue Chain = getRoot();
5097     SDValue RegName =
5098         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5099     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5100     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5101       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5102     setValue(&I, Res);
5103     DAG.setRoot(Res.getValue(1));
5104     return nullptr;
5105   }
5106   case Intrinsic::write_register: {
5107     Value *Reg = I.getArgOperand(0);
5108     Value *RegValue = I.getArgOperand(1);
5109     SDValue Chain = getRoot();
5110     SDValue RegName =
5111         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5112     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5113                             RegName, getValue(RegValue)));
5114     return nullptr;
5115   }
5116   case Intrinsic::setjmp:
5117     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
5118   case Intrinsic::longjmp:
5119     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
5120   case Intrinsic::memcpy: {
5121     const auto &MCI = cast<MemCpyInst>(I);
5122     SDValue Op1 = getValue(I.getArgOperand(0));
5123     SDValue Op2 = getValue(I.getArgOperand(1));
5124     SDValue Op3 = getValue(I.getArgOperand(2));
5125     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5126     unsigned DstAlign = std::max<unsigned>(MCI.getDestAlignment(), 1);
5127     unsigned SrcAlign = std::max<unsigned>(MCI.getSourceAlignment(), 1);
5128     unsigned Align = MinAlign(DstAlign, SrcAlign);
5129     bool isVol = MCI.isVolatile();
5130     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5131     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5132     // node.
5133     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5134                                false, isTC,
5135                                MachinePointerInfo(I.getArgOperand(0)),
5136                                MachinePointerInfo(I.getArgOperand(1)));
5137     updateDAGForMaybeTailCall(MC);
5138     return nullptr;
5139   }
5140   case Intrinsic::memset: {
5141     const auto &MSI = cast<MemSetInst>(I);
5142     SDValue Op1 = getValue(I.getArgOperand(0));
5143     SDValue Op2 = getValue(I.getArgOperand(1));
5144     SDValue Op3 = getValue(I.getArgOperand(2));
5145     // @llvm.memset defines 0 and 1 to both mean no alignment.
5146     unsigned Align = std::max<unsigned>(MSI.getDestAlignment(), 1);
5147     bool isVol = MSI.isVolatile();
5148     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5149     SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5150                                isTC, MachinePointerInfo(I.getArgOperand(0)));
5151     updateDAGForMaybeTailCall(MS);
5152     return nullptr;
5153   }
5154   case Intrinsic::memmove: {
5155     const auto &MMI = cast<MemMoveInst>(I);
5156     SDValue Op1 = getValue(I.getArgOperand(0));
5157     SDValue Op2 = getValue(I.getArgOperand(1));
5158     SDValue Op3 = getValue(I.getArgOperand(2));
5159     // @llvm.memmove defines 0 and 1 to both mean no alignment.
5160     unsigned DstAlign = std::max<unsigned>(MMI.getDestAlignment(), 1);
5161     unsigned SrcAlign = std::max<unsigned>(MMI.getSourceAlignment(), 1);
5162     unsigned Align = MinAlign(DstAlign, SrcAlign);
5163     bool isVol = MMI.isVolatile();
5164     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5165     // FIXME: Support passing different dest/src alignments to the memmove DAG
5166     // node.
5167     SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5168                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
5169                                 MachinePointerInfo(I.getArgOperand(1)));
5170     updateDAGForMaybeTailCall(MM);
5171     return nullptr;
5172   }
5173   case Intrinsic::memcpy_element_unordered_atomic: {
5174     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5175     SDValue Dst = getValue(MI.getRawDest());
5176     SDValue Src = getValue(MI.getRawSource());
5177     SDValue Length = getValue(MI.getLength());
5178 
5179     unsigned DstAlign = MI.getDestAlignment();
5180     unsigned SrcAlign = MI.getSourceAlignment();
5181     Type *LengthTy = MI.getLength()->getType();
5182     unsigned ElemSz = MI.getElementSizeInBytes();
5183     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5184     SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5185                                      SrcAlign, Length, LengthTy, ElemSz, isTC,
5186                                      MachinePointerInfo(MI.getRawDest()),
5187                                      MachinePointerInfo(MI.getRawSource()));
5188     updateDAGForMaybeTailCall(MC);
5189     return nullptr;
5190   }
5191   case Intrinsic::memmove_element_unordered_atomic: {
5192     auto &MI = cast<AtomicMemMoveInst>(I);
5193     SDValue Dst = getValue(MI.getRawDest());
5194     SDValue Src = getValue(MI.getRawSource());
5195     SDValue Length = getValue(MI.getLength());
5196 
5197     unsigned DstAlign = MI.getDestAlignment();
5198     unsigned SrcAlign = MI.getSourceAlignment();
5199     Type *LengthTy = MI.getLength()->getType();
5200     unsigned ElemSz = MI.getElementSizeInBytes();
5201     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5202     SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5203                                       SrcAlign, Length, LengthTy, ElemSz, isTC,
5204                                       MachinePointerInfo(MI.getRawDest()),
5205                                       MachinePointerInfo(MI.getRawSource()));
5206     updateDAGForMaybeTailCall(MC);
5207     return nullptr;
5208   }
5209   case Intrinsic::memset_element_unordered_atomic: {
5210     auto &MI = cast<AtomicMemSetInst>(I);
5211     SDValue Dst = getValue(MI.getRawDest());
5212     SDValue Val = getValue(MI.getValue());
5213     SDValue Length = getValue(MI.getLength());
5214 
5215     unsigned DstAlign = MI.getDestAlignment();
5216     Type *LengthTy = MI.getLength()->getType();
5217     unsigned ElemSz = MI.getElementSizeInBytes();
5218     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5219     SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5220                                      LengthTy, ElemSz, isTC,
5221                                      MachinePointerInfo(MI.getRawDest()));
5222     updateDAGForMaybeTailCall(MC);
5223     return nullptr;
5224   }
5225   case Intrinsic::dbg_addr:
5226   case Intrinsic::dbg_declare: {
5227     const auto &DI = cast<DbgVariableIntrinsic>(I);
5228     DILocalVariable *Variable = DI.getVariable();
5229     DIExpression *Expression = DI.getExpression();
5230     dropDanglingDebugInfo(Variable, Expression);
5231     assert(Variable && "Missing variable");
5232 
5233     // Check if address has undef value.
5234     const Value *Address = DI.getVariableLocation();
5235     if (!Address || isa<UndefValue>(Address) ||
5236         (Address->use_empty() && !isa<Argument>(Address))) {
5237       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5238       return nullptr;
5239     }
5240 
5241     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
5242 
5243     // Check if this variable can be described by a frame index, typically
5244     // either as a static alloca or a byval parameter.
5245     int FI = std::numeric_limits<int>::max();
5246     if (const auto *AI =
5247             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
5248       if (AI->isStaticAlloca()) {
5249         auto I = FuncInfo.StaticAllocaMap.find(AI);
5250         if (I != FuncInfo.StaticAllocaMap.end())
5251           FI = I->second;
5252       }
5253     } else if (const auto *Arg = dyn_cast<Argument>(
5254                    Address->stripInBoundsConstantOffsets())) {
5255       FI = FuncInfo.getArgumentFrameIndex(Arg);
5256     }
5257 
5258     // llvm.dbg.addr is control dependent and always generates indirect
5259     // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
5260     // the MachineFunction variable table.
5261     if (FI != std::numeric_limits<int>::max()) {
5262       if (Intrinsic == Intrinsic::dbg_addr) {
5263         SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
5264             Variable, Expression, FI, /*IsIndirect*/ true, dl, SDNodeOrder);
5265         DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter);
5266       }
5267       return nullptr;
5268     }
5269 
5270     SDValue &N = NodeMap[Address];
5271     if (!N.getNode() && isa<Argument>(Address))
5272       // Check unused arguments map.
5273       N = UnusedArgNodeMap[Address];
5274     SDDbgValue *SDV;
5275     if (N.getNode()) {
5276       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
5277         Address = BCI->getOperand(0);
5278       // Parameters are handled specially.
5279       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
5280       if (isParameter && FINode) {
5281         // Byval parameter. We have a frame index at this point.
5282         SDV =
5283             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
5284                                       /*IsIndirect*/ true, dl, SDNodeOrder);
5285       } else if (isa<Argument>(Address)) {
5286         // Address is an argument, so try to emit its dbg value using
5287         // virtual register info from the FuncInfo.ValueMap.
5288         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N);
5289         return nullptr;
5290       } else {
5291         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5292                               true, dl, SDNodeOrder);
5293       }
5294       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5295     } else {
5296       // If Address is an argument then try to emit its dbg value using
5297       // virtual register info from the FuncInfo.ValueMap.
5298       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true,
5299                                     N)) {
5300         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5301       }
5302     }
5303     return nullptr;
5304   }
5305   case Intrinsic::dbg_label: {
5306     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
5307     DILabel *Label = DI.getLabel();
5308     assert(Label && "Missing label");
5309 
5310     SDDbgLabel *SDV;
5311     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
5312     DAG.AddDbgLabel(SDV);
5313     return nullptr;
5314   }
5315   case Intrinsic::dbg_value: {
5316     const DbgValueInst &DI = cast<DbgValueInst>(I);
5317     assert(DI.getVariable() && "Missing variable");
5318 
5319     DILocalVariable *Variable = DI.getVariable();
5320     DIExpression *Expression = DI.getExpression();
5321     dropDanglingDebugInfo(Variable, Expression);
5322     const Value *V = DI.getValue();
5323     if (!V)
5324       return nullptr;
5325 
5326     SDDbgValue *SDV;
5327     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
5328         isa<ConstantPointerNull>(V)) {
5329       SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder);
5330       DAG.AddDbgValue(SDV, nullptr, false);
5331       return nullptr;
5332     }
5333 
5334     // Do not use getValue() in here; we don't want to generate code at
5335     // this point if it hasn't been done yet.
5336     SDValue N = NodeMap[V];
5337     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
5338       N = UnusedArgNodeMap[V];
5339     if (N.getNode()) {
5340       if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N))
5341         return nullptr;
5342       SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder);
5343       DAG.AddDbgValue(SDV, N.getNode(), false);
5344       return nullptr;
5345     }
5346 
5347     // The value is not used in this block yet (or it would have an SDNode).
5348     // We still want the value to appear for the user if possible -- if it has
5349     // an associated VReg, we can refer to that instead.
5350     if (!isa<Argument>(V)) {
5351       auto VMI = FuncInfo.ValueMap.find(V);
5352       if (VMI != FuncInfo.ValueMap.end()) {
5353         unsigned Reg = VMI->second;
5354         // If this is a PHI node, it may be split up into several MI PHI nodes
5355         // (in FunctionLoweringInfo::set).
5356         RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
5357                          V->getType(), None);
5358         if (RFV.occupiesMultipleRegs()) {
5359           unsigned Offset = 0;
5360           unsigned BitsToDescribe = 0;
5361           if (auto VarSize = Variable->getSizeInBits())
5362             BitsToDescribe = *VarSize;
5363           if (auto Fragment = Expression->getFragmentInfo())
5364             BitsToDescribe = Fragment->SizeInBits;
5365           for (auto RegAndSize : RFV.getRegsAndSizes()) {
5366             unsigned RegisterSize = RegAndSize.second;
5367             // Bail out if all bits are described already.
5368             if (Offset >= BitsToDescribe)
5369               break;
5370             unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
5371                 ? BitsToDescribe - Offset
5372                 : RegisterSize;
5373             auto FragmentExpr = DIExpression::createFragmentExpression(
5374                 Expression, Offset, FragmentSize);
5375             if (!FragmentExpr)
5376                 continue;
5377             SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first,
5378                                       false, dl, SDNodeOrder);
5379             DAG.AddDbgValue(SDV, nullptr, false);
5380             Offset += RegisterSize;
5381           }
5382         } else {
5383           SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl,
5384                                     SDNodeOrder);
5385           DAG.AddDbgValue(SDV, nullptr, false);
5386         }
5387         return nullptr;
5388       }
5389     }
5390 
5391     // TODO: When we get here we will either drop the dbg.value completely, or
5392     // we try to move it forward by letting it dangle for awhile. So we should
5393     // probably add an extra DbgValue to the DAG here, with a reference to
5394     // "noreg", to indicate that we have lost the debug location for the
5395     // variable.
5396 
5397     if (!V->use_empty() ) {
5398       // Do not call getValue(V) yet, as we don't want to generate code.
5399       // Remember it for later.
5400       DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder);
5401       return nullptr;
5402     }
5403 
5404     LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
5405     LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
5406     return nullptr;
5407   }
5408 
5409   case Intrinsic::eh_typeid_for: {
5410     // Find the type id for the given typeinfo.
5411     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5412     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5413     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5414     setValue(&I, Res);
5415     return nullptr;
5416   }
5417 
5418   case Intrinsic::eh_return_i32:
5419   case Intrinsic::eh_return_i64:
5420     DAG.getMachineFunction().setCallsEHReturn(true);
5421     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5422                             MVT::Other,
5423                             getControlRoot(),
5424                             getValue(I.getArgOperand(0)),
5425                             getValue(I.getArgOperand(1))));
5426     return nullptr;
5427   case Intrinsic::eh_unwind_init:
5428     DAG.getMachineFunction().setCallsUnwindInit(true);
5429     return nullptr;
5430   case Intrinsic::eh_dwarf_cfa:
5431     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5432                              TLI.getPointerTy(DAG.getDataLayout()),
5433                              getValue(I.getArgOperand(0))));
5434     return nullptr;
5435   case Intrinsic::eh_sjlj_callsite: {
5436     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5437     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5438     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
5439     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
5440 
5441     MMI.setCurrentCallSite(CI->getZExtValue());
5442     return nullptr;
5443   }
5444   case Intrinsic::eh_sjlj_functioncontext: {
5445     // Get and store the index of the function context.
5446     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5447     AllocaInst *FnCtx =
5448       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5449     int FI = FuncInfo.StaticAllocaMap[FnCtx];
5450     MFI.setFunctionContextIndex(FI);
5451     return nullptr;
5452   }
5453   case Intrinsic::eh_sjlj_setjmp: {
5454     SDValue Ops[2];
5455     Ops[0] = getRoot();
5456     Ops[1] = getValue(I.getArgOperand(0));
5457     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5458                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
5459     setValue(&I, Op.getValue(0));
5460     DAG.setRoot(Op.getValue(1));
5461     return nullptr;
5462   }
5463   case Intrinsic::eh_sjlj_longjmp:
5464     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5465                             getRoot(), getValue(I.getArgOperand(0))));
5466     return nullptr;
5467   case Intrinsic::eh_sjlj_setup_dispatch:
5468     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5469                             getRoot()));
5470     return nullptr;
5471   case Intrinsic::masked_gather:
5472     visitMaskedGather(I);
5473     return nullptr;
5474   case Intrinsic::masked_load:
5475     visitMaskedLoad(I);
5476     return nullptr;
5477   case Intrinsic::masked_scatter:
5478     visitMaskedScatter(I);
5479     return nullptr;
5480   case Intrinsic::masked_store:
5481     visitMaskedStore(I);
5482     return nullptr;
5483   case Intrinsic::masked_expandload:
5484     visitMaskedLoad(I, true /* IsExpanding */);
5485     return nullptr;
5486   case Intrinsic::masked_compressstore:
5487     visitMaskedStore(I, true /* IsCompressing */);
5488     return nullptr;
5489   case Intrinsic::x86_mmx_pslli_w:
5490   case Intrinsic::x86_mmx_pslli_d:
5491   case Intrinsic::x86_mmx_pslli_q:
5492   case Intrinsic::x86_mmx_psrli_w:
5493   case Intrinsic::x86_mmx_psrli_d:
5494   case Intrinsic::x86_mmx_psrli_q:
5495   case Intrinsic::x86_mmx_psrai_w:
5496   case Intrinsic::x86_mmx_psrai_d: {
5497     SDValue ShAmt = getValue(I.getArgOperand(1));
5498     if (isa<ConstantSDNode>(ShAmt)) {
5499       visitTargetIntrinsic(I, Intrinsic);
5500       return nullptr;
5501     }
5502     unsigned NewIntrinsic = 0;
5503     EVT ShAmtVT = MVT::v2i32;
5504     switch (Intrinsic) {
5505     case Intrinsic::x86_mmx_pslli_w:
5506       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5507       break;
5508     case Intrinsic::x86_mmx_pslli_d:
5509       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5510       break;
5511     case Intrinsic::x86_mmx_pslli_q:
5512       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5513       break;
5514     case Intrinsic::x86_mmx_psrli_w:
5515       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5516       break;
5517     case Intrinsic::x86_mmx_psrli_d:
5518       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5519       break;
5520     case Intrinsic::x86_mmx_psrli_q:
5521       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5522       break;
5523     case Intrinsic::x86_mmx_psrai_w:
5524       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5525       break;
5526     case Intrinsic::x86_mmx_psrai_d:
5527       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5528       break;
5529     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
5530     }
5531 
5532     // The vector shift intrinsics with scalars uses 32b shift amounts but
5533     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5534     // to be zero.
5535     // We must do this early because v2i32 is not a legal type.
5536     SDValue ShOps[2];
5537     ShOps[0] = ShAmt;
5538     ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5539     ShAmt =  DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5540     EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5541     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5542     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5543                        DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5544                        getValue(I.getArgOperand(0)), ShAmt);
5545     setValue(&I, Res);
5546     return nullptr;
5547   }
5548   case Intrinsic::powi:
5549     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5550                             getValue(I.getArgOperand(1)), DAG));
5551     return nullptr;
5552   case Intrinsic::log:
5553     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5554     return nullptr;
5555   case Intrinsic::log2:
5556     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5557     return nullptr;
5558   case Intrinsic::log10:
5559     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5560     return nullptr;
5561   case Intrinsic::exp:
5562     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5563     return nullptr;
5564   case Intrinsic::exp2:
5565     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5566     return nullptr;
5567   case Intrinsic::pow:
5568     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5569                            getValue(I.getArgOperand(1)), DAG, TLI));
5570     return nullptr;
5571   case Intrinsic::sqrt:
5572   case Intrinsic::fabs:
5573   case Intrinsic::sin:
5574   case Intrinsic::cos:
5575   case Intrinsic::floor:
5576   case Intrinsic::ceil:
5577   case Intrinsic::trunc:
5578   case Intrinsic::rint:
5579   case Intrinsic::nearbyint:
5580   case Intrinsic::round:
5581   case Intrinsic::canonicalize: {
5582     unsigned Opcode;
5583     switch (Intrinsic) {
5584     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
5585     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
5586     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
5587     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
5588     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
5589     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
5590     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
5591     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
5592     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
5593     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5594     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
5595     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5596     }
5597 
5598     setValue(&I, DAG.getNode(Opcode, sdl,
5599                              getValue(I.getArgOperand(0)).getValueType(),
5600                              getValue(I.getArgOperand(0))));
5601     return nullptr;
5602   }
5603   case Intrinsic::minnum: {
5604     auto VT = getValue(I.getArgOperand(0)).getValueType();
5605     unsigned Opc =
5606         I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT)
5607             ? ISD::FMINIMUM
5608             : ISD::FMINNUM;
5609     setValue(&I, DAG.getNode(Opc, sdl, VT,
5610                              getValue(I.getArgOperand(0)),
5611                              getValue(I.getArgOperand(1))));
5612     return nullptr;
5613   }
5614   case Intrinsic::maxnum: {
5615     auto VT = getValue(I.getArgOperand(0)).getValueType();
5616     unsigned Opc =
5617         I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT)
5618             ? ISD::FMAXIMUM
5619             : ISD::FMAXNUM;
5620     setValue(&I, DAG.getNode(Opc, sdl, VT,
5621                              getValue(I.getArgOperand(0)),
5622                              getValue(I.getArgOperand(1))));
5623     return nullptr;
5624   }
5625   case Intrinsic::minimum:
5626     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
5627                              getValue(I.getArgOperand(0)).getValueType(),
5628                              getValue(I.getArgOperand(0)),
5629                              getValue(I.getArgOperand(1))));
5630     return nullptr;
5631   case Intrinsic::maximum:
5632     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
5633                              getValue(I.getArgOperand(0)).getValueType(),
5634                              getValue(I.getArgOperand(0)),
5635                              getValue(I.getArgOperand(1))));
5636     return nullptr;
5637   case Intrinsic::copysign:
5638     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5639                              getValue(I.getArgOperand(0)).getValueType(),
5640                              getValue(I.getArgOperand(0)),
5641                              getValue(I.getArgOperand(1))));
5642     return nullptr;
5643   case Intrinsic::fma:
5644     setValue(&I, DAG.getNode(ISD::FMA, sdl,
5645                              getValue(I.getArgOperand(0)).getValueType(),
5646                              getValue(I.getArgOperand(0)),
5647                              getValue(I.getArgOperand(1)),
5648                              getValue(I.getArgOperand(2))));
5649     return nullptr;
5650   case Intrinsic::experimental_constrained_fadd:
5651   case Intrinsic::experimental_constrained_fsub:
5652   case Intrinsic::experimental_constrained_fmul:
5653   case Intrinsic::experimental_constrained_fdiv:
5654   case Intrinsic::experimental_constrained_frem:
5655   case Intrinsic::experimental_constrained_fma:
5656   case Intrinsic::experimental_constrained_sqrt:
5657   case Intrinsic::experimental_constrained_pow:
5658   case Intrinsic::experimental_constrained_powi:
5659   case Intrinsic::experimental_constrained_sin:
5660   case Intrinsic::experimental_constrained_cos:
5661   case Intrinsic::experimental_constrained_exp:
5662   case Intrinsic::experimental_constrained_exp2:
5663   case Intrinsic::experimental_constrained_log:
5664   case Intrinsic::experimental_constrained_log10:
5665   case Intrinsic::experimental_constrained_log2:
5666   case Intrinsic::experimental_constrained_rint:
5667   case Intrinsic::experimental_constrained_nearbyint:
5668   case Intrinsic::experimental_constrained_maxnum:
5669   case Intrinsic::experimental_constrained_minnum:
5670   case Intrinsic::experimental_constrained_ceil:
5671   case Intrinsic::experimental_constrained_floor:
5672   case Intrinsic::experimental_constrained_round:
5673   case Intrinsic::experimental_constrained_trunc:
5674     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
5675     return nullptr;
5676   case Intrinsic::fmuladd: {
5677     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5678     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5679         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5680       setValue(&I, DAG.getNode(ISD::FMA, sdl,
5681                                getValue(I.getArgOperand(0)).getValueType(),
5682                                getValue(I.getArgOperand(0)),
5683                                getValue(I.getArgOperand(1)),
5684                                getValue(I.getArgOperand(2))));
5685     } else {
5686       // TODO: Intrinsic calls should have fast-math-flags.
5687       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5688                                 getValue(I.getArgOperand(0)).getValueType(),
5689                                 getValue(I.getArgOperand(0)),
5690                                 getValue(I.getArgOperand(1)));
5691       SDValue Add = DAG.getNode(ISD::FADD, sdl,
5692                                 getValue(I.getArgOperand(0)).getValueType(),
5693                                 Mul,
5694                                 getValue(I.getArgOperand(2)));
5695       setValue(&I, Add);
5696     }
5697     return nullptr;
5698   }
5699   case Intrinsic::convert_to_fp16:
5700     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5701                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5702                                          getValue(I.getArgOperand(0)),
5703                                          DAG.getTargetConstant(0, sdl,
5704                                                                MVT::i32))));
5705     return nullptr;
5706   case Intrinsic::convert_from_fp16:
5707     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5708                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5709                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5710                                          getValue(I.getArgOperand(0)))));
5711     return nullptr;
5712   case Intrinsic::pcmarker: {
5713     SDValue Tmp = getValue(I.getArgOperand(0));
5714     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5715     return nullptr;
5716   }
5717   case Intrinsic::readcyclecounter: {
5718     SDValue Op = getRoot();
5719     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5720                       DAG.getVTList(MVT::i64, MVT::Other), Op);
5721     setValue(&I, Res);
5722     DAG.setRoot(Res.getValue(1));
5723     return nullptr;
5724   }
5725   case Intrinsic::bitreverse:
5726     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5727                              getValue(I.getArgOperand(0)).getValueType(),
5728                              getValue(I.getArgOperand(0))));
5729     return nullptr;
5730   case Intrinsic::bswap:
5731     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5732                              getValue(I.getArgOperand(0)).getValueType(),
5733                              getValue(I.getArgOperand(0))));
5734     return nullptr;
5735   case Intrinsic::cttz: {
5736     SDValue Arg = getValue(I.getArgOperand(0));
5737     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5738     EVT Ty = Arg.getValueType();
5739     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5740                              sdl, Ty, Arg));
5741     return nullptr;
5742   }
5743   case Intrinsic::ctlz: {
5744     SDValue Arg = getValue(I.getArgOperand(0));
5745     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5746     EVT Ty = Arg.getValueType();
5747     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5748                              sdl, Ty, Arg));
5749     return nullptr;
5750   }
5751   case Intrinsic::ctpop: {
5752     SDValue Arg = getValue(I.getArgOperand(0));
5753     EVT Ty = Arg.getValueType();
5754     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5755     return nullptr;
5756   }
5757   case Intrinsic::fshl:
5758   case Intrinsic::fshr: {
5759     bool IsFSHL = Intrinsic == Intrinsic::fshl;
5760     SDValue X = getValue(I.getArgOperand(0));
5761     SDValue Y = getValue(I.getArgOperand(1));
5762     SDValue Z = getValue(I.getArgOperand(2));
5763     EVT VT = X.getValueType();
5764     SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT);
5765     SDValue Zero = DAG.getConstant(0, sdl, VT);
5766     SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC);
5767 
5768     auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
5769     if (TLI.isOperationLegalOrCustom(FunnelOpcode, VT)) {
5770       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
5771       return nullptr;
5772     }
5773 
5774     // When X == Y, this is rotate. If the data type has a power-of-2 size, we
5775     // avoid the select that is necessary in the general case to filter out
5776     // the 0-shift possibility that leads to UB.
5777     if (X == Y && isPowerOf2_32(VT.getScalarSizeInBits())) {
5778       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
5779       if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) {
5780         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
5781         return nullptr;
5782       }
5783 
5784       // Some targets only rotate one way. Try the opposite direction.
5785       RotateOpcode = IsFSHL ? ISD::ROTR : ISD::ROTL;
5786       if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) {
5787         // Negate the shift amount because it is safe to ignore the high bits.
5788         SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
5789         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, NegShAmt));
5790         return nullptr;
5791       }
5792 
5793       // fshl (rotl): (X << (Z % BW)) | (X >> ((0 - Z) % BW))
5794       // fshr (rotr): (X << ((0 - Z) % BW)) | (X >> (Z % BW))
5795       SDValue NegZ = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
5796       SDValue NShAmt = DAG.getNode(ISD::UREM, sdl, VT, NegZ, BitWidthC);
5797       SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : NShAmt);
5798       SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, X, IsFSHL ? NShAmt : ShAmt);
5799       setValue(&I, DAG.getNode(ISD::OR, sdl, VT, ShX, ShY));
5800       return nullptr;
5801     }
5802 
5803     // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
5804     // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
5805     SDValue InvShAmt = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, ShAmt);
5806     SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt);
5807     SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt);
5808     SDValue Or = DAG.getNode(ISD::OR, sdl, VT, ShX, ShY);
5809 
5810     // If (Z % BW == 0), then the opposite direction shift is shift-by-bitwidth,
5811     // and that is undefined. We must compare and select to avoid UB.
5812     EVT CCVT = MVT::i1;
5813     if (VT.isVector())
5814       CCVT = EVT::getVectorVT(*Context, CCVT, VT.getVectorNumElements());
5815 
5816     // For fshl, 0-shift returns the 1st arg (X).
5817     // For fshr, 0-shift returns the 2nd arg (Y).
5818     SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ);
5819     setValue(&I, DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Or));
5820     return nullptr;
5821   }
5822   case Intrinsic::sadd_sat: {
5823     SDValue Op1 = getValue(I.getArgOperand(0));
5824     SDValue Op2 = getValue(I.getArgOperand(1));
5825     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
5826     return nullptr;
5827   }
5828   case Intrinsic::uadd_sat: {
5829     SDValue Op1 = getValue(I.getArgOperand(0));
5830     SDValue Op2 = getValue(I.getArgOperand(1));
5831     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
5832     return nullptr;
5833   }
5834   case Intrinsic::ssub_sat: {
5835     SDValue Op1 = getValue(I.getArgOperand(0));
5836     SDValue Op2 = getValue(I.getArgOperand(1));
5837     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
5838     return nullptr;
5839   }
5840   case Intrinsic::usub_sat: {
5841     SDValue Op1 = getValue(I.getArgOperand(0));
5842     SDValue Op2 = getValue(I.getArgOperand(1));
5843     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
5844     return nullptr;
5845   }
5846   case Intrinsic::smul_fix: {
5847     SDValue Op1 = getValue(I.getArgOperand(0));
5848     SDValue Op2 = getValue(I.getArgOperand(1));
5849     SDValue Op3 = getValue(I.getArgOperand(2));
5850     setValue(&I,
5851              DAG.getNode(ISD::SMULFIX, sdl, Op1.getValueType(), Op1, Op2, Op3));
5852     return nullptr;
5853   }
5854   case Intrinsic::stacksave: {
5855     SDValue Op = getRoot();
5856     Res = DAG.getNode(
5857         ISD::STACKSAVE, sdl,
5858         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5859     setValue(&I, Res);
5860     DAG.setRoot(Res.getValue(1));
5861     return nullptr;
5862   }
5863   case Intrinsic::stackrestore:
5864     Res = getValue(I.getArgOperand(0));
5865     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5866     return nullptr;
5867   case Intrinsic::get_dynamic_area_offset: {
5868     SDValue Op = getRoot();
5869     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5870     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5871     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5872     // target.
5873     if (PtrTy != ResTy)
5874       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5875                          " intrinsic!");
5876     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5877                       Op);
5878     DAG.setRoot(Op);
5879     setValue(&I, Res);
5880     return nullptr;
5881   }
5882   case Intrinsic::stackguard: {
5883     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5884     MachineFunction &MF = DAG.getMachineFunction();
5885     const Module &M = *MF.getFunction().getParent();
5886     SDValue Chain = getRoot();
5887     if (TLI.useLoadStackGuardNode()) {
5888       Res = getLoadStackGuard(DAG, sdl, Chain);
5889     } else {
5890       const Value *Global = TLI.getSDagStackGuard(M);
5891       unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5892       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5893                         MachinePointerInfo(Global, 0), Align,
5894                         MachineMemOperand::MOVolatile);
5895     }
5896     if (TLI.useStackGuardXorFP())
5897       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
5898     DAG.setRoot(Chain);
5899     setValue(&I, Res);
5900     return nullptr;
5901   }
5902   case Intrinsic::stackprotector: {
5903     // Emit code into the DAG to store the stack guard onto the stack.
5904     MachineFunction &MF = DAG.getMachineFunction();
5905     MachineFrameInfo &MFI = MF.getFrameInfo();
5906     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5907     SDValue Src, Chain = getRoot();
5908 
5909     if (TLI.useLoadStackGuardNode())
5910       Src = getLoadStackGuard(DAG, sdl, Chain);
5911     else
5912       Src = getValue(I.getArgOperand(0));   // The guard's value.
5913 
5914     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5915 
5916     int FI = FuncInfo.StaticAllocaMap[Slot];
5917     MFI.setStackProtectorIndex(FI);
5918 
5919     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5920 
5921     // Store the stack protector onto the stack.
5922     Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5923                                                  DAG.getMachineFunction(), FI),
5924                        /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5925     setValue(&I, Res);
5926     DAG.setRoot(Res);
5927     return nullptr;
5928   }
5929   case Intrinsic::objectsize: {
5930     // If we don't know by now, we're never going to know.
5931     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5932 
5933     assert(CI && "Non-constant type in __builtin_object_size?");
5934 
5935     SDValue Arg = getValue(I.getCalledValue());
5936     EVT Ty = Arg.getValueType();
5937 
5938     if (CI->isZero())
5939       Res = DAG.getConstant(-1ULL, sdl, Ty);
5940     else
5941       Res = DAG.getConstant(0, sdl, Ty);
5942 
5943     setValue(&I, Res);
5944     return nullptr;
5945   }
5946 
5947   case Intrinsic::is_constant:
5948     // If this wasn't constant-folded away by now, then it's not a
5949     // constant.
5950     setValue(&I, DAG.getConstant(0, sdl, MVT::i1));
5951     return nullptr;
5952 
5953   case Intrinsic::annotation:
5954   case Intrinsic::ptr_annotation:
5955   case Intrinsic::launder_invariant_group:
5956   case Intrinsic::strip_invariant_group:
5957     // Drop the intrinsic, but forward the value
5958     setValue(&I, getValue(I.getOperand(0)));
5959     return nullptr;
5960   case Intrinsic::assume:
5961   case Intrinsic::var_annotation:
5962   case Intrinsic::sideeffect:
5963     // Discard annotate attributes, assumptions, and artificial side-effects.
5964     return nullptr;
5965 
5966   case Intrinsic::codeview_annotation: {
5967     // Emit a label associated with this metadata.
5968     MachineFunction &MF = DAG.getMachineFunction();
5969     MCSymbol *Label =
5970         MF.getMMI().getContext().createTempSymbol("annotation", true);
5971     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
5972     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
5973     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
5974     DAG.setRoot(Res);
5975     return nullptr;
5976   }
5977 
5978   case Intrinsic::init_trampoline: {
5979     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5980 
5981     SDValue Ops[6];
5982     Ops[0] = getRoot();
5983     Ops[1] = getValue(I.getArgOperand(0));
5984     Ops[2] = getValue(I.getArgOperand(1));
5985     Ops[3] = getValue(I.getArgOperand(2));
5986     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5987     Ops[5] = DAG.getSrcValue(F);
5988 
5989     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5990 
5991     DAG.setRoot(Res);
5992     return nullptr;
5993   }
5994   case Intrinsic::adjust_trampoline:
5995     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5996                              TLI.getPointerTy(DAG.getDataLayout()),
5997                              getValue(I.getArgOperand(0))));
5998     return nullptr;
5999   case Intrinsic::gcroot: {
6000     assert(DAG.getMachineFunction().getFunction().hasGC() &&
6001            "only valid in functions with gc specified, enforced by Verifier");
6002     assert(GFI && "implied by previous");
6003     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
6004     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
6005 
6006     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
6007     GFI->addStackRoot(FI->getIndex(), TypeMap);
6008     return nullptr;
6009   }
6010   case Intrinsic::gcread:
6011   case Intrinsic::gcwrite:
6012     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
6013   case Intrinsic::flt_rounds:
6014     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
6015     return nullptr;
6016 
6017   case Intrinsic::expect:
6018     // Just replace __builtin_expect(exp, c) with EXP.
6019     setValue(&I, getValue(I.getArgOperand(0)));
6020     return nullptr;
6021 
6022   case Intrinsic::debugtrap:
6023   case Intrinsic::trap: {
6024     StringRef TrapFuncName =
6025         I.getAttributes()
6026             .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
6027             .getValueAsString();
6028     if (TrapFuncName.empty()) {
6029       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
6030         ISD::TRAP : ISD::DEBUGTRAP;
6031       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
6032       return nullptr;
6033     }
6034     TargetLowering::ArgListTy Args;
6035 
6036     TargetLowering::CallLoweringInfo CLI(DAG);
6037     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6038         CallingConv::C, I.getType(),
6039         DAG.getExternalSymbol(TrapFuncName.data(),
6040                               TLI.getPointerTy(DAG.getDataLayout())),
6041         std::move(Args));
6042 
6043     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6044     DAG.setRoot(Result.second);
6045     return nullptr;
6046   }
6047 
6048   case Intrinsic::uadd_with_overflow:
6049   case Intrinsic::sadd_with_overflow:
6050   case Intrinsic::usub_with_overflow:
6051   case Intrinsic::ssub_with_overflow:
6052   case Intrinsic::umul_with_overflow:
6053   case Intrinsic::smul_with_overflow: {
6054     ISD::NodeType Op;
6055     switch (Intrinsic) {
6056     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6057     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6058     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6059     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6060     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6061     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6062     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6063     }
6064     SDValue Op1 = getValue(I.getArgOperand(0));
6065     SDValue Op2 = getValue(I.getArgOperand(1));
6066 
6067     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
6068     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6069     return nullptr;
6070   }
6071   case Intrinsic::prefetch: {
6072     SDValue Ops[5];
6073     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6074     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6075     Ops[0] = DAG.getRoot();
6076     Ops[1] = getValue(I.getArgOperand(0));
6077     Ops[2] = getValue(I.getArgOperand(1));
6078     Ops[3] = getValue(I.getArgOperand(2));
6079     Ops[4] = getValue(I.getArgOperand(3));
6080     SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
6081                                              DAG.getVTList(MVT::Other), Ops,
6082                                              EVT::getIntegerVT(*Context, 8),
6083                                              MachinePointerInfo(I.getArgOperand(0)),
6084                                              0, /* align */
6085                                              Flags);
6086 
6087     // Chain the prefetch in parallell with any pending loads, to stay out of
6088     // the way of later optimizations.
6089     PendingLoads.push_back(Result);
6090     Result = getRoot();
6091     DAG.setRoot(Result);
6092     return nullptr;
6093   }
6094   case Intrinsic::lifetime_start:
6095   case Intrinsic::lifetime_end: {
6096     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6097     // Stack coloring is not enabled in O0, discard region information.
6098     if (TM.getOptLevel() == CodeGenOpt::None)
6099       return nullptr;
6100 
6101     SmallVector<Value *, 4> Allocas;
6102     GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
6103 
6104     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
6105            E = Allocas.end(); Object != E; ++Object) {
6106       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
6107 
6108       // Could not find an Alloca.
6109       if (!LifetimeObject)
6110         continue;
6111 
6112       // First check that the Alloca is static, otherwise it won't have a
6113       // valid frame index.
6114       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6115       if (SI == FuncInfo.StaticAllocaMap.end())
6116         return nullptr;
6117 
6118       int FI = SI->second;
6119 
6120       SDValue Ops[2];
6121       Ops[0] = getRoot();
6122       Ops[1] =
6123           DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
6124       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
6125 
6126       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
6127       DAG.setRoot(Res);
6128     }
6129     return nullptr;
6130   }
6131   case Intrinsic::invariant_start:
6132     // Discard region information.
6133     setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
6134     return nullptr;
6135   case Intrinsic::invariant_end:
6136     // Discard region information.
6137     return nullptr;
6138   case Intrinsic::clear_cache:
6139     return TLI.getClearCacheBuiltinName();
6140   case Intrinsic::donothing:
6141     // ignore
6142     return nullptr;
6143   case Intrinsic::experimental_stackmap:
6144     visitStackmap(I);
6145     return nullptr;
6146   case Intrinsic::experimental_patchpoint_void:
6147   case Intrinsic::experimental_patchpoint_i64:
6148     visitPatchpoint(&I);
6149     return nullptr;
6150   case Intrinsic::experimental_gc_statepoint:
6151     LowerStatepoint(ImmutableStatepoint(&I));
6152     return nullptr;
6153   case Intrinsic::experimental_gc_result:
6154     visitGCResult(cast<GCResultInst>(I));
6155     return nullptr;
6156   case Intrinsic::experimental_gc_relocate:
6157     visitGCRelocate(cast<GCRelocateInst>(I));
6158     return nullptr;
6159   case Intrinsic::instrprof_increment:
6160     llvm_unreachable("instrprof failed to lower an increment");
6161   case Intrinsic::instrprof_value_profile:
6162     llvm_unreachable("instrprof failed to lower a value profiling call");
6163   case Intrinsic::localescape: {
6164     MachineFunction &MF = DAG.getMachineFunction();
6165     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6166 
6167     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6168     // is the same on all targets.
6169     for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
6170       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6171       if (isa<ConstantPointerNull>(Arg))
6172         continue; // Skip null pointers. They represent a hole in index space.
6173       AllocaInst *Slot = cast<AllocaInst>(Arg);
6174       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
6175              "can only escape static allocas");
6176       int FI = FuncInfo.StaticAllocaMap[Slot];
6177       MCSymbol *FrameAllocSym =
6178           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6179               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6180       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6181               TII->get(TargetOpcode::LOCAL_ESCAPE))
6182           .addSym(FrameAllocSym)
6183           .addFrameIndex(FI);
6184     }
6185 
6186     MF.setHasLocalEscape(true);
6187 
6188     return nullptr;
6189   }
6190 
6191   case Intrinsic::localrecover: {
6192     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6193     MachineFunction &MF = DAG.getMachineFunction();
6194     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
6195 
6196     // Get the symbol that defines the frame offset.
6197     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6198     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6199     unsigned IdxVal =
6200         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6201     MCSymbol *FrameAllocSym =
6202         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6203             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6204 
6205     // Create a MCSymbol for the label to avoid any target lowering
6206     // that would make this PC relative.
6207     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6208     SDValue OffsetVal =
6209         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6210 
6211     // Add the offset to the FP.
6212     Value *FP = I.getArgOperand(1);
6213     SDValue FPVal = getValue(FP);
6214     SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
6215     setValue(&I, Add);
6216 
6217     return nullptr;
6218   }
6219 
6220   case Intrinsic::eh_exceptionpointer:
6221   case Intrinsic::eh_exceptioncode: {
6222     // Get the exception pointer vreg, copy from it, and resize it to fit.
6223     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
6224     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
6225     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
6226     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
6227     SDValue N =
6228         DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
6229     if (Intrinsic == Intrinsic::eh_exceptioncode)
6230       N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
6231     setValue(&I, N);
6232     return nullptr;
6233   }
6234   case Intrinsic::xray_customevent: {
6235     // Here we want to make sure that the intrinsic behaves as if it has a
6236     // specific calling convention, and only for x86_64.
6237     // FIXME: Support other platforms later.
6238     const auto &Triple = DAG.getTarget().getTargetTriple();
6239     if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6240       return nullptr;
6241 
6242     SDLoc DL = getCurSDLoc();
6243     SmallVector<SDValue, 8> Ops;
6244 
6245     // We want to say that we always want the arguments in registers.
6246     SDValue LogEntryVal = getValue(I.getArgOperand(0));
6247     SDValue StrSizeVal = getValue(I.getArgOperand(1));
6248     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6249     SDValue Chain = getRoot();
6250     Ops.push_back(LogEntryVal);
6251     Ops.push_back(StrSizeVal);
6252     Ops.push_back(Chain);
6253 
6254     // We need to enforce the calling convention for the callsite, so that
6255     // argument ordering is enforced correctly, and that register allocation can
6256     // see that some registers may be assumed clobbered and have to preserve
6257     // them across calls to the intrinsic.
6258     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
6259                                            DL, NodeTys, Ops);
6260     SDValue patchableNode = SDValue(MN, 0);
6261     DAG.setRoot(patchableNode);
6262     setValue(&I, patchableNode);
6263     return nullptr;
6264   }
6265   case Intrinsic::xray_typedevent: {
6266     // Here we want to make sure that the intrinsic behaves as if it has a
6267     // specific calling convention, and only for x86_64.
6268     // FIXME: Support other platforms later.
6269     const auto &Triple = DAG.getTarget().getTargetTriple();
6270     if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6271       return nullptr;
6272 
6273     SDLoc DL = getCurSDLoc();
6274     SmallVector<SDValue, 8> Ops;
6275 
6276     // We want to say that we always want the arguments in registers.
6277     // It's unclear to me how manipulating the selection DAG here forces callers
6278     // to provide arguments in registers instead of on the stack.
6279     SDValue LogTypeId = getValue(I.getArgOperand(0));
6280     SDValue LogEntryVal = getValue(I.getArgOperand(1));
6281     SDValue StrSizeVal = getValue(I.getArgOperand(2));
6282     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6283     SDValue Chain = getRoot();
6284     Ops.push_back(LogTypeId);
6285     Ops.push_back(LogEntryVal);
6286     Ops.push_back(StrSizeVal);
6287     Ops.push_back(Chain);
6288 
6289     // We need to enforce the calling convention for the callsite, so that
6290     // argument ordering is enforced correctly, and that register allocation can
6291     // see that some registers may be assumed clobbered and have to preserve
6292     // them across calls to the intrinsic.
6293     MachineSDNode *MN = DAG.getMachineNode(
6294         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, DL, NodeTys, Ops);
6295     SDValue patchableNode = SDValue(MN, 0);
6296     DAG.setRoot(patchableNode);
6297     setValue(&I, patchableNode);
6298     return nullptr;
6299   }
6300   case Intrinsic::experimental_deoptimize:
6301     LowerDeoptimizeCall(&I);
6302     return nullptr;
6303 
6304   case Intrinsic::experimental_vector_reduce_fadd:
6305   case Intrinsic::experimental_vector_reduce_fmul:
6306   case Intrinsic::experimental_vector_reduce_add:
6307   case Intrinsic::experimental_vector_reduce_mul:
6308   case Intrinsic::experimental_vector_reduce_and:
6309   case Intrinsic::experimental_vector_reduce_or:
6310   case Intrinsic::experimental_vector_reduce_xor:
6311   case Intrinsic::experimental_vector_reduce_smax:
6312   case Intrinsic::experimental_vector_reduce_smin:
6313   case Intrinsic::experimental_vector_reduce_umax:
6314   case Intrinsic::experimental_vector_reduce_umin:
6315   case Intrinsic::experimental_vector_reduce_fmax:
6316   case Intrinsic::experimental_vector_reduce_fmin:
6317     visitVectorReduce(I, Intrinsic);
6318     return nullptr;
6319 
6320   case Intrinsic::icall_branch_funnel: {
6321     SmallVector<SDValue, 16> Ops;
6322     Ops.push_back(DAG.getRoot());
6323     Ops.push_back(getValue(I.getArgOperand(0)));
6324 
6325     int64_t Offset;
6326     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6327         I.getArgOperand(1), Offset, DAG.getDataLayout()));
6328     if (!Base)
6329       report_fatal_error(
6330           "llvm.icall.branch.funnel operand must be a GlobalValue");
6331     Ops.push_back(DAG.getTargetGlobalAddress(Base, getCurSDLoc(), MVT::i64, 0));
6332 
6333     struct BranchFunnelTarget {
6334       int64_t Offset;
6335       SDValue Target;
6336     };
6337     SmallVector<BranchFunnelTarget, 8> Targets;
6338 
6339     for (unsigned Op = 1, N = I.getNumArgOperands(); Op != N; Op += 2) {
6340       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6341           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
6342       if (ElemBase != Base)
6343         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
6344                            "to the same GlobalValue");
6345 
6346       SDValue Val = getValue(I.getArgOperand(Op + 1));
6347       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
6348       if (!GA)
6349         report_fatal_error(
6350             "llvm.icall.branch.funnel operand must be a GlobalValue");
6351       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
6352                                      GA->getGlobal(), getCurSDLoc(),
6353                                      Val.getValueType(), GA->getOffset())});
6354     }
6355     llvm::sort(Targets,
6356                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
6357                  return T1.Offset < T2.Offset;
6358                });
6359 
6360     for (auto &T : Targets) {
6361       Ops.push_back(DAG.getTargetConstant(T.Offset, getCurSDLoc(), MVT::i32));
6362       Ops.push_back(T.Target);
6363     }
6364 
6365     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL,
6366                                  getCurSDLoc(), MVT::Other, Ops),
6367               0);
6368     DAG.setRoot(N);
6369     setValue(&I, N);
6370     HasTailCall = true;
6371     return nullptr;
6372   }
6373 
6374   case Intrinsic::wasm_landingpad_index:
6375     // Information this intrinsic contained has been transferred to
6376     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
6377     // delete it now.
6378     return nullptr;
6379   }
6380 }
6381 
6382 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
6383     const ConstrainedFPIntrinsic &FPI) {
6384   SDLoc sdl = getCurSDLoc();
6385   unsigned Opcode;
6386   switch (FPI.getIntrinsicID()) {
6387   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6388   case Intrinsic::experimental_constrained_fadd:
6389     Opcode = ISD::STRICT_FADD;
6390     break;
6391   case Intrinsic::experimental_constrained_fsub:
6392     Opcode = ISD::STRICT_FSUB;
6393     break;
6394   case Intrinsic::experimental_constrained_fmul:
6395     Opcode = ISD::STRICT_FMUL;
6396     break;
6397   case Intrinsic::experimental_constrained_fdiv:
6398     Opcode = ISD::STRICT_FDIV;
6399     break;
6400   case Intrinsic::experimental_constrained_frem:
6401     Opcode = ISD::STRICT_FREM;
6402     break;
6403   case Intrinsic::experimental_constrained_fma:
6404     Opcode = ISD::STRICT_FMA;
6405     break;
6406   case Intrinsic::experimental_constrained_sqrt:
6407     Opcode = ISD::STRICT_FSQRT;
6408     break;
6409   case Intrinsic::experimental_constrained_pow:
6410     Opcode = ISD::STRICT_FPOW;
6411     break;
6412   case Intrinsic::experimental_constrained_powi:
6413     Opcode = ISD::STRICT_FPOWI;
6414     break;
6415   case Intrinsic::experimental_constrained_sin:
6416     Opcode = ISD::STRICT_FSIN;
6417     break;
6418   case Intrinsic::experimental_constrained_cos:
6419     Opcode = ISD::STRICT_FCOS;
6420     break;
6421   case Intrinsic::experimental_constrained_exp:
6422     Opcode = ISD::STRICT_FEXP;
6423     break;
6424   case Intrinsic::experimental_constrained_exp2:
6425     Opcode = ISD::STRICT_FEXP2;
6426     break;
6427   case Intrinsic::experimental_constrained_log:
6428     Opcode = ISD::STRICT_FLOG;
6429     break;
6430   case Intrinsic::experimental_constrained_log10:
6431     Opcode = ISD::STRICT_FLOG10;
6432     break;
6433   case Intrinsic::experimental_constrained_log2:
6434     Opcode = ISD::STRICT_FLOG2;
6435     break;
6436   case Intrinsic::experimental_constrained_rint:
6437     Opcode = ISD::STRICT_FRINT;
6438     break;
6439   case Intrinsic::experimental_constrained_nearbyint:
6440     Opcode = ISD::STRICT_FNEARBYINT;
6441     break;
6442   case Intrinsic::experimental_constrained_maxnum:
6443     Opcode = ISD::STRICT_FMAXNUM;
6444     break;
6445   case Intrinsic::experimental_constrained_minnum:
6446     Opcode = ISD::STRICT_FMINNUM;
6447     break;
6448   case Intrinsic::experimental_constrained_ceil:
6449     Opcode = ISD::STRICT_FCEIL;
6450     break;
6451   case Intrinsic::experimental_constrained_floor:
6452     Opcode = ISD::STRICT_FFLOOR;
6453     break;
6454   case Intrinsic::experimental_constrained_round:
6455     Opcode = ISD::STRICT_FROUND;
6456     break;
6457   case Intrinsic::experimental_constrained_trunc:
6458     Opcode = ISD::STRICT_FTRUNC;
6459     break;
6460   }
6461   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6462   SDValue Chain = getRoot();
6463   SmallVector<EVT, 4> ValueVTs;
6464   ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
6465   ValueVTs.push_back(MVT::Other); // Out chain
6466 
6467   SDVTList VTs = DAG.getVTList(ValueVTs);
6468   SDValue Result;
6469   if (FPI.isUnaryOp())
6470     Result = DAG.getNode(Opcode, sdl, VTs,
6471                          { Chain, getValue(FPI.getArgOperand(0)) });
6472   else if (FPI.isTernaryOp())
6473     Result = DAG.getNode(Opcode, sdl, VTs,
6474                          { Chain, getValue(FPI.getArgOperand(0)),
6475                                   getValue(FPI.getArgOperand(1)),
6476                                   getValue(FPI.getArgOperand(2)) });
6477   else
6478     Result = DAG.getNode(Opcode, sdl, VTs,
6479                          { Chain, getValue(FPI.getArgOperand(0)),
6480                            getValue(FPI.getArgOperand(1))  });
6481 
6482   assert(Result.getNode()->getNumValues() == 2);
6483   SDValue OutChain = Result.getValue(1);
6484   DAG.setRoot(OutChain);
6485   SDValue FPResult = Result.getValue(0);
6486   setValue(&FPI, FPResult);
6487 }
6488 
6489 std::pair<SDValue, SDValue>
6490 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
6491                                     const BasicBlock *EHPadBB) {
6492   MachineFunction &MF = DAG.getMachineFunction();
6493   MachineModuleInfo &MMI = MF.getMMI();
6494   MCSymbol *BeginLabel = nullptr;
6495 
6496   if (EHPadBB) {
6497     // Insert a label before the invoke call to mark the try range.  This can be
6498     // used to detect deletion of the invoke via the MachineModuleInfo.
6499     BeginLabel = MMI.getContext().createTempSymbol();
6500 
6501     // For SjLj, keep track of which landing pads go with which invokes
6502     // so as to maintain the ordering of pads in the LSDA.
6503     unsigned CallSiteIndex = MMI.getCurrentCallSite();
6504     if (CallSiteIndex) {
6505       MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
6506       LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
6507 
6508       // Now that the call site is handled, stop tracking it.
6509       MMI.setCurrentCallSite(0);
6510     }
6511 
6512     // Both PendingLoads and PendingExports must be flushed here;
6513     // this call might not return.
6514     (void)getRoot();
6515     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
6516 
6517     CLI.setChain(getRoot());
6518   }
6519   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6520   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6521 
6522   assert((CLI.IsTailCall || Result.second.getNode()) &&
6523          "Non-null chain expected with non-tail call!");
6524   assert((Result.second.getNode() || !Result.first.getNode()) &&
6525          "Null value expected with tail call!");
6526 
6527   if (!Result.second.getNode()) {
6528     // As a special case, a null chain means that a tail call has been emitted
6529     // and the DAG root is already updated.
6530     HasTailCall = true;
6531 
6532     // Since there's no actual continuation from this block, nothing can be
6533     // relying on us setting vregs for them.
6534     PendingExports.clear();
6535   } else {
6536     DAG.setRoot(Result.second);
6537   }
6538 
6539   if (EHPadBB) {
6540     // Insert a label at the end of the invoke call to mark the try range.  This
6541     // can be used to detect deletion of the invoke via the MachineModuleInfo.
6542     MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
6543     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
6544 
6545     // Inform MachineModuleInfo of range.
6546     auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
6547     // There is a platform (e.g. wasm) that uses funclet style IR but does not
6548     // actually use outlined funclets and their LSDA info style.
6549     if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
6550       assert(CLI.CS);
6551       WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
6552       EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS.getInstruction()),
6553                                 BeginLabel, EndLabel);
6554     } else if (!isScopedEHPersonality(Pers)) {
6555       MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
6556     }
6557   }
6558 
6559   return Result;
6560 }
6561 
6562 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
6563                                       bool isTailCall,
6564                                       const BasicBlock *EHPadBB) {
6565   auto &DL = DAG.getDataLayout();
6566   FunctionType *FTy = CS.getFunctionType();
6567   Type *RetTy = CS.getType();
6568 
6569   TargetLowering::ArgListTy Args;
6570   Args.reserve(CS.arg_size());
6571 
6572   const Value *SwiftErrorVal = nullptr;
6573   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6574 
6575   // We can't tail call inside a function with a swifterror argument. Lowering
6576   // does not support this yet. It would have to move into the swifterror
6577   // register before the call.
6578   auto *Caller = CS.getInstruction()->getParent()->getParent();
6579   if (TLI.supportSwiftError() &&
6580       Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
6581     isTailCall = false;
6582 
6583   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
6584        i != e; ++i) {
6585     TargetLowering::ArgListEntry Entry;
6586     const Value *V = *i;
6587 
6588     // Skip empty types
6589     if (V->getType()->isEmptyTy())
6590       continue;
6591 
6592     SDValue ArgNode = getValue(V);
6593     Entry.Node = ArgNode; Entry.Ty = V->getType();
6594 
6595     Entry.setAttributes(&CS, i - CS.arg_begin());
6596 
6597     // Use swifterror virtual register as input to the call.
6598     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
6599       SwiftErrorVal = V;
6600       // We find the virtual register for the actual swifterror argument.
6601       // Instead of using the Value, we use the virtual register instead.
6602       Entry.Node = DAG.getRegister(FuncInfo
6603                                        .getOrCreateSwiftErrorVRegUseAt(
6604                                            CS.getInstruction(), FuncInfo.MBB, V)
6605                                        .first,
6606                                    EVT(TLI.getPointerTy(DL)));
6607     }
6608 
6609     Args.push_back(Entry);
6610 
6611     // If we have an explicit sret argument that is an Instruction, (i.e., it
6612     // might point to function-local memory), we can't meaningfully tail-call.
6613     if (Entry.IsSRet && isa<Instruction>(V))
6614       isTailCall = false;
6615   }
6616 
6617   // Check if target-independent constraints permit a tail call here.
6618   // Target-dependent constraints are checked within TLI->LowerCallTo.
6619   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
6620     isTailCall = false;
6621 
6622   // Disable tail calls if there is an swifterror argument. Targets have not
6623   // been updated to support tail calls.
6624   if (TLI.supportSwiftError() && SwiftErrorVal)
6625     isTailCall = false;
6626 
6627   TargetLowering::CallLoweringInfo CLI(DAG);
6628   CLI.setDebugLoc(getCurSDLoc())
6629       .setChain(getRoot())
6630       .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
6631       .setTailCall(isTailCall)
6632       .setConvergent(CS.isConvergent());
6633   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
6634 
6635   if (Result.first.getNode()) {
6636     const Instruction *Inst = CS.getInstruction();
6637     Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
6638     setValue(Inst, Result.first);
6639   }
6640 
6641   // The last element of CLI.InVals has the SDValue for swifterror return.
6642   // Here we copy it to a virtual register and update SwiftErrorMap for
6643   // book-keeping.
6644   if (SwiftErrorVal && TLI.supportSwiftError()) {
6645     // Get the last element of InVals.
6646     SDValue Src = CLI.InVals.back();
6647     unsigned VReg; bool CreatedVReg;
6648     std::tie(VReg, CreatedVReg) =
6649         FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction());
6650     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
6651     // We update the virtual register for the actual swifterror argument.
6652     if (CreatedVReg)
6653       FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
6654     DAG.setRoot(CopyNode);
6655   }
6656 }
6657 
6658 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
6659                              SelectionDAGBuilder &Builder) {
6660   // Check to see if this load can be trivially constant folded, e.g. if the
6661   // input is from a string literal.
6662   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
6663     // Cast pointer to the type we really want to load.
6664     Type *LoadTy =
6665         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
6666     if (LoadVT.isVector())
6667       LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
6668 
6669     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
6670                                          PointerType::getUnqual(LoadTy));
6671 
6672     if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
6673             const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
6674       return Builder.getValue(LoadCst);
6675   }
6676 
6677   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
6678   // still constant memory, the input chain can be the entry node.
6679   SDValue Root;
6680   bool ConstantMemory = false;
6681 
6682   // Do not serialize (non-volatile) loads of constant memory with anything.
6683   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
6684     Root = Builder.DAG.getEntryNode();
6685     ConstantMemory = true;
6686   } else {
6687     // Do not serialize non-volatile loads against each other.
6688     Root = Builder.DAG.getRoot();
6689   }
6690 
6691   SDValue Ptr = Builder.getValue(PtrVal);
6692   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6693                                         Ptr, MachinePointerInfo(PtrVal),
6694                                         /* Alignment = */ 1);
6695 
6696   if (!ConstantMemory)
6697     Builder.PendingLoads.push_back(LoadVal.getValue(1));
6698   return LoadVal;
6699 }
6700 
6701 /// Record the value for an instruction that produces an integer result,
6702 /// converting the type where necessary.
6703 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6704                                                   SDValue Value,
6705                                                   bool IsSigned) {
6706   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6707                                                     I.getType(), true);
6708   if (IsSigned)
6709     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6710   else
6711     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6712   setValue(&I, Value);
6713 }
6714 
6715 /// See if we can lower a memcmp call into an optimized form. If so, return
6716 /// true and lower it. Otherwise return false, and it will be lowered like a
6717 /// normal call.
6718 /// The caller already checked that \p I calls the appropriate LibFunc with a
6719 /// correct prototype.
6720 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6721   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6722   const Value *Size = I.getArgOperand(2);
6723   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6724   if (CSize && CSize->getZExtValue() == 0) {
6725     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6726                                                           I.getType(), true);
6727     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6728     return true;
6729   }
6730 
6731   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6732   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6733       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6734       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6735   if (Res.first.getNode()) {
6736     processIntegerCallValue(I, Res.first, true);
6737     PendingLoads.push_back(Res.second);
6738     return true;
6739   }
6740 
6741   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
6742   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
6743   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
6744     return false;
6745 
6746   // If the target has a fast compare for the given size, it will return a
6747   // preferred load type for that size. Require that the load VT is legal and
6748   // that the target supports unaligned loads of that type. Otherwise, return
6749   // INVALID.
6750   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6751     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6752     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6753     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6754       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6755       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6756       // TODO: Check alignment of src and dest ptrs.
6757       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6758       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6759       if (!TLI.isTypeLegal(LVT) ||
6760           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6761           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6762         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6763     }
6764 
6765     return LVT;
6766   };
6767 
6768   // This turns into unaligned loads. We only do this if the target natively
6769   // supports the MVT we'll be loading or if it is small enough (<= 4) that
6770   // we'll only produce a small number of byte loads.
6771   MVT LoadVT;
6772   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6773   switch (NumBitsToCompare) {
6774   default:
6775     return false;
6776   case 16:
6777     LoadVT = MVT::i16;
6778     break;
6779   case 32:
6780     LoadVT = MVT::i32;
6781     break;
6782   case 64:
6783   case 128:
6784   case 256:
6785     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6786     break;
6787   }
6788 
6789   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6790     return false;
6791 
6792   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6793   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6794 
6795   // Bitcast to a wide integer type if the loads are vectors.
6796   if (LoadVT.isVector()) {
6797     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6798     LoadL = DAG.getBitcast(CmpVT, LoadL);
6799     LoadR = DAG.getBitcast(CmpVT, LoadR);
6800   }
6801 
6802   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6803   processIntegerCallValue(I, Cmp, false);
6804   return true;
6805 }
6806 
6807 /// See if we can lower a memchr call into an optimized form. If so, return
6808 /// true and lower it. Otherwise return false, and it will be lowered like a
6809 /// normal call.
6810 /// The caller already checked that \p I calls the appropriate LibFunc with a
6811 /// correct prototype.
6812 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6813   const Value *Src = I.getArgOperand(0);
6814   const Value *Char = I.getArgOperand(1);
6815   const Value *Length = I.getArgOperand(2);
6816 
6817   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6818   std::pair<SDValue, SDValue> Res =
6819     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6820                                 getValue(Src), getValue(Char), getValue(Length),
6821                                 MachinePointerInfo(Src));
6822   if (Res.first.getNode()) {
6823     setValue(&I, Res.first);
6824     PendingLoads.push_back(Res.second);
6825     return true;
6826   }
6827 
6828   return false;
6829 }
6830 
6831 /// See if we can lower a mempcpy call into an optimized form. If so, return
6832 /// true and lower it. Otherwise return false, and it will be lowered like a
6833 /// normal call.
6834 /// The caller already checked that \p I calls the appropriate LibFunc with a
6835 /// correct prototype.
6836 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6837   SDValue Dst = getValue(I.getArgOperand(0));
6838   SDValue Src = getValue(I.getArgOperand(1));
6839   SDValue Size = getValue(I.getArgOperand(2));
6840 
6841   unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6842   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6843   unsigned Align = std::min(DstAlign, SrcAlign);
6844   if (Align == 0) // Alignment of one or both could not be inferred.
6845     Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6846 
6847   bool isVol = false;
6848   SDLoc sdl = getCurSDLoc();
6849 
6850   // In the mempcpy context we need to pass in a false value for isTailCall
6851   // because the return pointer needs to be adjusted by the size of
6852   // the copied memory.
6853   SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6854                              false, /*isTailCall=*/false,
6855                              MachinePointerInfo(I.getArgOperand(0)),
6856                              MachinePointerInfo(I.getArgOperand(1)));
6857   assert(MC.getNode() != nullptr &&
6858          "** memcpy should not be lowered as TailCall in mempcpy context **");
6859   DAG.setRoot(MC);
6860 
6861   // Check if Size needs to be truncated or extended.
6862   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6863 
6864   // Adjust return pointer to point just past the last dst byte.
6865   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6866                                     Dst, Size);
6867   setValue(&I, DstPlusSize);
6868   return true;
6869 }
6870 
6871 /// See if we can lower a strcpy call into an optimized form.  If so, return
6872 /// true and lower it, otherwise return false and it will be lowered like a
6873 /// normal call.
6874 /// The caller already checked that \p I calls the appropriate LibFunc with a
6875 /// correct prototype.
6876 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6877   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6878 
6879   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6880   std::pair<SDValue, SDValue> Res =
6881     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6882                                 getValue(Arg0), getValue(Arg1),
6883                                 MachinePointerInfo(Arg0),
6884                                 MachinePointerInfo(Arg1), isStpcpy);
6885   if (Res.first.getNode()) {
6886     setValue(&I, Res.first);
6887     DAG.setRoot(Res.second);
6888     return true;
6889   }
6890 
6891   return false;
6892 }
6893 
6894 /// See if we can lower a strcmp call into an optimized form.  If so, return
6895 /// true and lower it, otherwise return false and it will be lowered like a
6896 /// normal call.
6897 /// The caller already checked that \p I calls the appropriate LibFunc with a
6898 /// correct prototype.
6899 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6900   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6901 
6902   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6903   std::pair<SDValue, SDValue> Res =
6904     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6905                                 getValue(Arg0), getValue(Arg1),
6906                                 MachinePointerInfo(Arg0),
6907                                 MachinePointerInfo(Arg1));
6908   if (Res.first.getNode()) {
6909     processIntegerCallValue(I, Res.first, true);
6910     PendingLoads.push_back(Res.second);
6911     return true;
6912   }
6913 
6914   return false;
6915 }
6916 
6917 /// See if we can lower a strlen call into an optimized form.  If so, return
6918 /// true and lower it, otherwise return false and it will be lowered like a
6919 /// normal call.
6920 /// The caller already checked that \p I calls the appropriate LibFunc with a
6921 /// correct prototype.
6922 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6923   const Value *Arg0 = I.getArgOperand(0);
6924 
6925   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6926   std::pair<SDValue, SDValue> Res =
6927     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6928                                 getValue(Arg0), MachinePointerInfo(Arg0));
6929   if (Res.first.getNode()) {
6930     processIntegerCallValue(I, Res.first, false);
6931     PendingLoads.push_back(Res.second);
6932     return true;
6933   }
6934 
6935   return false;
6936 }
6937 
6938 /// See if we can lower a strnlen call into an optimized form.  If so, return
6939 /// true and lower it, otherwise return false and it will be lowered like a
6940 /// normal call.
6941 /// The caller already checked that \p I calls the appropriate LibFunc with a
6942 /// correct prototype.
6943 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6944   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6945 
6946   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6947   std::pair<SDValue, SDValue> Res =
6948     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6949                                  getValue(Arg0), getValue(Arg1),
6950                                  MachinePointerInfo(Arg0));
6951   if (Res.first.getNode()) {
6952     processIntegerCallValue(I, Res.first, false);
6953     PendingLoads.push_back(Res.second);
6954     return true;
6955   }
6956 
6957   return false;
6958 }
6959 
6960 /// See if we can lower a unary floating-point operation into an SDNode with
6961 /// the specified Opcode.  If so, return true and lower it, otherwise return
6962 /// false and it will be lowered like a normal call.
6963 /// The caller already checked that \p I calls the appropriate LibFunc with a
6964 /// correct prototype.
6965 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6966                                               unsigned Opcode) {
6967   // We already checked this call's prototype; verify it doesn't modify errno.
6968   if (!I.onlyReadsMemory())
6969     return false;
6970 
6971   SDValue Tmp = getValue(I.getArgOperand(0));
6972   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6973   return true;
6974 }
6975 
6976 /// See if we can lower a binary floating-point operation into an SDNode with
6977 /// the specified Opcode. If so, return true and lower it. Otherwise return
6978 /// false, and it will be lowered like a normal call.
6979 /// The caller already checked that \p I calls the appropriate LibFunc with a
6980 /// correct prototype.
6981 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6982                                                unsigned Opcode) {
6983   // We already checked this call's prototype; verify it doesn't modify errno.
6984   if (!I.onlyReadsMemory())
6985     return false;
6986 
6987   SDValue Tmp0 = getValue(I.getArgOperand(0));
6988   SDValue Tmp1 = getValue(I.getArgOperand(1));
6989   EVT VT = Tmp0.getValueType();
6990   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6991   return true;
6992 }
6993 
6994 void SelectionDAGBuilder::visitCall(const CallInst &I) {
6995   // Handle inline assembly differently.
6996   if (isa<InlineAsm>(I.getCalledValue())) {
6997     visitInlineAsm(&I);
6998     return;
6999   }
7000 
7001   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
7002   computeUsesVAFloatArgument(I, MMI);
7003 
7004   const char *RenameFn = nullptr;
7005   if (Function *F = I.getCalledFunction()) {
7006     if (F->isDeclaration()) {
7007       // Is this an LLVM intrinsic or a target-specific intrinsic?
7008       unsigned IID = F->getIntrinsicID();
7009       if (!IID)
7010         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
7011           IID = II->getIntrinsicID(F);
7012 
7013       if (IID) {
7014         RenameFn = visitIntrinsicCall(I, IID);
7015         if (!RenameFn)
7016           return;
7017       }
7018     }
7019 
7020     // Check for well-known libc/libm calls.  If the function is internal, it
7021     // can't be a library call.  Don't do the check if marked as nobuiltin for
7022     // some reason or the call site requires strict floating point semantics.
7023     LibFunc Func;
7024     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
7025         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
7026         LibInfo->hasOptimizedCodeGen(Func)) {
7027       switch (Func) {
7028       default: break;
7029       case LibFunc_copysign:
7030       case LibFunc_copysignf:
7031       case LibFunc_copysignl:
7032         // We already checked this call's prototype; verify it doesn't modify
7033         // errno.
7034         if (I.onlyReadsMemory()) {
7035           SDValue LHS = getValue(I.getArgOperand(0));
7036           SDValue RHS = getValue(I.getArgOperand(1));
7037           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
7038                                    LHS.getValueType(), LHS, RHS));
7039           return;
7040         }
7041         break;
7042       case LibFunc_fabs:
7043       case LibFunc_fabsf:
7044       case LibFunc_fabsl:
7045         if (visitUnaryFloatCall(I, ISD::FABS))
7046           return;
7047         break;
7048       case LibFunc_fmin:
7049       case LibFunc_fminf:
7050       case LibFunc_fminl:
7051         if (visitBinaryFloatCall(I, ISD::FMINNUM))
7052           return;
7053         break;
7054       case LibFunc_fmax:
7055       case LibFunc_fmaxf:
7056       case LibFunc_fmaxl:
7057         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
7058           return;
7059         break;
7060       case LibFunc_sin:
7061       case LibFunc_sinf:
7062       case LibFunc_sinl:
7063         if (visitUnaryFloatCall(I, ISD::FSIN))
7064           return;
7065         break;
7066       case LibFunc_cos:
7067       case LibFunc_cosf:
7068       case LibFunc_cosl:
7069         if (visitUnaryFloatCall(I, ISD::FCOS))
7070           return;
7071         break;
7072       case LibFunc_sqrt:
7073       case LibFunc_sqrtf:
7074       case LibFunc_sqrtl:
7075       case LibFunc_sqrt_finite:
7076       case LibFunc_sqrtf_finite:
7077       case LibFunc_sqrtl_finite:
7078         if (visitUnaryFloatCall(I, ISD::FSQRT))
7079           return;
7080         break;
7081       case LibFunc_floor:
7082       case LibFunc_floorf:
7083       case LibFunc_floorl:
7084         if (visitUnaryFloatCall(I, ISD::FFLOOR))
7085           return;
7086         break;
7087       case LibFunc_nearbyint:
7088       case LibFunc_nearbyintf:
7089       case LibFunc_nearbyintl:
7090         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
7091           return;
7092         break;
7093       case LibFunc_ceil:
7094       case LibFunc_ceilf:
7095       case LibFunc_ceill:
7096         if (visitUnaryFloatCall(I, ISD::FCEIL))
7097           return;
7098         break;
7099       case LibFunc_rint:
7100       case LibFunc_rintf:
7101       case LibFunc_rintl:
7102         if (visitUnaryFloatCall(I, ISD::FRINT))
7103           return;
7104         break;
7105       case LibFunc_round:
7106       case LibFunc_roundf:
7107       case LibFunc_roundl:
7108         if (visitUnaryFloatCall(I, ISD::FROUND))
7109           return;
7110         break;
7111       case LibFunc_trunc:
7112       case LibFunc_truncf:
7113       case LibFunc_truncl:
7114         if (visitUnaryFloatCall(I, ISD::FTRUNC))
7115           return;
7116         break;
7117       case LibFunc_log2:
7118       case LibFunc_log2f:
7119       case LibFunc_log2l:
7120         if (visitUnaryFloatCall(I, ISD::FLOG2))
7121           return;
7122         break;
7123       case LibFunc_exp2:
7124       case LibFunc_exp2f:
7125       case LibFunc_exp2l:
7126         if (visitUnaryFloatCall(I, ISD::FEXP2))
7127           return;
7128         break;
7129       case LibFunc_memcmp:
7130         if (visitMemCmpCall(I))
7131           return;
7132         break;
7133       case LibFunc_mempcpy:
7134         if (visitMemPCpyCall(I))
7135           return;
7136         break;
7137       case LibFunc_memchr:
7138         if (visitMemChrCall(I))
7139           return;
7140         break;
7141       case LibFunc_strcpy:
7142         if (visitStrCpyCall(I, false))
7143           return;
7144         break;
7145       case LibFunc_stpcpy:
7146         if (visitStrCpyCall(I, true))
7147           return;
7148         break;
7149       case LibFunc_strcmp:
7150         if (visitStrCmpCall(I))
7151           return;
7152         break;
7153       case LibFunc_strlen:
7154         if (visitStrLenCall(I))
7155           return;
7156         break;
7157       case LibFunc_strnlen:
7158         if (visitStrNLenCall(I))
7159           return;
7160         break;
7161       }
7162     }
7163   }
7164 
7165   SDValue Callee;
7166   if (!RenameFn)
7167     Callee = getValue(I.getCalledValue());
7168   else
7169     Callee = DAG.getExternalSymbol(
7170         RenameFn,
7171         DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
7172 
7173   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
7174   // have to do anything here to lower funclet bundles.
7175   assert(!I.hasOperandBundlesOtherThan(
7176              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
7177          "Cannot lower calls with arbitrary operand bundles!");
7178 
7179   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
7180     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
7181   else
7182     // Check if we can potentially perform a tail call. More detailed checking
7183     // is be done within LowerCallTo, after more information about the call is
7184     // known.
7185     LowerCallTo(&I, Callee, I.isTailCall());
7186 }
7187 
7188 namespace {
7189 
7190 /// AsmOperandInfo - This contains information for each constraint that we are
7191 /// lowering.
7192 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
7193 public:
7194   /// CallOperand - If this is the result output operand or a clobber
7195   /// this is null, otherwise it is the incoming operand to the CallInst.
7196   /// This gets modified as the asm is processed.
7197   SDValue CallOperand;
7198 
7199   /// AssignedRegs - If this is a register or register class operand, this
7200   /// contains the set of register corresponding to the operand.
7201   RegsForValue AssignedRegs;
7202 
7203   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
7204     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
7205   }
7206 
7207   /// Whether or not this operand accesses memory
7208   bool hasMemory(const TargetLowering &TLI) const {
7209     // Indirect operand accesses access memory.
7210     if (isIndirect)
7211       return true;
7212 
7213     for (const auto &Code : Codes)
7214       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
7215         return true;
7216 
7217     return false;
7218   }
7219 
7220   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
7221   /// corresponds to.  If there is no Value* for this operand, it returns
7222   /// MVT::Other.
7223   EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
7224                            const DataLayout &DL) const {
7225     if (!CallOperandVal) return MVT::Other;
7226 
7227     if (isa<BasicBlock>(CallOperandVal))
7228       return TLI.getPointerTy(DL);
7229 
7230     llvm::Type *OpTy = CallOperandVal->getType();
7231 
7232     // FIXME: code duplicated from TargetLowering::ParseConstraints().
7233     // If this is an indirect operand, the operand is a pointer to the
7234     // accessed type.
7235     if (isIndirect) {
7236       PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
7237       if (!PtrTy)
7238         report_fatal_error("Indirect operand for inline asm not a pointer!");
7239       OpTy = PtrTy->getElementType();
7240     }
7241 
7242     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
7243     if (StructType *STy = dyn_cast<StructType>(OpTy))
7244       if (STy->getNumElements() == 1)
7245         OpTy = STy->getElementType(0);
7246 
7247     // If OpTy is not a single value, it may be a struct/union that we
7248     // can tile with integers.
7249     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
7250       unsigned BitSize = DL.getTypeSizeInBits(OpTy);
7251       switch (BitSize) {
7252       default: break;
7253       case 1:
7254       case 8:
7255       case 16:
7256       case 32:
7257       case 64:
7258       case 128:
7259         OpTy = IntegerType::get(Context, BitSize);
7260         break;
7261       }
7262     }
7263 
7264     return TLI.getValueType(DL, OpTy, true);
7265   }
7266 };
7267 
7268 using SDISelAsmOperandInfoVector = SmallVector<SDISelAsmOperandInfo, 16>;
7269 
7270 } // end anonymous namespace
7271 
7272 /// Make sure that the output operand \p OpInfo and its corresponding input
7273 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
7274 /// out).
7275 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
7276                                SDISelAsmOperandInfo &MatchingOpInfo,
7277                                SelectionDAG &DAG) {
7278   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
7279     return;
7280 
7281   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
7282   const auto &TLI = DAG.getTargetLoweringInfo();
7283 
7284   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
7285       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
7286                                        OpInfo.ConstraintVT);
7287   std::pair<unsigned, const TargetRegisterClass *> InputRC =
7288       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
7289                                        MatchingOpInfo.ConstraintVT);
7290   if ((OpInfo.ConstraintVT.isInteger() !=
7291        MatchingOpInfo.ConstraintVT.isInteger()) ||
7292       (MatchRC.second != InputRC.second)) {
7293     // FIXME: error out in a more elegant fashion
7294     report_fatal_error("Unsupported asm: input constraint"
7295                        " with a matching output constraint of"
7296                        " incompatible type!");
7297   }
7298   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
7299 }
7300 
7301 /// Get a direct memory input to behave well as an indirect operand.
7302 /// This may introduce stores, hence the need for a \p Chain.
7303 /// \return The (possibly updated) chain.
7304 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
7305                                         SDISelAsmOperandInfo &OpInfo,
7306                                         SelectionDAG &DAG) {
7307   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7308 
7309   // If we don't have an indirect input, put it in the constpool if we can,
7310   // otherwise spill it to a stack slot.
7311   // TODO: This isn't quite right. We need to handle these according to
7312   // the addressing mode that the constraint wants. Also, this may take
7313   // an additional register for the computation and we don't want that
7314   // either.
7315 
7316   // If the operand is a float, integer, or vector constant, spill to a
7317   // constant pool entry to get its address.
7318   const Value *OpVal = OpInfo.CallOperandVal;
7319   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
7320       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
7321     OpInfo.CallOperand = DAG.getConstantPool(
7322         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
7323     return Chain;
7324   }
7325 
7326   // Otherwise, create a stack slot and emit a store to it before the asm.
7327   Type *Ty = OpVal->getType();
7328   auto &DL = DAG.getDataLayout();
7329   uint64_t TySize = DL.getTypeAllocSize(Ty);
7330   unsigned Align = DL.getPrefTypeAlignment(Ty);
7331   MachineFunction &MF = DAG.getMachineFunction();
7332   int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7333   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
7334   Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
7335                        MachinePointerInfo::getFixedStack(MF, SSFI));
7336   OpInfo.CallOperand = StackSlot;
7337 
7338   return Chain;
7339 }
7340 
7341 /// GetRegistersForValue - Assign registers (virtual or physical) for the
7342 /// specified operand.  We prefer to assign virtual registers, to allow the
7343 /// register allocator to handle the assignment process.  However, if the asm
7344 /// uses features that we can't model on machineinstrs, we have SDISel do the
7345 /// allocation.  This produces generally horrible, but correct, code.
7346 ///
7347 ///   OpInfo describes the operand
7348 ///   RefOpInfo describes the matching operand if any, the operand otherwise
7349 static void GetRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
7350                                  SDISelAsmOperandInfo &OpInfo,
7351                                  SDISelAsmOperandInfo &RefOpInfo) {
7352   LLVMContext &Context = *DAG.getContext();
7353   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7354 
7355   MachineFunction &MF = DAG.getMachineFunction();
7356   SmallVector<unsigned, 4> Regs;
7357   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7358 
7359   // If this is a constraint for a single physreg, or a constraint for a
7360   // register class, find it.
7361   unsigned AssignedReg;
7362   const TargetRegisterClass *RC;
7363   std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
7364       &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
7365   // RC is unset only on failure. Return immediately.
7366   if (!RC)
7367     return;
7368 
7369   // Get the actual register value type.  This is important, because the user
7370   // may have asked for (e.g.) the AX register in i32 type.  We need to
7371   // remember that AX is actually i16 to get the right extension.
7372   const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
7373 
7374   if (OpInfo.ConstraintVT != MVT::Other) {
7375     // If this is an FP operand in an integer register (or visa versa), or more
7376     // generally if the operand value disagrees with the register class we plan
7377     // to stick it in, fix the operand type.
7378     //
7379     // If this is an input value, the bitcast to the new type is done now.
7380     // Bitcast for output value is done at the end of visitInlineAsm().
7381     if ((OpInfo.Type == InlineAsm::isOutput ||
7382          OpInfo.Type == InlineAsm::isInput) &&
7383         !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
7384       // Try to convert to the first EVT that the reg class contains.  If the
7385       // types are identical size, use a bitcast to convert (e.g. two differing
7386       // vector types).  Note: output bitcast is done at the end of
7387       // visitInlineAsm().
7388       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
7389         // Exclude indirect inputs while they are unsupported because the code
7390         // to perform the load is missing and thus OpInfo.CallOperand still
7391         // refers to the input address rather than the pointed-to value.
7392         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
7393           OpInfo.CallOperand =
7394               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
7395         OpInfo.ConstraintVT = RegVT;
7396         // If the operand is an FP value and we want it in integer registers,
7397         // use the corresponding integer type. This turns an f64 value into
7398         // i64, which can be passed with two i32 values on a 32-bit machine.
7399       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
7400         MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
7401         if (OpInfo.Type == InlineAsm::isInput)
7402           OpInfo.CallOperand =
7403               DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
7404         OpInfo.ConstraintVT = VT;
7405       }
7406     }
7407   }
7408 
7409   // No need to allocate a matching input constraint since the constraint it's
7410   // matching to has already been allocated.
7411   if (OpInfo.isMatchingInputConstraint())
7412     return;
7413 
7414   EVT ValueVT = OpInfo.ConstraintVT;
7415   if (OpInfo.ConstraintVT == MVT::Other)
7416     ValueVT = RegVT;
7417 
7418   // Initialize NumRegs.
7419   unsigned NumRegs = 1;
7420   if (OpInfo.ConstraintVT != MVT::Other)
7421     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
7422 
7423   // If this is a constraint for a specific physical register, like {r17},
7424   // assign it now.
7425 
7426   // If this associated to a specific register, initialize iterator to correct
7427   // place. If virtual, make sure we have enough registers
7428 
7429   // Initialize iterator if necessary
7430   TargetRegisterClass::iterator I = RC->begin();
7431   MachineRegisterInfo &RegInfo = MF.getRegInfo();
7432 
7433   // Do not check for single registers.
7434   if (AssignedReg) {
7435       for (; *I != AssignedReg; ++I)
7436         assert(I != RC->end() && "AssignedReg should be member of RC");
7437   }
7438 
7439   for (; NumRegs; --NumRegs, ++I) {
7440     assert(I != RC->end() && "Ran out of registers to allocate!");
7441     auto R = (AssignedReg) ? *I : RegInfo.createVirtualRegister(RC);
7442     Regs.push_back(R);
7443   }
7444 
7445   OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
7446 }
7447 
7448 static unsigned
7449 findMatchingInlineAsmOperand(unsigned OperandNo,
7450                              const std::vector<SDValue> &AsmNodeOperands) {
7451   // Scan until we find the definition we already emitted of this operand.
7452   unsigned CurOp = InlineAsm::Op_FirstOperand;
7453   for (; OperandNo; --OperandNo) {
7454     // Advance to the next operand.
7455     unsigned OpFlag =
7456         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7457     assert((InlineAsm::isRegDefKind(OpFlag) ||
7458             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
7459             InlineAsm::isMemKind(OpFlag)) &&
7460            "Skipped past definitions?");
7461     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
7462   }
7463   return CurOp;
7464 }
7465 
7466 namespace {
7467 
7468 class ExtraFlags {
7469   unsigned Flags = 0;
7470 
7471 public:
7472   explicit ExtraFlags(ImmutableCallSite CS) {
7473     const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7474     if (IA->hasSideEffects())
7475       Flags |= InlineAsm::Extra_HasSideEffects;
7476     if (IA->isAlignStack())
7477       Flags |= InlineAsm::Extra_IsAlignStack;
7478     if (CS.isConvergent())
7479       Flags |= InlineAsm::Extra_IsConvergent;
7480     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
7481   }
7482 
7483   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
7484     // Ideally, we would only check against memory constraints.  However, the
7485     // meaning of an Other constraint can be target-specific and we can't easily
7486     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
7487     // for Other constraints as well.
7488     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
7489         OpInfo.ConstraintType == TargetLowering::C_Other) {
7490       if (OpInfo.Type == InlineAsm::isInput)
7491         Flags |= InlineAsm::Extra_MayLoad;
7492       else if (OpInfo.Type == InlineAsm::isOutput)
7493         Flags |= InlineAsm::Extra_MayStore;
7494       else if (OpInfo.Type == InlineAsm::isClobber)
7495         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
7496     }
7497   }
7498 
7499   unsigned get() const { return Flags; }
7500 };
7501 
7502 } // end anonymous namespace
7503 
7504 /// visitInlineAsm - Handle a call to an InlineAsm object.
7505 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
7506   const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7507 
7508   /// ConstraintOperands - Information about all of the constraints.
7509   SDISelAsmOperandInfoVector ConstraintOperands;
7510 
7511   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7512   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
7513       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
7514 
7515   bool hasMemory = false;
7516 
7517   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7518   ExtraFlags ExtraInfo(CS);
7519 
7520   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
7521   unsigned ResNo = 0;   // ResNo - The result number of the next output.
7522   for (auto &T : TargetConstraints) {
7523     ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
7524     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
7525 
7526     // Compute the value type for each operand.
7527     if (OpInfo.Type == InlineAsm::isInput ||
7528         (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
7529       OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
7530 
7531       // Process the call argument. BasicBlocks are labels, currently appearing
7532       // only in asm's.
7533       if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
7534         OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
7535       } else {
7536         OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
7537       }
7538 
7539       OpInfo.ConstraintVT =
7540           OpInfo
7541               .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
7542               .getSimpleVT();
7543     } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
7544       // The return value of the call is this value.  As such, there is no
7545       // corresponding argument.
7546       assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7547       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
7548         OpInfo.ConstraintVT = TLI.getSimpleValueType(
7549             DAG.getDataLayout(), STy->getElementType(ResNo));
7550       } else {
7551         assert(ResNo == 0 && "Asm only has one result!");
7552         OpInfo.ConstraintVT =
7553             TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
7554       }
7555       ++ResNo;
7556     } else {
7557       OpInfo.ConstraintVT = MVT::Other;
7558     }
7559 
7560     if (!hasMemory)
7561       hasMemory = OpInfo.hasMemory(TLI);
7562 
7563     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
7564     // FIXME: Could we compute this on OpInfo rather than T?
7565 
7566     // Compute the constraint code and ConstraintType to use.
7567     TLI.ComputeConstraintToUse(T, SDValue());
7568 
7569     ExtraInfo.update(T);
7570   }
7571 
7572   SDValue Chain, Flag;
7573 
7574   // We won't need to flush pending loads if this asm doesn't touch
7575   // memory and is nonvolatile.
7576   if (hasMemory || IA->hasSideEffects())
7577     Chain = getRoot();
7578   else
7579     Chain = DAG.getRoot();
7580 
7581   // Second pass over the constraints: compute which constraint option to use
7582   // and assign registers to constraints that want a specific physreg.
7583   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
7584     // If this is an output operand with a matching input operand, look up the
7585     // matching input. If their types mismatch, e.g. one is an integer, the
7586     // other is floating point, or their sizes are different, flag it as an
7587     // error.
7588     if (OpInfo.hasMatchingInput()) {
7589       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
7590       patchMatchingInput(OpInfo, Input, DAG);
7591     }
7592 
7593     // Compute the constraint code and ConstraintType to use.
7594     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
7595 
7596     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7597         OpInfo.Type == InlineAsm::isClobber)
7598       continue;
7599 
7600     // If this is a memory input, and if the operand is not indirect, do what we
7601     // need to provide an address for the memory input.
7602     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7603         !OpInfo.isIndirect) {
7604       assert((OpInfo.isMultipleAlternative ||
7605               (OpInfo.Type == InlineAsm::isInput)) &&
7606              "Can only indirectify direct input operands!");
7607 
7608       // Memory operands really want the address of the value.
7609       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
7610 
7611       // There is no longer a Value* corresponding to this operand.
7612       OpInfo.CallOperandVal = nullptr;
7613 
7614       // It is now an indirect operand.
7615       OpInfo.isIndirect = true;
7616     }
7617 
7618     // If this constraint is for a specific register, allocate it before
7619     // anything else.
7620     SDISelAsmOperandInfo &RefOpInfo =
7621         OpInfo.isMatchingInputConstraint()
7622             ? ConstraintOperands[OpInfo.getMatchedOperand()]
7623             : OpInfo;
7624     if (RefOpInfo.ConstraintType == TargetLowering::C_Register)
7625       GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
7626   }
7627 
7628   // Third pass - Loop over all of the operands, assigning virtual or physregs
7629   // to register class operands.
7630   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
7631     SDISelAsmOperandInfo &RefOpInfo =
7632         OpInfo.isMatchingInputConstraint()
7633             ? ConstraintOperands[OpInfo.getMatchedOperand()]
7634             : OpInfo;
7635 
7636     // C_Register operands have already been allocated, Other/Memory don't need
7637     // to be.
7638     if (RefOpInfo.ConstraintType == TargetLowering::C_RegisterClass)
7639       GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
7640   }
7641 
7642   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
7643   std::vector<SDValue> AsmNodeOperands;
7644   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
7645   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
7646       IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
7647 
7648   // If we have a !srcloc metadata node associated with it, we want to attach
7649   // this to the ultimately generated inline asm machineinstr.  To do this, we
7650   // pass in the third operand as this (potentially null) inline asm MDNode.
7651   const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
7652   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
7653 
7654   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7655   // bits as operand 3.
7656   AsmNodeOperands.push_back(DAG.getTargetConstant(
7657       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7658 
7659   // Loop over all of the inputs, copying the operand values into the
7660   // appropriate registers and processing the output regs.
7661   RegsForValue RetValRegs;
7662 
7663   // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
7664   std::vector<std::pair<RegsForValue, Value *>> IndirectStoresToEmit;
7665 
7666   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
7667     switch (OpInfo.Type) {
7668     case InlineAsm::isOutput:
7669       if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7670           OpInfo.ConstraintType != TargetLowering::C_Register) {
7671         // Memory output, or 'other' output (e.g. 'X' constraint).
7672         assert(OpInfo.isIndirect && "Memory output must be indirect operand");
7673 
7674         unsigned ConstraintID =
7675             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7676         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7677                "Failed to convert memory constraint code to constraint id.");
7678 
7679         // Add information to the INLINEASM node to know about this output.
7680         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7681         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7682         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7683                                                         MVT::i32));
7684         AsmNodeOperands.push_back(OpInfo.CallOperand);
7685         break;
7686       }
7687 
7688       // Otherwise, this is a register or register class output.
7689 
7690       // Copy the output from the appropriate register.  Find a register that
7691       // we can use.
7692       if (OpInfo.AssignedRegs.Regs.empty()) {
7693         emitInlineAsmError(
7694             CS, "couldn't allocate output register for constraint '" +
7695                     Twine(OpInfo.ConstraintCode) + "'");
7696         return;
7697       }
7698 
7699       // If this is an indirect operand, store through the pointer after the
7700       // asm.
7701       if (OpInfo.isIndirect) {
7702         IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7703                                                       OpInfo.CallOperandVal));
7704       } else {
7705         // This is the result value of the call.
7706         assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7707         // Concatenate this output onto the outputs list.
7708         RetValRegs.append(OpInfo.AssignedRegs);
7709       }
7710 
7711       // Add information to the INLINEASM node to know that this register is
7712       // set.
7713       OpInfo.AssignedRegs
7714           .AddInlineAsmOperands(OpInfo.isEarlyClobber
7715                                     ? InlineAsm::Kind_RegDefEarlyClobber
7716                                     : InlineAsm::Kind_RegDef,
7717                                 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7718       break;
7719 
7720     case InlineAsm::isInput: {
7721       SDValue InOperandVal = OpInfo.CallOperand;
7722 
7723       if (OpInfo.isMatchingInputConstraint()) {
7724         // If this is required to match an output register we have already set,
7725         // just use its register.
7726         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7727                                                   AsmNodeOperands);
7728         unsigned OpFlag =
7729           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7730         if (InlineAsm::isRegDefKind(OpFlag) ||
7731             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7732           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7733           if (OpInfo.isIndirect) {
7734             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7735             emitInlineAsmError(CS, "inline asm not supported yet:"
7736                                    " don't know how to handle tied "
7737                                    "indirect register inputs");
7738             return;
7739           }
7740 
7741           MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7742           SmallVector<unsigned, 4> Regs;
7743 
7744           if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT)) {
7745             unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
7746             MachineRegisterInfo &RegInfo =
7747                 DAG.getMachineFunction().getRegInfo();
7748             for (unsigned i = 0; i != NumRegs; ++i)
7749               Regs.push_back(RegInfo.createVirtualRegister(RC));
7750           } else {
7751             emitInlineAsmError(CS, "inline asm error: This value type register "
7752                                    "class is not natively supported!");
7753             return;
7754           }
7755 
7756           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7757 
7758           SDLoc dl = getCurSDLoc();
7759           // Use the produced MatchedRegs object to
7760           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
7761                                     CS.getInstruction());
7762           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7763                                            true, OpInfo.getMatchedOperand(), dl,
7764                                            DAG, AsmNodeOperands);
7765           break;
7766         }
7767 
7768         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
7769         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
7770                "Unexpected number of operands");
7771         // Add information to the INLINEASM node to know about this input.
7772         // See InlineAsm.h isUseOperandTiedToDef.
7773         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7774         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7775                                                     OpInfo.getMatchedOperand());
7776         AsmNodeOperands.push_back(DAG.getTargetConstant(
7777             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7778         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7779         break;
7780       }
7781 
7782       // Treat indirect 'X' constraint as memory.
7783       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7784           OpInfo.isIndirect)
7785         OpInfo.ConstraintType = TargetLowering::C_Memory;
7786 
7787       if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7788         std::vector<SDValue> Ops;
7789         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7790                                           Ops, DAG);
7791         if (Ops.empty()) {
7792           emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7793                                      Twine(OpInfo.ConstraintCode) + "'");
7794           return;
7795         }
7796 
7797         // Add information to the INLINEASM node to know about this input.
7798         unsigned ResOpType =
7799           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7800         AsmNodeOperands.push_back(DAG.getTargetConstant(
7801             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7802         AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7803         break;
7804       }
7805 
7806       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7807         assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
7808         assert(InOperandVal.getValueType() ==
7809                    TLI.getPointerTy(DAG.getDataLayout()) &&
7810                "Memory operands expect pointer values");
7811 
7812         unsigned ConstraintID =
7813             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7814         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7815                "Failed to convert memory constraint code to constraint id.");
7816 
7817         // Add information to the INLINEASM node to know about this input.
7818         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7819         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7820         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7821                                                         getCurSDLoc(),
7822                                                         MVT::i32));
7823         AsmNodeOperands.push_back(InOperandVal);
7824         break;
7825       }
7826 
7827       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
7828               OpInfo.ConstraintType == TargetLowering::C_Register) &&
7829              "Unknown constraint type!");
7830 
7831       // TODO: Support this.
7832       if (OpInfo.isIndirect) {
7833         emitInlineAsmError(
7834             CS, "Don't know how to handle indirect register inputs yet "
7835                 "for constraint '" +
7836                     Twine(OpInfo.ConstraintCode) + "'");
7837         return;
7838       }
7839 
7840       // Copy the input into the appropriate registers.
7841       if (OpInfo.AssignedRegs.Regs.empty()) {
7842         emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7843                                    Twine(OpInfo.ConstraintCode) + "'");
7844         return;
7845       }
7846 
7847       SDLoc dl = getCurSDLoc();
7848 
7849       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7850                                         Chain, &Flag, CS.getInstruction());
7851 
7852       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7853                                                dl, DAG, AsmNodeOperands);
7854       break;
7855     }
7856     case InlineAsm::isClobber:
7857       // Add the clobbered value to the operand list, so that the register
7858       // allocator is aware that the physreg got clobbered.
7859       if (!OpInfo.AssignedRegs.Regs.empty())
7860         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7861                                                  false, 0, getCurSDLoc(), DAG,
7862                                                  AsmNodeOperands);
7863       break;
7864     }
7865   }
7866 
7867   // Finish up input operands.  Set the input chain and add the flag last.
7868   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7869   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7870 
7871   Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7872                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7873   Flag = Chain.getValue(1);
7874 
7875   // If this asm returns a register value, copy the result from that register
7876   // and set it as the value of the call.
7877   if (!RetValRegs.Regs.empty()) {
7878     SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7879                                              Chain, &Flag, CS.getInstruction());
7880 
7881     llvm::Type *CSResultType = CS.getType();
7882     unsigned numRet;
7883     ArrayRef<Type *> ResultTypes;
7884     SmallVector<SDValue, 1> ResultValues(1);
7885     if (StructType *StructResult = dyn_cast<StructType>(CSResultType)) {
7886       numRet = StructResult->getNumElements();
7887       assert(Val->getNumOperands() == numRet &&
7888              "Mismatch in number of output operands in asm result");
7889       ResultTypes = StructResult->elements();
7890       ArrayRef<SDUse> ValueUses = Val->ops();
7891       ResultValues.resize(numRet);
7892       std::transform(ValueUses.begin(), ValueUses.end(), ResultValues.begin(),
7893                      [](const SDUse &u) -> SDValue { return u.get(); });
7894     } else {
7895       numRet = 1;
7896       ResultValues[0] = Val;
7897       ResultTypes = makeArrayRef(CSResultType);
7898     }
7899     SmallVector<EVT, 1> ResultVTs(numRet);
7900     for (unsigned i = 0; i < numRet; i++) {
7901       EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), ResultTypes[i]);
7902       SDValue Val = ResultValues[i];
7903       assert(ResultTypes[i]->isSized() && "Unexpected unsized type");
7904       // If the type of the inline asm call site return value is different but
7905       // has same size as the type of the asm output bitcast it.  One example
7906       // of this is for vectors with different width / number of elements.
7907       // This can happen for register classes that can contain multiple
7908       // different value types.  The preg or vreg allocated may not have the
7909       // same VT as was expected.
7910       //
7911       // This can also happen for a return value that disagrees with the
7912       // register class it is put in, eg. a double in a general-purpose
7913       // register on a 32-bit machine.
7914       if (ResultVT != Val.getValueType() &&
7915           ResultVT.getSizeInBits() == Val.getValueSizeInBits())
7916         Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, Val);
7917       else if (ResultVT != Val.getValueType() && ResultVT.isInteger() &&
7918                Val.getValueType().isInteger()) {
7919         // If a result value was tied to an input value, the computed result
7920         // may have a wider width than the expected result.  Extract the
7921         // relevant portion.
7922         Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, Val);
7923       }
7924 
7925       assert(ResultVT == Val.getValueType() && "Asm result value mismatch!");
7926       ResultVTs[i] = ResultVT;
7927       ResultValues[i] = Val;
7928     }
7929 
7930     Val = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
7931                       DAG.getVTList(ResultVTs), ResultValues);
7932     setValue(CS.getInstruction(), Val);
7933     // Don't need to use this as a chain in this case.
7934     if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7935       return;
7936   }
7937 
7938   std::vector<std::pair<SDValue, const Value *>> StoresToEmit;
7939 
7940   // Process indirect outputs, first output all of the flagged copies out of
7941   // physregs.
7942   for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7943     RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7944     const Value *Ptr = IndirectStoresToEmit[i].second;
7945     SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7946                                              Chain, &Flag, IA);
7947     StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7948   }
7949 
7950   // Emit the non-flagged stores from the physregs.
7951   SmallVector<SDValue, 8> OutChains;
7952   for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7953     SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7954                                getValue(StoresToEmit[i].second),
7955                                MachinePointerInfo(StoresToEmit[i].second));
7956     OutChains.push_back(Val);
7957   }
7958 
7959   if (!OutChains.empty())
7960     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7961 
7962   DAG.setRoot(Chain);
7963 }
7964 
7965 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7966                                              const Twine &Message) {
7967   LLVMContext &Ctx = *DAG.getContext();
7968   Ctx.emitError(CS.getInstruction(), Message);
7969 
7970   // Make sure we leave the DAG in a valid state
7971   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7972   SmallVector<EVT, 1> ValueVTs;
7973   ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7974 
7975   if (ValueVTs.empty())
7976     return;
7977 
7978   SmallVector<SDValue, 1> Ops;
7979   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
7980     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
7981 
7982   setValue(CS.getInstruction(), DAG.getMergeValues(Ops, getCurSDLoc()));
7983 }
7984 
7985 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7986   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7987                           MVT::Other, getRoot(),
7988                           getValue(I.getArgOperand(0)),
7989                           DAG.getSrcValue(I.getArgOperand(0))));
7990 }
7991 
7992 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7993   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7994   const DataLayout &DL = DAG.getDataLayout();
7995   SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7996                            getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7997                            DAG.getSrcValue(I.getOperand(0)),
7998                            DL.getABITypeAlignment(I.getType()));
7999   setValue(&I, V);
8000   DAG.setRoot(V.getValue(1));
8001 }
8002 
8003 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
8004   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
8005                           MVT::Other, getRoot(),
8006                           getValue(I.getArgOperand(0)),
8007                           DAG.getSrcValue(I.getArgOperand(0))));
8008 }
8009 
8010 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
8011   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
8012                           MVT::Other, getRoot(),
8013                           getValue(I.getArgOperand(0)),
8014                           getValue(I.getArgOperand(1)),
8015                           DAG.getSrcValue(I.getArgOperand(0)),
8016                           DAG.getSrcValue(I.getArgOperand(1))));
8017 }
8018 
8019 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
8020                                                     const Instruction &I,
8021                                                     SDValue Op) {
8022   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
8023   if (!Range)
8024     return Op;
8025 
8026   ConstantRange CR = getConstantRangeFromMetadata(*Range);
8027   if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
8028     return Op;
8029 
8030   APInt Lo = CR.getUnsignedMin();
8031   if (!Lo.isMinValue())
8032     return Op;
8033 
8034   APInt Hi = CR.getUnsignedMax();
8035   unsigned Bits = std::max(Hi.getActiveBits(),
8036                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
8037 
8038   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
8039 
8040   SDLoc SL = getCurSDLoc();
8041 
8042   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
8043                              DAG.getValueType(SmallVT));
8044   unsigned NumVals = Op.getNode()->getNumValues();
8045   if (NumVals == 1)
8046     return ZExt;
8047 
8048   SmallVector<SDValue, 4> Ops;
8049 
8050   Ops.push_back(ZExt);
8051   for (unsigned I = 1; I != NumVals; ++I)
8052     Ops.push_back(Op.getValue(I));
8053 
8054   return DAG.getMergeValues(Ops, SL);
8055 }
8056 
8057 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
8058 /// the call being lowered.
8059 ///
8060 /// This is a helper for lowering intrinsics that follow a target calling
8061 /// convention or require stack pointer adjustment. Only a subset of the
8062 /// intrinsic's operands need to participate in the calling convention.
8063 void SelectionDAGBuilder::populateCallLoweringInfo(
8064     TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
8065     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
8066     bool IsPatchPoint) {
8067   TargetLowering::ArgListTy Args;
8068   Args.reserve(NumArgs);
8069 
8070   // Populate the argument list.
8071   // Attributes for args start at offset 1, after the return attribute.
8072   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
8073        ArgI != ArgE; ++ArgI) {
8074     const Value *V = CS->getOperand(ArgI);
8075 
8076     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
8077 
8078     TargetLowering::ArgListEntry Entry;
8079     Entry.Node = getValue(V);
8080     Entry.Ty = V->getType();
8081     Entry.setAttributes(&CS, ArgI);
8082     Args.push_back(Entry);
8083   }
8084 
8085   CLI.setDebugLoc(getCurSDLoc())
8086       .setChain(getRoot())
8087       .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
8088       .setDiscardResult(CS->use_empty())
8089       .setIsPatchPoint(IsPatchPoint);
8090 }
8091 
8092 /// Add a stack map intrinsic call's live variable operands to a stackmap
8093 /// or patchpoint target node's operand list.
8094 ///
8095 /// Constants are converted to TargetConstants purely as an optimization to
8096 /// avoid constant materialization and register allocation.
8097 ///
8098 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
8099 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
8100 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
8101 /// address materialization and register allocation, but may also be required
8102 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
8103 /// alloca in the entry block, then the runtime may assume that the alloca's
8104 /// StackMap location can be read immediately after compilation and that the
8105 /// location is valid at any point during execution (this is similar to the
8106 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
8107 /// only available in a register, then the runtime would need to trap when
8108 /// execution reaches the StackMap in order to read the alloca's location.
8109 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
8110                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
8111                                 SelectionDAGBuilder &Builder) {
8112   for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
8113     SDValue OpVal = Builder.getValue(CS.getArgument(i));
8114     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
8115       Ops.push_back(
8116         Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
8117       Ops.push_back(
8118         Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
8119     } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
8120       const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
8121       Ops.push_back(Builder.DAG.getTargetFrameIndex(
8122           FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
8123     } else
8124       Ops.push_back(OpVal);
8125   }
8126 }
8127 
8128 /// Lower llvm.experimental.stackmap directly to its target opcode.
8129 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
8130   // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
8131   //                                  [live variables...])
8132 
8133   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
8134 
8135   SDValue Chain, InFlag, Callee, NullPtr;
8136   SmallVector<SDValue, 32> Ops;
8137 
8138   SDLoc DL = getCurSDLoc();
8139   Callee = getValue(CI.getCalledValue());
8140   NullPtr = DAG.getIntPtrConstant(0, DL, true);
8141 
8142   // The stackmap intrinsic only records the live variables (the arguemnts
8143   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
8144   // intrinsic, this won't be lowered to a function call. This means we don't
8145   // have to worry about calling conventions and target specific lowering code.
8146   // Instead we perform the call lowering right here.
8147   //
8148   // chain, flag = CALLSEQ_START(chain, 0, 0)
8149   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
8150   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
8151   //
8152   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
8153   InFlag = Chain.getValue(1);
8154 
8155   // Add the <id> and <numBytes> constants.
8156   SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
8157   Ops.push_back(DAG.getTargetConstant(
8158                   cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
8159   SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
8160   Ops.push_back(DAG.getTargetConstant(
8161                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
8162                   MVT::i32));
8163 
8164   // Push live variables for the stack map.
8165   addStackMapLiveVars(&CI, 2, DL, Ops, *this);
8166 
8167   // We are not pushing any register mask info here on the operands list,
8168   // because the stackmap doesn't clobber anything.
8169 
8170   // Push the chain and the glue flag.
8171   Ops.push_back(Chain);
8172   Ops.push_back(InFlag);
8173 
8174   // Create the STACKMAP node.
8175   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8176   SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
8177   Chain = SDValue(SM, 0);
8178   InFlag = Chain.getValue(1);
8179 
8180   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
8181 
8182   // Stackmaps don't generate values, so nothing goes into the NodeMap.
8183 
8184   // Set the root to the target-lowered call chain.
8185   DAG.setRoot(Chain);
8186 
8187   // Inform the Frame Information that we have a stackmap in this function.
8188   FuncInfo.MF->getFrameInfo().setHasStackMap();
8189 }
8190 
8191 /// Lower llvm.experimental.patchpoint directly to its target opcode.
8192 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
8193                                           const BasicBlock *EHPadBB) {
8194   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
8195   //                                                 i32 <numBytes>,
8196   //                                                 i8* <target>,
8197   //                                                 i32 <numArgs>,
8198   //                                                 [Args...],
8199   //                                                 [live variables...])
8200 
8201   CallingConv::ID CC = CS.getCallingConv();
8202   bool IsAnyRegCC = CC == CallingConv::AnyReg;
8203   bool HasDef = !CS->getType()->isVoidTy();
8204   SDLoc dl = getCurSDLoc();
8205   SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
8206 
8207   // Handle immediate and symbolic callees.
8208   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
8209     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
8210                                    /*isTarget=*/true);
8211   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
8212     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
8213                                          SDLoc(SymbolicCallee),
8214                                          SymbolicCallee->getValueType(0));
8215 
8216   // Get the real number of arguments participating in the call <numArgs>
8217   SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
8218   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
8219 
8220   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
8221   // Intrinsics include all meta-operands up to but not including CC.
8222   unsigned NumMetaOpers = PatchPointOpers::CCPos;
8223   assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
8224          "Not enough arguments provided to the patchpoint intrinsic");
8225 
8226   // For AnyRegCC the arguments are lowered later on manually.
8227   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
8228   Type *ReturnTy =
8229     IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
8230 
8231   TargetLowering::CallLoweringInfo CLI(DAG);
8232   populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
8233                            true);
8234   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8235 
8236   SDNode *CallEnd = Result.second.getNode();
8237   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
8238     CallEnd = CallEnd->getOperand(0).getNode();
8239 
8240   /// Get a call instruction from the call sequence chain.
8241   /// Tail calls are not allowed.
8242   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
8243          "Expected a callseq node.");
8244   SDNode *Call = CallEnd->getOperand(0).getNode();
8245   bool HasGlue = Call->getGluedNode();
8246 
8247   // Replace the target specific call node with the patchable intrinsic.
8248   SmallVector<SDValue, 8> Ops;
8249 
8250   // Add the <id> and <numBytes> constants.
8251   SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
8252   Ops.push_back(DAG.getTargetConstant(
8253                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
8254   SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
8255   Ops.push_back(DAG.getTargetConstant(
8256                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
8257                   MVT::i32));
8258 
8259   // Add the callee.
8260   Ops.push_back(Callee);
8261 
8262   // Adjust <numArgs> to account for any arguments that have been passed on the
8263   // stack instead.
8264   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
8265   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
8266   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
8267   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
8268 
8269   // Add the calling convention
8270   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
8271 
8272   // Add the arguments we omitted previously. The register allocator should
8273   // place these in any free register.
8274   if (IsAnyRegCC)
8275     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
8276       Ops.push_back(getValue(CS.getArgument(i)));
8277 
8278   // Push the arguments from the call instruction up to the register mask.
8279   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
8280   Ops.append(Call->op_begin() + 2, e);
8281 
8282   // Push live variables for the stack map.
8283   addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
8284 
8285   // Push the register mask info.
8286   if (HasGlue)
8287     Ops.push_back(*(Call->op_end()-2));
8288   else
8289     Ops.push_back(*(Call->op_end()-1));
8290 
8291   // Push the chain (this is originally the first operand of the call, but
8292   // becomes now the last or second to last operand).
8293   Ops.push_back(*(Call->op_begin()));
8294 
8295   // Push the glue flag (last operand).
8296   if (HasGlue)
8297     Ops.push_back(*(Call->op_end()-1));
8298 
8299   SDVTList NodeTys;
8300   if (IsAnyRegCC && HasDef) {
8301     // Create the return types based on the intrinsic definition
8302     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8303     SmallVector<EVT, 3> ValueVTs;
8304     ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
8305     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
8306 
8307     // There is always a chain and a glue type at the end
8308     ValueVTs.push_back(MVT::Other);
8309     ValueVTs.push_back(MVT::Glue);
8310     NodeTys = DAG.getVTList(ValueVTs);
8311   } else
8312     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8313 
8314   // Replace the target specific call node with a PATCHPOINT node.
8315   MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
8316                                          dl, NodeTys, Ops);
8317 
8318   // Update the NodeMap.
8319   if (HasDef) {
8320     if (IsAnyRegCC)
8321       setValue(CS.getInstruction(), SDValue(MN, 0));
8322     else
8323       setValue(CS.getInstruction(), Result.first);
8324   }
8325 
8326   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
8327   // call sequence. Furthermore the location of the chain and glue can change
8328   // when the AnyReg calling convention is used and the intrinsic returns a
8329   // value.
8330   if (IsAnyRegCC && HasDef) {
8331     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
8332     SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
8333     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
8334   } else
8335     DAG.ReplaceAllUsesWith(Call, MN);
8336   DAG.DeleteNode(Call);
8337 
8338   // Inform the Frame Information that we have a patchpoint in this function.
8339   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
8340 }
8341 
8342 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
8343                                             unsigned Intrinsic) {
8344   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8345   SDValue Op1 = getValue(I.getArgOperand(0));
8346   SDValue Op2;
8347   if (I.getNumArgOperands() > 1)
8348     Op2 = getValue(I.getArgOperand(1));
8349   SDLoc dl = getCurSDLoc();
8350   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8351   SDValue Res;
8352   FastMathFlags FMF;
8353   if (isa<FPMathOperator>(I))
8354     FMF = I.getFastMathFlags();
8355 
8356   switch (Intrinsic) {
8357   case Intrinsic::experimental_vector_reduce_fadd:
8358     if (FMF.isFast())
8359       Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2);
8360     else
8361       Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
8362     break;
8363   case Intrinsic::experimental_vector_reduce_fmul:
8364     if (FMF.isFast())
8365       Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2);
8366     else
8367       Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
8368     break;
8369   case Intrinsic::experimental_vector_reduce_add:
8370     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
8371     break;
8372   case Intrinsic::experimental_vector_reduce_mul:
8373     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
8374     break;
8375   case Intrinsic::experimental_vector_reduce_and:
8376     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
8377     break;
8378   case Intrinsic::experimental_vector_reduce_or:
8379     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
8380     break;
8381   case Intrinsic::experimental_vector_reduce_xor:
8382     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
8383     break;
8384   case Intrinsic::experimental_vector_reduce_smax:
8385     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
8386     break;
8387   case Intrinsic::experimental_vector_reduce_smin:
8388     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
8389     break;
8390   case Intrinsic::experimental_vector_reduce_umax:
8391     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
8392     break;
8393   case Intrinsic::experimental_vector_reduce_umin:
8394     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
8395     break;
8396   case Intrinsic::experimental_vector_reduce_fmax:
8397     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1);
8398     break;
8399   case Intrinsic::experimental_vector_reduce_fmin:
8400     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1);
8401     break;
8402   default:
8403     llvm_unreachable("Unhandled vector reduce intrinsic");
8404   }
8405   setValue(&I, Res);
8406 }
8407 
8408 /// Returns an AttributeList representing the attributes applied to the return
8409 /// value of the given call.
8410 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
8411   SmallVector<Attribute::AttrKind, 2> Attrs;
8412   if (CLI.RetSExt)
8413     Attrs.push_back(Attribute::SExt);
8414   if (CLI.RetZExt)
8415     Attrs.push_back(Attribute::ZExt);
8416   if (CLI.IsInReg)
8417     Attrs.push_back(Attribute::InReg);
8418 
8419   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
8420                             Attrs);
8421 }
8422 
8423 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
8424 /// implementation, which just calls LowerCall.
8425 /// FIXME: When all targets are
8426 /// migrated to using LowerCall, this hook should be integrated into SDISel.
8427 std::pair<SDValue, SDValue>
8428 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
8429   // Handle the incoming return values from the call.
8430   CLI.Ins.clear();
8431   Type *OrigRetTy = CLI.RetTy;
8432   SmallVector<EVT, 4> RetTys;
8433   SmallVector<uint64_t, 4> Offsets;
8434   auto &DL = CLI.DAG.getDataLayout();
8435   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
8436 
8437   if (CLI.IsPostTypeLegalization) {
8438     // If we are lowering a libcall after legalization, split the return type.
8439     SmallVector<EVT, 4> OldRetTys = std::move(RetTys);
8440     SmallVector<uint64_t, 4> OldOffsets = std::move(Offsets);
8441     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
8442       EVT RetVT = OldRetTys[i];
8443       uint64_t Offset = OldOffsets[i];
8444       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
8445       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
8446       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
8447       RetTys.append(NumRegs, RegisterVT);
8448       for (unsigned j = 0; j != NumRegs; ++j)
8449         Offsets.push_back(Offset + j * RegisterVTByteSZ);
8450     }
8451   }
8452 
8453   SmallVector<ISD::OutputArg, 4> Outs;
8454   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
8455 
8456   bool CanLowerReturn =
8457       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
8458                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
8459 
8460   SDValue DemoteStackSlot;
8461   int DemoteStackIdx = -100;
8462   if (!CanLowerReturn) {
8463     // FIXME: equivalent assert?
8464     // assert(!CS.hasInAllocaArgument() &&
8465     //        "sret demotion is incompatible with inalloca");
8466     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
8467     unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
8468     MachineFunction &MF = CLI.DAG.getMachineFunction();
8469     DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
8470     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
8471                                               DL.getAllocaAddrSpace());
8472 
8473     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
8474     ArgListEntry Entry;
8475     Entry.Node = DemoteStackSlot;
8476     Entry.Ty = StackSlotPtrType;
8477     Entry.IsSExt = false;
8478     Entry.IsZExt = false;
8479     Entry.IsInReg = false;
8480     Entry.IsSRet = true;
8481     Entry.IsNest = false;
8482     Entry.IsByVal = false;
8483     Entry.IsReturned = false;
8484     Entry.IsSwiftSelf = false;
8485     Entry.IsSwiftError = false;
8486     Entry.Alignment = Align;
8487     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
8488     CLI.NumFixedArgs += 1;
8489     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
8490 
8491     // sret demotion isn't compatible with tail-calls, since the sret argument
8492     // points into the callers stack frame.
8493     CLI.IsTailCall = false;
8494   } else {
8495     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8496       EVT VT = RetTys[I];
8497       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8498                                                      CLI.CallConv, VT);
8499       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8500                                                        CLI.CallConv, VT);
8501       for (unsigned i = 0; i != NumRegs; ++i) {
8502         ISD::InputArg MyFlags;
8503         MyFlags.VT = RegisterVT;
8504         MyFlags.ArgVT = VT;
8505         MyFlags.Used = CLI.IsReturnValueUsed;
8506         if (CLI.RetSExt)
8507           MyFlags.Flags.setSExt();
8508         if (CLI.RetZExt)
8509           MyFlags.Flags.setZExt();
8510         if (CLI.IsInReg)
8511           MyFlags.Flags.setInReg();
8512         CLI.Ins.push_back(MyFlags);
8513       }
8514     }
8515   }
8516 
8517   // We push in swifterror return as the last element of CLI.Ins.
8518   ArgListTy &Args = CLI.getArgs();
8519   if (supportSwiftError()) {
8520     for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8521       if (Args[i].IsSwiftError) {
8522         ISD::InputArg MyFlags;
8523         MyFlags.VT = getPointerTy(DL);
8524         MyFlags.ArgVT = EVT(getPointerTy(DL));
8525         MyFlags.Flags.setSwiftError();
8526         CLI.Ins.push_back(MyFlags);
8527       }
8528     }
8529   }
8530 
8531   // Handle all of the outgoing arguments.
8532   CLI.Outs.clear();
8533   CLI.OutVals.clear();
8534   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8535     SmallVector<EVT, 4> ValueVTs;
8536     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
8537     // FIXME: Split arguments if CLI.IsPostTypeLegalization
8538     Type *FinalType = Args[i].Ty;
8539     if (Args[i].IsByVal)
8540       FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
8541     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
8542         FinalType, CLI.CallConv, CLI.IsVarArg);
8543     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
8544          ++Value) {
8545       EVT VT = ValueVTs[Value];
8546       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
8547       SDValue Op = SDValue(Args[i].Node.getNode(),
8548                            Args[i].Node.getResNo() + Value);
8549       ISD::ArgFlagsTy Flags;
8550 
8551       // Certain targets (such as MIPS), may have a different ABI alignment
8552       // for a type depending on the context. Give the target a chance to
8553       // specify the alignment it wants.
8554       unsigned OriginalAlignment = getABIAlignmentForCallingConv(ArgTy, DL);
8555 
8556       if (Args[i].IsZExt)
8557         Flags.setZExt();
8558       if (Args[i].IsSExt)
8559         Flags.setSExt();
8560       if (Args[i].IsInReg) {
8561         // If we are using vectorcall calling convention, a structure that is
8562         // passed InReg - is surely an HVA
8563         if (CLI.CallConv == CallingConv::X86_VectorCall &&
8564             isa<StructType>(FinalType)) {
8565           // The first value of a structure is marked
8566           if (0 == Value)
8567             Flags.setHvaStart();
8568           Flags.setHva();
8569         }
8570         // Set InReg Flag
8571         Flags.setInReg();
8572       }
8573       if (Args[i].IsSRet)
8574         Flags.setSRet();
8575       if (Args[i].IsSwiftSelf)
8576         Flags.setSwiftSelf();
8577       if (Args[i].IsSwiftError)
8578         Flags.setSwiftError();
8579       if (Args[i].IsByVal)
8580         Flags.setByVal();
8581       if (Args[i].IsInAlloca) {
8582         Flags.setInAlloca();
8583         // Set the byval flag for CCAssignFn callbacks that don't know about
8584         // inalloca.  This way we can know how many bytes we should've allocated
8585         // and how many bytes a callee cleanup function will pop.  If we port
8586         // inalloca to more targets, we'll have to add custom inalloca handling
8587         // in the various CC lowering callbacks.
8588         Flags.setByVal();
8589       }
8590       if (Args[i].IsByVal || Args[i].IsInAlloca) {
8591         PointerType *Ty = cast<PointerType>(Args[i].Ty);
8592         Type *ElementTy = Ty->getElementType();
8593         Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8594         // For ByVal, alignment should come from FE.  BE will guess if this
8595         // info is not there but there are cases it cannot get right.
8596         unsigned FrameAlign;
8597         if (Args[i].Alignment)
8598           FrameAlign = Args[i].Alignment;
8599         else
8600           FrameAlign = getByValTypeAlignment(ElementTy, DL);
8601         Flags.setByValAlign(FrameAlign);
8602       }
8603       if (Args[i].IsNest)
8604         Flags.setNest();
8605       if (NeedsRegBlock)
8606         Flags.setInConsecutiveRegs();
8607       Flags.setOrigAlign(OriginalAlignment);
8608 
8609       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8610                                                  CLI.CallConv, VT);
8611       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8612                                                         CLI.CallConv, VT);
8613       SmallVector<SDValue, 4> Parts(NumParts);
8614       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
8615 
8616       if (Args[i].IsSExt)
8617         ExtendKind = ISD::SIGN_EXTEND;
8618       else if (Args[i].IsZExt)
8619         ExtendKind = ISD::ZERO_EXTEND;
8620 
8621       // Conservatively only handle 'returned' on non-vectors that can be lowered,
8622       // for now.
8623       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
8624           CanLowerReturn) {
8625         assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
8626                "unexpected use of 'returned'");
8627         // Before passing 'returned' to the target lowering code, ensure that
8628         // either the register MVT and the actual EVT are the same size or that
8629         // the return value and argument are extended in the same way; in these
8630         // cases it's safe to pass the argument register value unchanged as the
8631         // return register value (although it's at the target's option whether
8632         // to do so)
8633         // TODO: allow code generation to take advantage of partially preserved
8634         // registers rather than clobbering the entire register when the
8635         // parameter extension method is not compatible with the return
8636         // extension method
8637         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
8638             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
8639              CLI.RetZExt == Args[i].IsZExt))
8640           Flags.setReturned();
8641       }
8642 
8643       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
8644                      CLI.CS.getInstruction(), CLI.CallConv, ExtendKind);
8645 
8646       for (unsigned j = 0; j != NumParts; ++j) {
8647         // if it isn't first piece, alignment must be 1
8648         ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
8649                                i < CLI.NumFixedArgs,
8650                                i, j*Parts[j].getValueType().getStoreSize());
8651         if (NumParts > 1 && j == 0)
8652           MyFlags.Flags.setSplit();
8653         else if (j != 0) {
8654           MyFlags.Flags.setOrigAlign(1);
8655           if (j == NumParts - 1)
8656             MyFlags.Flags.setSplitEnd();
8657         }
8658 
8659         CLI.Outs.push_back(MyFlags);
8660         CLI.OutVals.push_back(Parts[j]);
8661       }
8662 
8663       if (NeedsRegBlock && Value == NumValues - 1)
8664         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
8665     }
8666   }
8667 
8668   SmallVector<SDValue, 4> InVals;
8669   CLI.Chain = LowerCall(CLI, InVals);
8670 
8671   // Update CLI.InVals to use outside of this function.
8672   CLI.InVals = InVals;
8673 
8674   // Verify that the target's LowerCall behaved as expected.
8675   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
8676          "LowerCall didn't return a valid chain!");
8677   assert((!CLI.IsTailCall || InVals.empty()) &&
8678          "LowerCall emitted a return value for a tail call!");
8679   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
8680          "LowerCall didn't emit the correct number of values!");
8681 
8682   // For a tail call, the return value is merely live-out and there aren't
8683   // any nodes in the DAG representing it. Return a special value to
8684   // indicate that a tail call has been emitted and no more Instructions
8685   // should be processed in the current block.
8686   if (CLI.IsTailCall) {
8687     CLI.DAG.setRoot(CLI.Chain);
8688     return std::make_pair(SDValue(), SDValue());
8689   }
8690 
8691 #ifndef NDEBUG
8692   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
8693     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
8694     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
8695            "LowerCall emitted a value with the wrong type!");
8696   }
8697 #endif
8698 
8699   SmallVector<SDValue, 4> ReturnValues;
8700   if (!CanLowerReturn) {
8701     // The instruction result is the result of loading from the
8702     // hidden sret parameter.
8703     SmallVector<EVT, 1> PVTs;
8704     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
8705 
8706     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
8707     assert(PVTs.size() == 1 && "Pointers should fit in one register");
8708     EVT PtrVT = PVTs[0];
8709 
8710     unsigned NumValues = RetTys.size();
8711     ReturnValues.resize(NumValues);
8712     SmallVector<SDValue, 4> Chains(NumValues);
8713 
8714     // An aggregate return value cannot wrap around the address space, so
8715     // offsets to its parts don't wrap either.
8716     SDNodeFlags Flags;
8717     Flags.setNoUnsignedWrap(true);
8718 
8719     for (unsigned i = 0; i < NumValues; ++i) {
8720       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
8721                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
8722                                                         PtrVT), Flags);
8723       SDValue L = CLI.DAG.getLoad(
8724           RetTys[i], CLI.DL, CLI.Chain, Add,
8725           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
8726                                             DemoteStackIdx, Offsets[i]),
8727           /* Alignment = */ 1);
8728       ReturnValues[i] = L;
8729       Chains[i] = L.getValue(1);
8730     }
8731 
8732     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
8733   } else {
8734     // Collect the legal value parts into potentially illegal values
8735     // that correspond to the original function's return values.
8736     Optional<ISD::NodeType> AssertOp;
8737     if (CLI.RetSExt)
8738       AssertOp = ISD::AssertSext;
8739     else if (CLI.RetZExt)
8740       AssertOp = ISD::AssertZext;
8741     unsigned CurReg = 0;
8742     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8743       EVT VT = RetTys[I];
8744       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8745                                                      CLI.CallConv, VT);
8746       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8747                                                        CLI.CallConv, VT);
8748 
8749       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
8750                                               NumRegs, RegisterVT, VT, nullptr,
8751                                               CLI.CallConv, AssertOp));
8752       CurReg += NumRegs;
8753     }
8754 
8755     // For a function returning void, there is no return value. We can't create
8756     // such a node, so we just return a null return value in that case. In
8757     // that case, nothing will actually look at the value.
8758     if (ReturnValues.empty())
8759       return std::make_pair(SDValue(), CLI.Chain);
8760   }
8761 
8762   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
8763                                 CLI.DAG.getVTList(RetTys), ReturnValues);
8764   return std::make_pair(Res, CLI.Chain);
8765 }
8766 
8767 void TargetLowering::LowerOperationWrapper(SDNode *N,
8768                                            SmallVectorImpl<SDValue> &Results,
8769                                            SelectionDAG &DAG) const {
8770   if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
8771     Results.push_back(Res);
8772 }
8773 
8774 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
8775   llvm_unreachable("LowerOperation not implemented for this target!");
8776 }
8777 
8778 void
8779 SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
8780   SDValue Op = getNonRegisterValue(V);
8781   assert((Op.getOpcode() != ISD::CopyFromReg ||
8782           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
8783          "Copy from a reg to the same reg!");
8784   assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
8785 
8786   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8787   // If this is an InlineAsm we have to match the registers required, not the
8788   // notional registers required by the type.
8789 
8790   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
8791                    None); // This is not an ABI copy.
8792   SDValue Chain = DAG.getEntryNode();
8793 
8794   ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
8795                               FuncInfo.PreferredExtendType.end())
8796                                  ? ISD::ANY_EXTEND
8797                                  : FuncInfo.PreferredExtendType[V];
8798   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
8799   PendingExports.push_back(Chain);
8800 }
8801 
8802 #include "llvm/CodeGen/SelectionDAGISel.h"
8803 
8804 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
8805 /// entry block, return true.  This includes arguments used by switches, since
8806 /// the switch may expand into multiple basic blocks.
8807 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
8808   // With FastISel active, we may be splitting blocks, so force creation
8809   // of virtual registers for all non-dead arguments.
8810   if (FastISel)
8811     return A->use_empty();
8812 
8813   const BasicBlock &Entry = A->getParent()->front();
8814   for (const User *U : A->users())
8815     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
8816       return false;  // Use not in entry block.
8817 
8818   return true;
8819 }
8820 
8821 using ArgCopyElisionMapTy =
8822     DenseMap<const Argument *,
8823              std::pair<const AllocaInst *, const StoreInst *>>;
8824 
8825 /// Scan the entry block of the function in FuncInfo for arguments that look
8826 /// like copies into a local alloca. Record any copied arguments in
8827 /// ArgCopyElisionCandidates.
8828 static void
8829 findArgumentCopyElisionCandidates(const DataLayout &DL,
8830                                   FunctionLoweringInfo *FuncInfo,
8831                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
8832   // Record the state of every static alloca used in the entry block. Argument
8833   // allocas are all used in the entry block, so we need approximately as many
8834   // entries as we have arguments.
8835   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
8836   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
8837   unsigned NumArgs = FuncInfo->Fn->arg_size();
8838   StaticAllocas.reserve(NumArgs * 2);
8839 
8840   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
8841     if (!V)
8842       return nullptr;
8843     V = V->stripPointerCasts();
8844     const auto *AI = dyn_cast<AllocaInst>(V);
8845     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
8846       return nullptr;
8847     auto Iter = StaticAllocas.insert({AI, Unknown});
8848     return &Iter.first->second;
8849   };
8850 
8851   // Look for stores of arguments to static allocas. Look through bitcasts and
8852   // GEPs to handle type coercions, as long as the alloca is fully initialized
8853   // by the store. Any non-store use of an alloca escapes it and any subsequent
8854   // unanalyzed store might write it.
8855   // FIXME: Handle structs initialized with multiple stores.
8856   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
8857     // Look for stores, and handle non-store uses conservatively.
8858     const auto *SI = dyn_cast<StoreInst>(&I);
8859     if (!SI) {
8860       // We will look through cast uses, so ignore them completely.
8861       if (I.isCast())
8862         continue;
8863       // Ignore debug info intrinsics, they don't escape or store to allocas.
8864       if (isa<DbgInfoIntrinsic>(I))
8865         continue;
8866       // This is an unknown instruction. Assume it escapes or writes to all
8867       // static alloca operands.
8868       for (const Use &U : I.operands()) {
8869         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
8870           *Info = StaticAllocaInfo::Clobbered;
8871       }
8872       continue;
8873     }
8874 
8875     // If the stored value is a static alloca, mark it as escaped.
8876     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
8877       *Info = StaticAllocaInfo::Clobbered;
8878 
8879     // Check if the destination is a static alloca.
8880     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
8881     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
8882     if (!Info)
8883       continue;
8884     const AllocaInst *AI = cast<AllocaInst>(Dst);
8885 
8886     // Skip allocas that have been initialized or clobbered.
8887     if (*Info != StaticAllocaInfo::Unknown)
8888       continue;
8889 
8890     // Check if the stored value is an argument, and that this store fully
8891     // initializes the alloca. Don't elide copies from the same argument twice.
8892     const Value *Val = SI->getValueOperand()->stripPointerCasts();
8893     const auto *Arg = dyn_cast<Argument>(Val);
8894     if (!Arg || Arg->hasInAllocaAttr() || Arg->hasByValAttr() ||
8895         Arg->getType()->isEmptyTy() ||
8896         DL.getTypeStoreSize(Arg->getType()) !=
8897             DL.getTypeAllocSize(AI->getAllocatedType()) ||
8898         ArgCopyElisionCandidates.count(Arg)) {
8899       *Info = StaticAllocaInfo::Clobbered;
8900       continue;
8901     }
8902 
8903     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
8904                       << '\n');
8905 
8906     // Mark this alloca and store for argument copy elision.
8907     *Info = StaticAllocaInfo::Elidable;
8908     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
8909 
8910     // Stop scanning if we've seen all arguments. This will happen early in -O0
8911     // builds, which is useful, because -O0 builds have large entry blocks and
8912     // many allocas.
8913     if (ArgCopyElisionCandidates.size() == NumArgs)
8914       break;
8915   }
8916 }
8917 
8918 /// Try to elide argument copies from memory into a local alloca. Succeeds if
8919 /// ArgVal is a load from a suitable fixed stack object.
8920 static void tryToElideArgumentCopy(
8921     FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
8922     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
8923     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
8924     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
8925     SDValue ArgVal, bool &ArgHasUses) {
8926   // Check if this is a load from a fixed stack object.
8927   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
8928   if (!LNode)
8929     return;
8930   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
8931   if (!FINode)
8932     return;
8933 
8934   // Check that the fixed stack object is the right size and alignment.
8935   // Look at the alignment that the user wrote on the alloca instead of looking
8936   // at the stack object.
8937   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
8938   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
8939   const AllocaInst *AI = ArgCopyIter->second.first;
8940   int FixedIndex = FINode->getIndex();
8941   int &AllocaIndex = FuncInfo->StaticAllocaMap[AI];
8942   int OldIndex = AllocaIndex;
8943   MachineFrameInfo &MFI = FuncInfo->MF->getFrameInfo();
8944   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
8945     LLVM_DEBUG(
8946         dbgs() << "  argument copy elision failed due to bad fixed stack "
8947                   "object size\n");
8948     return;
8949   }
8950   unsigned RequiredAlignment = AI->getAlignment();
8951   if (!RequiredAlignment) {
8952     RequiredAlignment = FuncInfo->MF->getDataLayout().getABITypeAlignment(
8953         AI->getAllocatedType());
8954   }
8955   if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
8956     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
8957                          "greater than stack argument alignment ("
8958                       << RequiredAlignment << " vs "
8959                       << MFI.getObjectAlignment(FixedIndex) << ")\n");
8960     return;
8961   }
8962 
8963   // Perform the elision. Delete the old stack object and replace its only use
8964   // in the variable info map. Mark the stack object as mutable.
8965   LLVM_DEBUG({
8966     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
8967            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
8968            << '\n';
8969   });
8970   MFI.RemoveStackObject(OldIndex);
8971   MFI.setIsImmutableObjectIndex(FixedIndex, false);
8972   AllocaIndex = FixedIndex;
8973   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
8974   Chains.push_back(ArgVal.getValue(1));
8975 
8976   // Avoid emitting code for the store implementing the copy.
8977   const StoreInst *SI = ArgCopyIter->second.second;
8978   ElidedArgCopyInstrs.insert(SI);
8979 
8980   // Check for uses of the argument again so that we can avoid exporting ArgVal
8981   // if it is't used by anything other than the store.
8982   for (const Value *U : Arg.users()) {
8983     if (U != SI) {
8984       ArgHasUses = true;
8985       break;
8986     }
8987   }
8988 }
8989 
8990 void SelectionDAGISel::LowerArguments(const Function &F) {
8991   SelectionDAG &DAG = SDB->DAG;
8992   SDLoc dl = SDB->getCurSDLoc();
8993   const DataLayout &DL = DAG.getDataLayout();
8994   SmallVector<ISD::InputArg, 16> Ins;
8995 
8996   if (!FuncInfo->CanLowerReturn) {
8997     // Put in an sret pointer parameter before all the other parameters.
8998     SmallVector<EVT, 1> ValueVTs;
8999     ComputeValueVTs(*TLI, DAG.getDataLayout(),
9000                     F.getReturnType()->getPointerTo(
9001                         DAG.getDataLayout().getAllocaAddrSpace()),
9002                     ValueVTs);
9003 
9004     // NOTE: Assuming that a pointer will never break down to more than one VT
9005     // or one register.
9006     ISD::ArgFlagsTy Flags;
9007     Flags.setSRet();
9008     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
9009     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
9010                          ISD::InputArg::NoArgIndex, 0);
9011     Ins.push_back(RetArg);
9012   }
9013 
9014   // Look for stores of arguments to static allocas. Mark such arguments with a
9015   // flag to ask the target to give us the memory location of that argument if
9016   // available.
9017   ArgCopyElisionMapTy ArgCopyElisionCandidates;
9018   findArgumentCopyElisionCandidates(DL, FuncInfo, ArgCopyElisionCandidates);
9019 
9020   // Set up the incoming argument description vector.
9021   for (const Argument &Arg : F.args()) {
9022     unsigned ArgNo = Arg.getArgNo();
9023     SmallVector<EVT, 4> ValueVTs;
9024     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
9025     bool isArgValueUsed = !Arg.use_empty();
9026     unsigned PartBase = 0;
9027     Type *FinalType = Arg.getType();
9028     if (Arg.hasAttribute(Attribute::ByVal))
9029       FinalType = cast<PointerType>(FinalType)->getElementType();
9030     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
9031         FinalType, F.getCallingConv(), F.isVarArg());
9032     for (unsigned Value = 0, NumValues = ValueVTs.size();
9033          Value != NumValues; ++Value) {
9034       EVT VT = ValueVTs[Value];
9035       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
9036       ISD::ArgFlagsTy Flags;
9037 
9038       // Certain targets (such as MIPS), may have a different ABI alignment
9039       // for a type depending on the context. Give the target a chance to
9040       // specify the alignment it wants.
9041       unsigned OriginalAlignment =
9042           TLI->getABIAlignmentForCallingConv(ArgTy, DL);
9043 
9044       if (Arg.hasAttribute(Attribute::ZExt))
9045         Flags.setZExt();
9046       if (Arg.hasAttribute(Attribute::SExt))
9047         Flags.setSExt();
9048       if (Arg.hasAttribute(Attribute::InReg)) {
9049         // If we are using vectorcall calling convention, a structure that is
9050         // passed InReg - is surely an HVA
9051         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
9052             isa<StructType>(Arg.getType())) {
9053           // The first value of a structure is marked
9054           if (0 == Value)
9055             Flags.setHvaStart();
9056           Flags.setHva();
9057         }
9058         // Set InReg Flag
9059         Flags.setInReg();
9060       }
9061       if (Arg.hasAttribute(Attribute::StructRet))
9062         Flags.setSRet();
9063       if (Arg.hasAttribute(Attribute::SwiftSelf))
9064         Flags.setSwiftSelf();
9065       if (Arg.hasAttribute(Attribute::SwiftError))
9066         Flags.setSwiftError();
9067       if (Arg.hasAttribute(Attribute::ByVal))
9068         Flags.setByVal();
9069       if (Arg.hasAttribute(Attribute::InAlloca)) {
9070         Flags.setInAlloca();
9071         // Set the byval flag for CCAssignFn callbacks that don't know about
9072         // inalloca.  This way we can know how many bytes we should've allocated
9073         // and how many bytes a callee cleanup function will pop.  If we port
9074         // inalloca to more targets, we'll have to add custom inalloca handling
9075         // in the various CC lowering callbacks.
9076         Flags.setByVal();
9077       }
9078       if (F.getCallingConv() == CallingConv::X86_INTR) {
9079         // IA Interrupt passes frame (1st parameter) by value in the stack.
9080         if (ArgNo == 0)
9081           Flags.setByVal();
9082       }
9083       if (Flags.isByVal() || Flags.isInAlloca()) {
9084         PointerType *Ty = cast<PointerType>(Arg.getType());
9085         Type *ElementTy = Ty->getElementType();
9086         Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
9087         // For ByVal, alignment should be passed from FE.  BE will guess if
9088         // this info is not there but there are cases it cannot get right.
9089         unsigned FrameAlign;
9090         if (Arg.getParamAlignment())
9091           FrameAlign = Arg.getParamAlignment();
9092         else
9093           FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
9094         Flags.setByValAlign(FrameAlign);
9095       }
9096       if (Arg.hasAttribute(Attribute::Nest))
9097         Flags.setNest();
9098       if (NeedsRegBlock)
9099         Flags.setInConsecutiveRegs();
9100       Flags.setOrigAlign(OriginalAlignment);
9101       if (ArgCopyElisionCandidates.count(&Arg))
9102         Flags.setCopyElisionCandidate();
9103 
9104       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
9105           *CurDAG->getContext(), F.getCallingConv(), VT);
9106       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
9107           *CurDAG->getContext(), F.getCallingConv(), VT);
9108       for (unsigned i = 0; i != NumRegs; ++i) {
9109         ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
9110                               ArgNo, PartBase+i*RegisterVT.getStoreSize());
9111         if (NumRegs > 1 && i == 0)
9112           MyFlags.Flags.setSplit();
9113         // if it isn't first piece, alignment must be 1
9114         else if (i > 0) {
9115           MyFlags.Flags.setOrigAlign(1);
9116           if (i == NumRegs - 1)
9117             MyFlags.Flags.setSplitEnd();
9118         }
9119         Ins.push_back(MyFlags);
9120       }
9121       if (NeedsRegBlock && Value == NumValues - 1)
9122         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
9123       PartBase += VT.getStoreSize();
9124     }
9125   }
9126 
9127   // Call the target to set up the argument values.
9128   SmallVector<SDValue, 8> InVals;
9129   SDValue NewRoot = TLI->LowerFormalArguments(
9130       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
9131 
9132   // Verify that the target's LowerFormalArguments behaved as expected.
9133   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
9134          "LowerFormalArguments didn't return a valid chain!");
9135   assert(InVals.size() == Ins.size() &&
9136          "LowerFormalArguments didn't emit the correct number of values!");
9137   LLVM_DEBUG({
9138     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
9139       assert(InVals[i].getNode() &&
9140              "LowerFormalArguments emitted a null value!");
9141       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
9142              "LowerFormalArguments emitted a value with the wrong type!");
9143     }
9144   });
9145 
9146   // Update the DAG with the new chain value resulting from argument lowering.
9147   DAG.setRoot(NewRoot);
9148 
9149   // Set up the argument values.
9150   unsigned i = 0;
9151   if (!FuncInfo->CanLowerReturn) {
9152     // Create a virtual register for the sret pointer, and put in a copy
9153     // from the sret argument into it.
9154     SmallVector<EVT, 1> ValueVTs;
9155     ComputeValueVTs(*TLI, DAG.getDataLayout(),
9156                     F.getReturnType()->getPointerTo(
9157                         DAG.getDataLayout().getAllocaAddrSpace()),
9158                     ValueVTs);
9159     MVT VT = ValueVTs[0].getSimpleVT();
9160     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
9161     Optional<ISD::NodeType> AssertOp = None;
9162     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
9163                                         nullptr, F.getCallingConv(), AssertOp);
9164 
9165     MachineFunction& MF = SDB->DAG.getMachineFunction();
9166     MachineRegisterInfo& RegInfo = MF.getRegInfo();
9167     unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
9168     FuncInfo->DemoteRegister = SRetReg;
9169     NewRoot =
9170         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
9171     DAG.setRoot(NewRoot);
9172 
9173     // i indexes lowered arguments.  Bump it past the hidden sret argument.
9174     ++i;
9175   }
9176 
9177   SmallVector<SDValue, 4> Chains;
9178   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
9179   for (const Argument &Arg : F.args()) {
9180     SmallVector<SDValue, 4> ArgValues;
9181     SmallVector<EVT, 4> ValueVTs;
9182     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
9183     unsigned NumValues = ValueVTs.size();
9184     if (NumValues == 0)
9185       continue;
9186 
9187     bool ArgHasUses = !Arg.use_empty();
9188 
9189     // Elide the copying store if the target loaded this argument from a
9190     // suitable fixed stack object.
9191     if (Ins[i].Flags.isCopyElisionCandidate()) {
9192       tryToElideArgumentCopy(FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
9193                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
9194                              InVals[i], ArgHasUses);
9195     }
9196 
9197     // If this argument is unused then remember its value. It is used to generate
9198     // debugging information.
9199     bool isSwiftErrorArg =
9200         TLI->supportSwiftError() &&
9201         Arg.hasAttribute(Attribute::SwiftError);
9202     if (!ArgHasUses && !isSwiftErrorArg) {
9203       SDB->setUnusedArgValue(&Arg, InVals[i]);
9204 
9205       // Also remember any frame index for use in FastISel.
9206       if (FrameIndexSDNode *FI =
9207           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
9208         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9209     }
9210 
9211     for (unsigned Val = 0; Val != NumValues; ++Val) {
9212       EVT VT = ValueVTs[Val];
9213       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
9214                                                       F.getCallingConv(), VT);
9215       unsigned NumParts = TLI->getNumRegistersForCallingConv(
9216           *CurDAG->getContext(), F.getCallingConv(), VT);
9217 
9218       // Even an apparant 'unused' swifterror argument needs to be returned. So
9219       // we do generate a copy for it that can be used on return from the
9220       // function.
9221       if (ArgHasUses || isSwiftErrorArg) {
9222         Optional<ISD::NodeType> AssertOp;
9223         if (Arg.hasAttribute(Attribute::SExt))
9224           AssertOp = ISD::AssertSext;
9225         else if (Arg.hasAttribute(Attribute::ZExt))
9226           AssertOp = ISD::AssertZext;
9227 
9228         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
9229                                              PartVT, VT, nullptr,
9230                                              F.getCallingConv(), AssertOp));
9231       }
9232 
9233       i += NumParts;
9234     }
9235 
9236     // We don't need to do anything else for unused arguments.
9237     if (ArgValues.empty())
9238       continue;
9239 
9240     // Note down frame index.
9241     if (FrameIndexSDNode *FI =
9242         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
9243       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9244 
9245     SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
9246                                      SDB->getCurSDLoc());
9247 
9248     SDB->setValue(&Arg, Res);
9249     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
9250       // We want to associate the argument with the frame index, among
9251       // involved operands, that correspond to the lowest address. The
9252       // getCopyFromParts function, called earlier, is swapping the order of
9253       // the operands to BUILD_PAIR depending on endianness. The result of
9254       // that swapping is that the least significant bits of the argument will
9255       // be in the first operand of the BUILD_PAIR node, and the most
9256       // significant bits will be in the second operand.
9257       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9258       if (LoadSDNode *LNode =
9259           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
9260         if (FrameIndexSDNode *FI =
9261             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
9262           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9263     }
9264 
9265     // Update the SwiftErrorVRegDefMap.
9266     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
9267       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9268       if (TargetRegisterInfo::isVirtualRegister(Reg))
9269         FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB,
9270                                            FuncInfo->SwiftErrorArg, Reg);
9271     }
9272 
9273     // If this argument is live outside of the entry block, insert a copy from
9274     // wherever we got it to the vreg that other BB's will reference it as.
9275     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
9276       // If we can, though, try to skip creating an unnecessary vreg.
9277       // FIXME: This isn't very clean... it would be nice to make this more
9278       // general.  It's also subtly incompatible with the hacks FastISel
9279       // uses with vregs.
9280       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9281       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
9282         FuncInfo->ValueMap[&Arg] = Reg;
9283         continue;
9284       }
9285     }
9286     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
9287       FuncInfo->InitializeRegForValue(&Arg);
9288       SDB->CopyToExportRegsIfNeeded(&Arg);
9289     }
9290   }
9291 
9292   if (!Chains.empty()) {
9293     Chains.push_back(NewRoot);
9294     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
9295   }
9296 
9297   DAG.setRoot(NewRoot);
9298 
9299   assert(i == InVals.size() && "Argument register count mismatch!");
9300 
9301   // If any argument copy elisions occurred and we have debug info, update the
9302   // stale frame indices used in the dbg.declare variable info table.
9303   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
9304   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
9305     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
9306       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
9307       if (I != ArgCopyElisionFrameIndexMap.end())
9308         VI.Slot = I->second;
9309     }
9310   }
9311 
9312   // Finally, if the target has anything special to do, allow it to do so.
9313   EmitFunctionEntryCode();
9314 }
9315 
9316 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
9317 /// ensure constants are generated when needed.  Remember the virtual registers
9318 /// that need to be added to the Machine PHI nodes as input.  We cannot just
9319 /// directly add them, because expansion might result in multiple MBB's for one
9320 /// BB.  As such, the start of the BB might correspond to a different MBB than
9321 /// the end.
9322 void
9323 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
9324   const Instruction *TI = LLVMBB->getTerminator();
9325 
9326   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
9327 
9328   // Check PHI nodes in successors that expect a value to be available from this
9329   // block.
9330   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
9331     const BasicBlock *SuccBB = TI->getSuccessor(succ);
9332     if (!isa<PHINode>(SuccBB->begin())) continue;
9333     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
9334 
9335     // If this terminator has multiple identical successors (common for
9336     // switches), only handle each succ once.
9337     if (!SuccsHandled.insert(SuccMBB).second)
9338       continue;
9339 
9340     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
9341 
9342     // At this point we know that there is a 1-1 correspondence between LLVM PHI
9343     // nodes and Machine PHI nodes, but the incoming operands have not been
9344     // emitted yet.
9345     for (const PHINode &PN : SuccBB->phis()) {
9346       // Ignore dead phi's.
9347       if (PN.use_empty())
9348         continue;
9349 
9350       // Skip empty types
9351       if (PN.getType()->isEmptyTy())
9352         continue;
9353 
9354       unsigned Reg;
9355       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
9356 
9357       if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
9358         unsigned &RegOut = ConstantsOut[C];
9359         if (RegOut == 0) {
9360           RegOut = FuncInfo.CreateRegs(C->getType());
9361           CopyValueToVirtualRegister(C, RegOut);
9362         }
9363         Reg = RegOut;
9364       } else {
9365         DenseMap<const Value *, unsigned>::iterator I =
9366           FuncInfo.ValueMap.find(PHIOp);
9367         if (I != FuncInfo.ValueMap.end())
9368           Reg = I->second;
9369         else {
9370           assert(isa<AllocaInst>(PHIOp) &&
9371                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
9372                  "Didn't codegen value into a register!??");
9373           Reg = FuncInfo.CreateRegs(PHIOp->getType());
9374           CopyValueToVirtualRegister(PHIOp, Reg);
9375         }
9376       }
9377 
9378       // Remember that this register needs to added to the machine PHI node as
9379       // the input for this MBB.
9380       SmallVector<EVT, 4> ValueVTs;
9381       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9382       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
9383       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
9384         EVT VT = ValueVTs[vti];
9385         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
9386         for (unsigned i = 0, e = NumRegisters; i != e; ++i)
9387           FuncInfo.PHINodesToUpdate.push_back(
9388               std::make_pair(&*MBBI++, Reg + i));
9389         Reg += NumRegisters;
9390       }
9391     }
9392   }
9393 
9394   ConstantsOut.clear();
9395 }
9396 
9397 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
9398 /// is 0.
9399 MachineBasicBlock *
9400 SelectionDAGBuilder::StackProtectorDescriptor::
9401 AddSuccessorMBB(const BasicBlock *BB,
9402                 MachineBasicBlock *ParentMBB,
9403                 bool IsLikely,
9404                 MachineBasicBlock *SuccMBB) {
9405   // If SuccBB has not been created yet, create it.
9406   if (!SuccMBB) {
9407     MachineFunction *MF = ParentMBB->getParent();
9408     MachineFunction::iterator BBI(ParentMBB);
9409     SuccMBB = MF->CreateMachineBasicBlock(BB);
9410     MF->insert(++BBI, SuccMBB);
9411   }
9412   // Add it as a successor of ParentMBB.
9413   ParentMBB->addSuccessor(
9414       SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
9415   return SuccMBB;
9416 }
9417 
9418 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
9419   MachineFunction::iterator I(MBB);
9420   if (++I == FuncInfo.MF->end())
9421     return nullptr;
9422   return &*I;
9423 }
9424 
9425 /// During lowering new call nodes can be created (such as memset, etc.).
9426 /// Those will become new roots of the current DAG, but complications arise
9427 /// when they are tail calls. In such cases, the call lowering will update
9428 /// the root, but the builder still needs to know that a tail call has been
9429 /// lowered in order to avoid generating an additional return.
9430 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
9431   // If the node is null, we do have a tail call.
9432   if (MaybeTC.getNode() != nullptr)
9433     DAG.setRoot(MaybeTC);
9434   else
9435     HasTailCall = true;
9436 }
9437 
9438 uint64_t
9439 SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters,
9440                                        unsigned First, unsigned Last) const {
9441   assert(Last >= First);
9442   const APInt &LowCase = Clusters[First].Low->getValue();
9443   const APInt &HighCase = Clusters[Last].High->getValue();
9444   assert(LowCase.getBitWidth() == HighCase.getBitWidth());
9445 
9446   // FIXME: A range of consecutive cases has 100% density, but only requires one
9447   // comparison to lower. We should discriminate against such consecutive ranges
9448   // in jump tables.
9449 
9450   return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1;
9451 }
9452 
9453 uint64_t SelectionDAGBuilder::getJumpTableNumCases(
9454     const SmallVectorImpl<unsigned> &TotalCases, unsigned First,
9455     unsigned Last) const {
9456   assert(Last >= First);
9457   assert(TotalCases[Last] >= TotalCases[First]);
9458   uint64_t NumCases =
9459       TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
9460   return NumCases;
9461 }
9462 
9463 bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
9464                                          unsigned First, unsigned Last,
9465                                          const SwitchInst *SI,
9466                                          MachineBasicBlock *DefaultMBB,
9467                                          CaseCluster &JTCluster) {
9468   assert(First <= Last);
9469 
9470   auto Prob = BranchProbability::getZero();
9471   unsigned NumCmps = 0;
9472   std::vector<MachineBasicBlock*> Table;
9473   DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
9474 
9475   // Initialize probabilities in JTProbs.
9476   for (unsigned I = First; I <= Last; ++I)
9477     JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
9478 
9479   for (unsigned I = First; I <= Last; ++I) {
9480     assert(Clusters[I].Kind == CC_Range);
9481     Prob += Clusters[I].Prob;
9482     const APInt &Low = Clusters[I].Low->getValue();
9483     const APInt &High = Clusters[I].High->getValue();
9484     NumCmps += (Low == High) ? 1 : 2;
9485     if (I != First) {
9486       // Fill the gap between this and the previous cluster.
9487       const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
9488       assert(PreviousHigh.slt(Low));
9489       uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
9490       for (uint64_t J = 0; J < Gap; J++)
9491         Table.push_back(DefaultMBB);
9492     }
9493     uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
9494     for (uint64_t J = 0; J < ClusterSize; ++J)
9495       Table.push_back(Clusters[I].MBB);
9496     JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
9497   }
9498 
9499   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9500   unsigned NumDests = JTProbs.size();
9501   if (TLI.isSuitableForBitTests(
9502           NumDests, NumCmps, Clusters[First].Low->getValue(),
9503           Clusters[Last].High->getValue(), DAG.getDataLayout())) {
9504     // Clusters[First..Last] should be lowered as bit tests instead.
9505     return false;
9506   }
9507 
9508   // Create the MBB that will load from and jump through the table.
9509   // Note: We create it here, but it's not inserted into the function yet.
9510   MachineFunction *CurMF = FuncInfo.MF;
9511   MachineBasicBlock *JumpTableMBB =
9512       CurMF->CreateMachineBasicBlock(SI->getParent());
9513 
9514   // Add successors. Note: use table order for determinism.
9515   SmallPtrSet<MachineBasicBlock *, 8> Done;
9516   for (MachineBasicBlock *Succ : Table) {
9517     if (Done.count(Succ))
9518       continue;
9519     addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
9520     Done.insert(Succ);
9521   }
9522   JumpTableMBB->normalizeSuccProbs();
9523 
9524   unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
9525                      ->createJumpTableIndex(Table);
9526 
9527   // Set up the jump table info.
9528   JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
9529   JumpTableHeader JTH(Clusters[First].Low->getValue(),
9530                       Clusters[Last].High->getValue(), SI->getCondition(),
9531                       nullptr, false);
9532   JTCases.emplace_back(std::move(JTH), std::move(JT));
9533 
9534   JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
9535                                      JTCases.size() - 1, Prob);
9536   return true;
9537 }
9538 
9539 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
9540                                          const SwitchInst *SI,
9541                                          MachineBasicBlock *DefaultMBB) {
9542 #ifndef NDEBUG
9543   // Clusters must be non-empty, sorted, and only contain Range clusters.
9544   assert(!Clusters.empty());
9545   for (CaseCluster &C : Clusters)
9546     assert(C.Kind == CC_Range);
9547   for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
9548     assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
9549 #endif
9550 
9551   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9552   if (!TLI.areJTsAllowed(SI->getParent()->getParent()))
9553     return;
9554 
9555   const int64_t N = Clusters.size();
9556   const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
9557   const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
9558 
9559   if (N < 2 || N < MinJumpTableEntries)
9560     return;
9561 
9562   // TotalCases[i]: Total nbr of cases in Clusters[0..i].
9563   SmallVector<unsigned, 8> TotalCases(N);
9564   for (unsigned i = 0; i < N; ++i) {
9565     const APInt &Hi = Clusters[i].High->getValue();
9566     const APInt &Lo = Clusters[i].Low->getValue();
9567     TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
9568     if (i != 0)
9569       TotalCases[i] += TotalCases[i - 1];
9570   }
9571 
9572   // Cheap case: the whole range may be suitable for jump table.
9573   uint64_t Range = getJumpTableRange(Clusters,0, N - 1);
9574   uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1);
9575   assert(NumCases < UINT64_MAX / 100);
9576   assert(Range >= NumCases);
9577   if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9578     CaseCluster JTCluster;
9579     if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
9580       Clusters[0] = JTCluster;
9581       Clusters.resize(1);
9582       return;
9583     }
9584   }
9585 
9586   // The algorithm below is not suitable for -O0.
9587   if (TM.getOptLevel() == CodeGenOpt::None)
9588     return;
9589 
9590   // Split Clusters into minimum number of dense partitions. The algorithm uses
9591   // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
9592   // for the Case Statement'" (1994), but builds the MinPartitions array in
9593   // reverse order to make it easier to reconstruct the partitions in ascending
9594   // order. In the choice between two optimal partitionings, it picks the one
9595   // which yields more jump tables.
9596 
9597   // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9598   SmallVector<unsigned, 8> MinPartitions(N);
9599   // LastElement[i] is the last element of the partition starting at i.
9600   SmallVector<unsigned, 8> LastElement(N);
9601   // PartitionsScore[i] is used to break ties when choosing between two
9602   // partitionings resulting in the same number of partitions.
9603   SmallVector<unsigned, 8> PartitionsScore(N);
9604   // For PartitionsScore, a small number of comparisons is considered as good as
9605   // a jump table and a single comparison is considered better than a jump
9606   // table.
9607   enum PartitionScores : unsigned {
9608     NoTable = 0,
9609     Table = 1,
9610     FewCases = 1,
9611     SingleCase = 2
9612   };
9613 
9614   // Base case: There is only one way to partition Clusters[N-1].
9615   MinPartitions[N - 1] = 1;
9616   LastElement[N - 1] = N - 1;
9617   PartitionsScore[N - 1] = PartitionScores::SingleCase;
9618 
9619   // Note: loop indexes are signed to avoid underflow.
9620   for (int64_t i = N - 2; i >= 0; i--) {
9621     // Find optimal partitioning of Clusters[i..N-1].
9622     // Baseline: Put Clusters[i] into a partition on its own.
9623     MinPartitions[i] = MinPartitions[i + 1] + 1;
9624     LastElement[i] = i;
9625     PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
9626 
9627     // Search for a solution that results in fewer partitions.
9628     for (int64_t j = N - 1; j > i; j--) {
9629       // Try building a partition from Clusters[i..j].
9630       uint64_t Range = getJumpTableRange(Clusters, i, j);
9631       uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j);
9632       assert(NumCases < UINT64_MAX / 100);
9633       assert(Range >= NumCases);
9634       if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9635         unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9636         unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
9637         int64_t NumEntries = j - i + 1;
9638 
9639         if (NumEntries == 1)
9640           Score += PartitionScores::SingleCase;
9641         else if (NumEntries <= SmallNumberOfEntries)
9642           Score += PartitionScores::FewCases;
9643         else if (NumEntries >= MinJumpTableEntries)
9644           Score += PartitionScores::Table;
9645 
9646         // If this leads to fewer partitions, or to the same number of
9647         // partitions with better score, it is a better partitioning.
9648         if (NumPartitions < MinPartitions[i] ||
9649             (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
9650           MinPartitions[i] = NumPartitions;
9651           LastElement[i] = j;
9652           PartitionsScore[i] = Score;
9653         }
9654       }
9655     }
9656   }
9657 
9658   // Iterate over the partitions, replacing some with jump tables in-place.
9659   unsigned DstIndex = 0;
9660   for (unsigned First = 0, Last; First < N; First = Last + 1) {
9661     Last = LastElement[First];
9662     assert(Last >= First);
9663     assert(DstIndex <= First);
9664     unsigned NumClusters = Last - First + 1;
9665 
9666     CaseCluster JTCluster;
9667     if (NumClusters >= MinJumpTableEntries &&
9668         buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
9669       Clusters[DstIndex++] = JTCluster;
9670     } else {
9671       for (unsigned I = First; I <= Last; ++I)
9672         std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
9673     }
9674   }
9675   Clusters.resize(DstIndex);
9676 }
9677 
9678 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
9679                                         unsigned First, unsigned Last,
9680                                         const SwitchInst *SI,
9681                                         CaseCluster &BTCluster) {
9682   assert(First <= Last);
9683   if (First == Last)
9684     return false;
9685 
9686   BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9687   unsigned NumCmps = 0;
9688   for (int64_t I = First; I <= Last; ++I) {
9689     assert(Clusters[I].Kind == CC_Range);
9690     Dests.set(Clusters[I].MBB->getNumber());
9691     NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
9692   }
9693   unsigned NumDests = Dests.count();
9694 
9695   APInt Low = Clusters[First].Low->getValue();
9696   APInt High = Clusters[Last].High->getValue();
9697   assert(Low.slt(High));
9698 
9699   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9700   const DataLayout &DL = DAG.getDataLayout();
9701   if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL))
9702     return false;
9703 
9704   APInt LowBound;
9705   APInt CmpRange;
9706 
9707   const int BitWidth = TLI.getPointerTy(DL).getSizeInBits();
9708   assert(TLI.rangeFitsInWord(Low, High, DL) &&
9709          "Case range must fit in bit mask!");
9710 
9711   // Check if the clusters cover a contiguous range such that no value in the
9712   // range will jump to the default statement.
9713   bool ContiguousRange = true;
9714   for (int64_t I = First + 1; I <= Last; ++I) {
9715     if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
9716       ContiguousRange = false;
9717       break;
9718     }
9719   }
9720 
9721   if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
9722     // Optimize the case where all the case values fit in a word without having
9723     // to subtract minValue. In this case, we can optimize away the subtraction.
9724     LowBound = APInt::getNullValue(Low.getBitWidth());
9725     CmpRange = High;
9726     ContiguousRange = false;
9727   } else {
9728     LowBound = Low;
9729     CmpRange = High - Low;
9730   }
9731 
9732   CaseBitsVector CBV;
9733   auto TotalProb = BranchProbability::getZero();
9734   for (unsigned i = First; i <= Last; ++i) {
9735     // Find the CaseBits for this destination.
9736     unsigned j;
9737     for (j = 0; j < CBV.size(); ++j)
9738       if (CBV[j].BB == Clusters[i].MBB)
9739         break;
9740     if (j == CBV.size())
9741       CBV.push_back(
9742           CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
9743     CaseBits *CB = &CBV[j];
9744 
9745     // Update Mask, Bits and ExtraProb.
9746     uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
9747     uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
9748     assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
9749     CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
9750     CB->Bits += Hi - Lo + 1;
9751     CB->ExtraProb += Clusters[i].Prob;
9752     TotalProb += Clusters[i].Prob;
9753   }
9754 
9755   BitTestInfo BTI;
9756   llvm::sort(CBV, [](const CaseBits &a, const CaseBits &b) {
9757     // Sort by probability first, number of bits second, bit mask third.
9758     if (a.ExtraProb != b.ExtraProb)
9759       return a.ExtraProb > b.ExtraProb;
9760     if (a.Bits != b.Bits)
9761       return a.Bits > b.Bits;
9762     return a.Mask < b.Mask;
9763   });
9764 
9765   for (auto &CB : CBV) {
9766     MachineBasicBlock *BitTestBB =
9767         FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
9768     BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
9769   }
9770   BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
9771                             SI->getCondition(), -1U, MVT::Other, false,
9772                             ContiguousRange, nullptr, nullptr, std::move(BTI),
9773                             TotalProb);
9774 
9775   BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
9776                                     BitTestCases.size() - 1, TotalProb);
9777   return true;
9778 }
9779 
9780 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
9781                                               const SwitchInst *SI) {
9782 // Partition Clusters into as few subsets as possible, where each subset has a
9783 // range that fits in a machine word and has <= 3 unique destinations.
9784 
9785 #ifndef NDEBUG
9786   // Clusters must be sorted and contain Range or JumpTable clusters.
9787   assert(!Clusters.empty());
9788   assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
9789   for (const CaseCluster &C : Clusters)
9790     assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
9791   for (unsigned i = 1; i < Clusters.size(); ++i)
9792     assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
9793 #endif
9794 
9795   // The algorithm below is not suitable for -O0.
9796   if (TM.getOptLevel() == CodeGenOpt::None)
9797     return;
9798 
9799   // If target does not have legal shift left, do not emit bit tests at all.
9800   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9801   const DataLayout &DL = DAG.getDataLayout();
9802 
9803   EVT PTy = TLI.getPointerTy(DL);
9804   if (!TLI.isOperationLegal(ISD::SHL, PTy))
9805     return;
9806 
9807   int BitWidth = PTy.getSizeInBits();
9808   const int64_t N = Clusters.size();
9809 
9810   // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9811   SmallVector<unsigned, 8> MinPartitions(N);
9812   // LastElement[i] is the last element of the partition starting at i.
9813   SmallVector<unsigned, 8> LastElement(N);
9814 
9815   // FIXME: This might not be the best algorithm for finding bit test clusters.
9816 
9817   // Base case: There is only one way to partition Clusters[N-1].
9818   MinPartitions[N - 1] = 1;
9819   LastElement[N - 1] = N - 1;
9820 
9821   // Note: loop indexes are signed to avoid underflow.
9822   for (int64_t i = N - 2; i >= 0; --i) {
9823     // Find optimal partitioning of Clusters[i..N-1].
9824     // Baseline: Put Clusters[i] into a partition on its own.
9825     MinPartitions[i] = MinPartitions[i + 1] + 1;
9826     LastElement[i] = i;
9827 
9828     // Search for a solution that results in fewer partitions.
9829     // Note: the search is limited by BitWidth, reducing time complexity.
9830     for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
9831       // Try building a partition from Clusters[i..j].
9832 
9833       // Check the range.
9834       if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(),
9835                                Clusters[j].High->getValue(), DL))
9836         continue;
9837 
9838       // Check nbr of destinations and cluster types.
9839       // FIXME: This works, but doesn't seem very efficient.
9840       bool RangesOnly = true;
9841       BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9842       for (int64_t k = i; k <= j; k++) {
9843         if (Clusters[k].Kind != CC_Range) {
9844           RangesOnly = false;
9845           break;
9846         }
9847         Dests.set(Clusters[k].MBB->getNumber());
9848       }
9849       if (!RangesOnly || Dests.count() > 3)
9850         break;
9851 
9852       // Check if it's a better partition.
9853       unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9854       if (NumPartitions < MinPartitions[i]) {
9855         // Found a better partition.
9856         MinPartitions[i] = NumPartitions;
9857         LastElement[i] = j;
9858       }
9859     }
9860   }
9861 
9862   // Iterate over the partitions, replacing with bit-test clusters in-place.
9863   unsigned DstIndex = 0;
9864   for (unsigned First = 0, Last; First < N; First = Last + 1) {
9865     Last = LastElement[First];
9866     assert(First <= Last);
9867     assert(DstIndex <= First);
9868 
9869     CaseCluster BitTestCluster;
9870     if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
9871       Clusters[DstIndex++] = BitTestCluster;
9872     } else {
9873       size_t NumClusters = Last - First + 1;
9874       std::memmove(&Clusters[DstIndex], &Clusters[First],
9875                    sizeof(Clusters[0]) * NumClusters);
9876       DstIndex += NumClusters;
9877     }
9878   }
9879   Clusters.resize(DstIndex);
9880 }
9881 
9882 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
9883                                         MachineBasicBlock *SwitchMBB,
9884                                         MachineBasicBlock *DefaultMBB) {
9885   MachineFunction *CurMF = FuncInfo.MF;
9886   MachineBasicBlock *NextMBB = nullptr;
9887   MachineFunction::iterator BBI(W.MBB);
9888   if (++BBI != FuncInfo.MF->end())
9889     NextMBB = &*BBI;
9890 
9891   unsigned Size = W.LastCluster - W.FirstCluster + 1;
9892 
9893   BranchProbabilityInfo *BPI = FuncInfo.BPI;
9894 
9895   if (Size == 2 && W.MBB == SwitchMBB) {
9896     // If any two of the cases has the same destination, and if one value
9897     // is the same as the other, but has one bit unset that the other has set,
9898     // use bit manipulation to do two compares at once.  For example:
9899     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
9900     // TODO: This could be extended to merge any 2 cases in switches with 3
9901     // cases.
9902     // TODO: Handle cases where W.CaseBB != SwitchBB.
9903     CaseCluster &Small = *W.FirstCluster;
9904     CaseCluster &Big = *W.LastCluster;
9905 
9906     if (Small.Low == Small.High && Big.Low == Big.High &&
9907         Small.MBB == Big.MBB) {
9908       const APInt &SmallValue = Small.Low->getValue();
9909       const APInt &BigValue = Big.Low->getValue();
9910 
9911       // Check that there is only one bit different.
9912       APInt CommonBit = BigValue ^ SmallValue;
9913       if (CommonBit.isPowerOf2()) {
9914         SDValue CondLHS = getValue(Cond);
9915         EVT VT = CondLHS.getValueType();
9916         SDLoc DL = getCurSDLoc();
9917 
9918         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
9919                                  DAG.getConstant(CommonBit, DL, VT));
9920         SDValue Cond = DAG.getSetCC(
9921             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
9922             ISD::SETEQ);
9923 
9924         // Update successor info.
9925         // Both Small and Big will jump to Small.BB, so we sum up the
9926         // probabilities.
9927         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
9928         if (BPI)
9929           addSuccessorWithProb(
9930               SwitchMBB, DefaultMBB,
9931               // The default destination is the first successor in IR.
9932               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
9933         else
9934           addSuccessorWithProb(SwitchMBB, DefaultMBB);
9935 
9936         // Insert the true branch.
9937         SDValue BrCond =
9938             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
9939                         DAG.getBasicBlock(Small.MBB));
9940         // Insert the false branch.
9941         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
9942                              DAG.getBasicBlock(DefaultMBB));
9943 
9944         DAG.setRoot(BrCond);
9945         return;
9946       }
9947     }
9948   }
9949 
9950   if (TM.getOptLevel() != CodeGenOpt::None) {
9951     // Here, we order cases by probability so the most likely case will be
9952     // checked first. However, two clusters can have the same probability in
9953     // which case their relative ordering is non-deterministic. So we use Low
9954     // as a tie-breaker as clusters are guaranteed to never overlap.
9955     llvm::sort(W.FirstCluster, W.LastCluster + 1,
9956                [](const CaseCluster &a, const CaseCluster &b) {
9957       return a.Prob != b.Prob ?
9958              a.Prob > b.Prob :
9959              a.Low->getValue().slt(b.Low->getValue());
9960     });
9961 
9962     // Rearrange the case blocks so that the last one falls through if possible
9963     // without changing the order of probabilities.
9964     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
9965       --I;
9966       if (I->Prob > W.LastCluster->Prob)
9967         break;
9968       if (I->Kind == CC_Range && I->MBB == NextMBB) {
9969         std::swap(*I, *W.LastCluster);
9970         break;
9971       }
9972     }
9973   }
9974 
9975   // Compute total probability.
9976   BranchProbability DefaultProb = W.DefaultProb;
9977   BranchProbability UnhandledProbs = DefaultProb;
9978   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
9979     UnhandledProbs += I->Prob;
9980 
9981   MachineBasicBlock *CurMBB = W.MBB;
9982   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
9983     MachineBasicBlock *Fallthrough;
9984     if (I == W.LastCluster) {
9985       // For the last cluster, fall through to the default destination.
9986       Fallthrough = DefaultMBB;
9987     } else {
9988       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
9989       CurMF->insert(BBI, Fallthrough);
9990       // Put Cond in a virtual register to make it available from the new blocks.
9991       ExportFromCurrentBlock(Cond);
9992     }
9993     UnhandledProbs -= I->Prob;
9994 
9995     switch (I->Kind) {
9996       case CC_JumpTable: {
9997         // FIXME: Optimize away range check based on pivot comparisons.
9998         JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
9999         JumpTable *JT = &JTCases[I->JTCasesIndex].second;
10000 
10001         // The jump block hasn't been inserted yet; insert it here.
10002         MachineBasicBlock *JumpMBB = JT->MBB;
10003         CurMF->insert(BBI, JumpMBB);
10004 
10005         auto JumpProb = I->Prob;
10006         auto FallthroughProb = UnhandledProbs;
10007 
10008         // If the default statement is a target of the jump table, we evenly
10009         // distribute the default probability to successors of CurMBB. Also
10010         // update the probability on the edge from JumpMBB to Fallthrough.
10011         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
10012                                               SE = JumpMBB->succ_end();
10013              SI != SE; ++SI) {
10014           if (*SI == DefaultMBB) {
10015             JumpProb += DefaultProb / 2;
10016             FallthroughProb -= DefaultProb / 2;
10017             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
10018             JumpMBB->normalizeSuccProbs();
10019             break;
10020           }
10021         }
10022 
10023         addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
10024         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
10025         CurMBB->normalizeSuccProbs();
10026 
10027         // The jump table header will be inserted in our current block, do the
10028         // range check, and fall through to our fallthrough block.
10029         JTH->HeaderBB = CurMBB;
10030         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
10031 
10032         // If we're in the right place, emit the jump table header right now.
10033         if (CurMBB == SwitchMBB) {
10034           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
10035           JTH->Emitted = true;
10036         }
10037         break;
10038       }
10039       case CC_BitTests: {
10040         // FIXME: Optimize away range check based on pivot comparisons.
10041         BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
10042 
10043         // The bit test blocks haven't been inserted yet; insert them here.
10044         for (BitTestCase &BTC : BTB->Cases)
10045           CurMF->insert(BBI, BTC.ThisBB);
10046 
10047         // Fill in fields of the BitTestBlock.
10048         BTB->Parent = CurMBB;
10049         BTB->Default = Fallthrough;
10050 
10051         BTB->DefaultProb = UnhandledProbs;
10052         // If the cases in bit test don't form a contiguous range, we evenly
10053         // distribute the probability on the edge to Fallthrough to two
10054         // successors of CurMBB.
10055         if (!BTB->ContiguousRange) {
10056           BTB->Prob += DefaultProb / 2;
10057           BTB->DefaultProb -= DefaultProb / 2;
10058         }
10059 
10060         // If we're in the right place, emit the bit test header right now.
10061         if (CurMBB == SwitchMBB) {
10062           visitBitTestHeader(*BTB, SwitchMBB);
10063           BTB->Emitted = true;
10064         }
10065         break;
10066       }
10067       case CC_Range: {
10068         const Value *RHS, *LHS, *MHS;
10069         ISD::CondCode CC;
10070         if (I->Low == I->High) {
10071           // Check Cond == I->Low.
10072           CC = ISD::SETEQ;
10073           LHS = Cond;
10074           RHS=I->Low;
10075           MHS = nullptr;
10076         } else {
10077           // Check I->Low <= Cond <= I->High.
10078           CC = ISD::SETLE;
10079           LHS = I->Low;
10080           MHS = Cond;
10081           RHS = I->High;
10082         }
10083 
10084         // The false probability is the sum of all unhandled cases.
10085         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
10086                      getCurSDLoc(), I->Prob, UnhandledProbs);
10087 
10088         if (CurMBB == SwitchMBB)
10089           visitSwitchCase(CB, SwitchMBB);
10090         else
10091           SwitchCases.push_back(CB);
10092 
10093         break;
10094       }
10095     }
10096     CurMBB = Fallthrough;
10097   }
10098 }
10099 
10100 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
10101                                               CaseClusterIt First,
10102                                               CaseClusterIt Last) {
10103   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
10104     if (X.Prob != CC.Prob)
10105       return X.Prob > CC.Prob;
10106 
10107     // Ties are broken by comparing the case value.
10108     return X.Low->getValue().slt(CC.Low->getValue());
10109   });
10110 }
10111 
10112 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
10113                                         const SwitchWorkListItem &W,
10114                                         Value *Cond,
10115                                         MachineBasicBlock *SwitchMBB) {
10116   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
10117          "Clusters not sorted?");
10118 
10119   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
10120 
10121   // Balance the tree based on branch probabilities to create a near-optimal (in
10122   // terms of search time given key frequency) binary search tree. See e.g. Kurt
10123   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
10124   CaseClusterIt LastLeft = W.FirstCluster;
10125   CaseClusterIt FirstRight = W.LastCluster;
10126   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
10127   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
10128 
10129   // Move LastLeft and FirstRight towards each other from opposite directions to
10130   // find a partitioning of the clusters which balances the probability on both
10131   // sides. If LeftProb and RightProb are equal, alternate which side is
10132   // taken to ensure 0-probability nodes are distributed evenly.
10133   unsigned I = 0;
10134   while (LastLeft + 1 < FirstRight) {
10135     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
10136       LeftProb += (++LastLeft)->Prob;
10137     else
10138       RightProb += (--FirstRight)->Prob;
10139     I++;
10140   }
10141 
10142   while (true) {
10143     // Our binary search tree differs from a typical BST in that ours can have up
10144     // to three values in each leaf. The pivot selection above doesn't take that
10145     // into account, which means the tree might require more nodes and be less
10146     // efficient. We compensate for this here.
10147 
10148     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
10149     unsigned NumRight = W.LastCluster - FirstRight + 1;
10150 
10151     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
10152       // If one side has less than 3 clusters, and the other has more than 3,
10153       // consider taking a cluster from the other side.
10154 
10155       if (NumLeft < NumRight) {
10156         // Consider moving the first cluster on the right to the left side.
10157         CaseCluster &CC = *FirstRight;
10158         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10159         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10160         if (LeftSideRank <= RightSideRank) {
10161           // Moving the cluster to the left does not demote it.
10162           ++LastLeft;
10163           ++FirstRight;
10164           continue;
10165         }
10166       } else {
10167         assert(NumRight < NumLeft);
10168         // Consider moving the last element on the left to the right side.
10169         CaseCluster &CC = *LastLeft;
10170         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10171         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10172         if (RightSideRank <= LeftSideRank) {
10173           // Moving the cluster to the right does not demot it.
10174           --LastLeft;
10175           --FirstRight;
10176           continue;
10177         }
10178       }
10179     }
10180     break;
10181   }
10182 
10183   assert(LastLeft + 1 == FirstRight);
10184   assert(LastLeft >= W.FirstCluster);
10185   assert(FirstRight <= W.LastCluster);
10186 
10187   // Use the first element on the right as pivot since we will make less-than
10188   // comparisons against it.
10189   CaseClusterIt PivotCluster = FirstRight;
10190   assert(PivotCluster > W.FirstCluster);
10191   assert(PivotCluster <= W.LastCluster);
10192 
10193   CaseClusterIt FirstLeft = W.FirstCluster;
10194   CaseClusterIt LastRight = W.LastCluster;
10195 
10196   const ConstantInt *Pivot = PivotCluster->Low;
10197 
10198   // New blocks will be inserted immediately after the current one.
10199   MachineFunction::iterator BBI(W.MBB);
10200   ++BBI;
10201 
10202   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
10203   // we can branch to its destination directly if it's squeezed exactly in
10204   // between the known lower bound and Pivot - 1.
10205   MachineBasicBlock *LeftMBB;
10206   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
10207       FirstLeft->Low == W.GE &&
10208       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
10209     LeftMBB = FirstLeft->MBB;
10210   } else {
10211     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10212     FuncInfo.MF->insert(BBI, LeftMBB);
10213     WorkList.push_back(
10214         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
10215     // Put Cond in a virtual register to make it available from the new blocks.
10216     ExportFromCurrentBlock(Cond);
10217   }
10218 
10219   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
10220   // single cluster, RHS.Low == Pivot, and we can branch to its destination
10221   // directly if RHS.High equals the current upper bound.
10222   MachineBasicBlock *RightMBB;
10223   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
10224       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
10225     RightMBB = FirstRight->MBB;
10226   } else {
10227     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10228     FuncInfo.MF->insert(BBI, RightMBB);
10229     WorkList.push_back(
10230         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
10231     // Put Cond in a virtual register to make it available from the new blocks.
10232     ExportFromCurrentBlock(Cond);
10233   }
10234 
10235   // Create the CaseBlock record that will be used to lower the branch.
10236   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
10237                getCurSDLoc(), LeftProb, RightProb);
10238 
10239   if (W.MBB == SwitchMBB)
10240     visitSwitchCase(CB, SwitchMBB);
10241   else
10242     SwitchCases.push_back(CB);
10243 }
10244 
10245 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
10246 // from the swith statement.
10247 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
10248                                             BranchProbability PeeledCaseProb) {
10249   if (PeeledCaseProb == BranchProbability::getOne())
10250     return BranchProbability::getZero();
10251   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
10252 
10253   uint32_t Numerator = CaseProb.getNumerator();
10254   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
10255   return BranchProbability(Numerator, std::max(Numerator, Denominator));
10256 }
10257 
10258 // Try to peel the top probability case if it exceeds the threshold.
10259 // Return current MachineBasicBlock for the switch statement if the peeling
10260 // does not occur.
10261 // If the peeling is performed, return the newly created MachineBasicBlock
10262 // for the peeled switch statement. Also update Clusters to remove the peeled
10263 // case. PeeledCaseProb is the BranchProbability for the peeled case.
10264 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
10265     const SwitchInst &SI, CaseClusterVector &Clusters,
10266     BranchProbability &PeeledCaseProb) {
10267   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10268   // Don't perform if there is only one cluster or optimizing for size.
10269   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
10270       TM.getOptLevel() == CodeGenOpt::None ||
10271       SwitchMBB->getParent()->getFunction().optForMinSize())
10272     return SwitchMBB;
10273 
10274   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
10275   unsigned PeeledCaseIndex = 0;
10276   bool SwitchPeeled = false;
10277   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
10278     CaseCluster &CC = Clusters[Index];
10279     if (CC.Prob < TopCaseProb)
10280       continue;
10281     TopCaseProb = CC.Prob;
10282     PeeledCaseIndex = Index;
10283     SwitchPeeled = true;
10284   }
10285   if (!SwitchPeeled)
10286     return SwitchMBB;
10287 
10288   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
10289                     << TopCaseProb << "\n");
10290 
10291   // Record the MBB for the peeled switch statement.
10292   MachineFunction::iterator BBI(SwitchMBB);
10293   ++BBI;
10294   MachineBasicBlock *PeeledSwitchMBB =
10295       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
10296   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
10297 
10298   ExportFromCurrentBlock(SI.getCondition());
10299   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
10300   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
10301                           nullptr,   nullptr,      TopCaseProb.getCompl()};
10302   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
10303 
10304   Clusters.erase(PeeledCaseIt);
10305   for (CaseCluster &CC : Clusters) {
10306     LLVM_DEBUG(
10307         dbgs() << "Scale the probablity for one cluster, before scaling: "
10308                << CC.Prob << "\n");
10309     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
10310     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
10311   }
10312   PeeledCaseProb = TopCaseProb;
10313   return PeeledSwitchMBB;
10314 }
10315 
10316 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
10317   // Extract cases from the switch.
10318   BranchProbabilityInfo *BPI = FuncInfo.BPI;
10319   CaseClusterVector Clusters;
10320   Clusters.reserve(SI.getNumCases());
10321   for (auto I : SI.cases()) {
10322     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
10323     const ConstantInt *CaseVal = I.getCaseValue();
10324     BranchProbability Prob =
10325         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
10326             : BranchProbability(1, SI.getNumCases() + 1);
10327     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
10328   }
10329 
10330   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
10331 
10332   // Cluster adjacent cases with the same destination. We do this at all
10333   // optimization levels because it's cheap to do and will make codegen faster
10334   // if there are many clusters.
10335   sortAndRangeify(Clusters);
10336 
10337   if (TM.getOptLevel() != CodeGenOpt::None) {
10338     // Replace an unreachable default with the most popular destination.
10339     // FIXME: Exploit unreachable default more aggressively.
10340     bool UnreachableDefault =
10341         isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
10342     if (UnreachableDefault && !Clusters.empty()) {
10343       DenseMap<const BasicBlock *, unsigned> Popularity;
10344       unsigned MaxPop = 0;
10345       const BasicBlock *MaxBB = nullptr;
10346       for (auto I : SI.cases()) {
10347         const BasicBlock *BB = I.getCaseSuccessor();
10348         if (++Popularity[BB] > MaxPop) {
10349           MaxPop = Popularity[BB];
10350           MaxBB = BB;
10351         }
10352       }
10353       // Set new default.
10354       assert(MaxPop > 0 && MaxBB);
10355       DefaultMBB = FuncInfo.MBBMap[MaxBB];
10356 
10357       // Remove cases that were pointing to the destination that is now the
10358       // default.
10359       CaseClusterVector New;
10360       New.reserve(Clusters.size());
10361       for (CaseCluster &CC : Clusters) {
10362         if (CC.MBB != DefaultMBB)
10363           New.push_back(CC);
10364       }
10365       Clusters = std::move(New);
10366     }
10367   }
10368 
10369   // The branch probablity of the peeled case.
10370   BranchProbability PeeledCaseProb = BranchProbability::getZero();
10371   MachineBasicBlock *PeeledSwitchMBB =
10372       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
10373 
10374   // If there is only the default destination, jump there directly.
10375   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10376   if (Clusters.empty()) {
10377     assert(PeeledSwitchMBB == SwitchMBB);
10378     SwitchMBB->addSuccessor(DefaultMBB);
10379     if (DefaultMBB != NextBlock(SwitchMBB)) {
10380       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
10381                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
10382     }
10383     return;
10384   }
10385 
10386   findJumpTables(Clusters, &SI, DefaultMBB);
10387   findBitTestClusters(Clusters, &SI);
10388 
10389   LLVM_DEBUG({
10390     dbgs() << "Case clusters: ";
10391     for (const CaseCluster &C : Clusters) {
10392       if (C.Kind == CC_JumpTable)
10393         dbgs() << "JT:";
10394       if (C.Kind == CC_BitTests)
10395         dbgs() << "BT:";
10396 
10397       C.Low->getValue().print(dbgs(), true);
10398       if (C.Low != C.High) {
10399         dbgs() << '-';
10400         C.High->getValue().print(dbgs(), true);
10401       }
10402       dbgs() << ' ';
10403     }
10404     dbgs() << '\n';
10405   });
10406 
10407   assert(!Clusters.empty());
10408   SwitchWorkList WorkList;
10409   CaseClusterIt First = Clusters.begin();
10410   CaseClusterIt Last = Clusters.end() - 1;
10411   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
10412   // Scale the branchprobability for DefaultMBB if the peel occurs and
10413   // DefaultMBB is not replaced.
10414   if (PeeledCaseProb != BranchProbability::getZero() &&
10415       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
10416     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
10417   WorkList.push_back(
10418       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
10419 
10420   while (!WorkList.empty()) {
10421     SwitchWorkListItem W = WorkList.back();
10422     WorkList.pop_back();
10423     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
10424 
10425     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
10426         !DefaultMBB->getParent()->getFunction().optForMinSize()) {
10427       // For optimized builds, lower large range as a balanced binary tree.
10428       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
10429       continue;
10430     }
10431 
10432     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
10433   }
10434 }
10435