xref: /llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision a06b163d5cee8114cd34c4565bbf1d5f28554684)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/APInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/None.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/ADT/Triple.h"
29 #include "llvm/ADT/Twine.h"
30 #include "llvm/Analysis/AliasAnalysis.h"
31 #include "llvm/Analysis/BranchProbabilityInfo.h"
32 #include "llvm/Analysis/ConstantFolding.h"
33 #include "llvm/Analysis/EHPersonalities.h"
34 #include "llvm/Analysis/Loads.h"
35 #include "llvm/Analysis/MemoryLocation.h"
36 #include "llvm/Analysis/TargetLibraryInfo.h"
37 #include "llvm/Analysis/ValueTracking.h"
38 #include "llvm/Analysis/VectorUtils.h"
39 #include "llvm/CodeGen/Analysis.h"
40 #include "llvm/CodeGen/FunctionLoweringInfo.h"
41 #include "llvm/CodeGen/GCMetadata.h"
42 #include "llvm/CodeGen/ISDOpcodes.h"
43 #include "llvm/CodeGen/MachineBasicBlock.h"
44 #include "llvm/CodeGen/MachineFrameInfo.h"
45 #include "llvm/CodeGen/MachineFunction.h"
46 #include "llvm/CodeGen/MachineInstr.h"
47 #include "llvm/CodeGen/MachineInstrBuilder.h"
48 #include "llvm/CodeGen/MachineJumpTableInfo.h"
49 #include "llvm/CodeGen/MachineMemOperand.h"
50 #include "llvm/CodeGen/MachineModuleInfo.h"
51 #include "llvm/CodeGen/MachineOperand.h"
52 #include "llvm/CodeGen/MachineRegisterInfo.h"
53 #include "llvm/CodeGen/RuntimeLibcalls.h"
54 #include "llvm/CodeGen/SelectionDAG.h"
55 #include "llvm/CodeGen/SelectionDAGNodes.h"
56 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
57 #include "llvm/CodeGen/StackMaps.h"
58 #include "llvm/CodeGen/TargetFrameLowering.h"
59 #include "llvm/CodeGen/TargetInstrInfo.h"
60 #include "llvm/CodeGen/TargetLowering.h"
61 #include "llvm/CodeGen/TargetOpcodes.h"
62 #include "llvm/CodeGen/TargetRegisterInfo.h"
63 #include "llvm/CodeGen/TargetSubtargetInfo.h"
64 #include "llvm/CodeGen/ValueTypes.h"
65 #include "llvm/CodeGen/WinEHFuncInfo.h"
66 #include "llvm/IR/Argument.h"
67 #include "llvm/IR/Attributes.h"
68 #include "llvm/IR/BasicBlock.h"
69 #include "llvm/IR/CFG.h"
70 #include "llvm/IR/CallSite.h"
71 #include "llvm/IR/CallingConv.h"
72 #include "llvm/IR/Constant.h"
73 #include "llvm/IR/ConstantRange.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfoMetadata.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
80 #include "llvm/IR/GetElementPtrTypeIterator.h"
81 #include "llvm/IR/InlineAsm.h"
82 #include "llvm/IR/InstrTypes.h"
83 #include "llvm/IR/Instruction.h"
84 #include "llvm/IR/Instructions.h"
85 #include "llvm/IR/IntrinsicInst.h"
86 #include "llvm/IR/Intrinsics.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Metadata.h"
89 #include "llvm/IR/Module.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Statepoint.h"
93 #include "llvm/IR/Type.h"
94 #include "llvm/IR/User.h"
95 #include "llvm/IR/Value.h"
96 #include "llvm/MC/MCContext.h"
97 #include "llvm/MC/MCSymbol.h"
98 #include "llvm/Support/AtomicOrdering.h"
99 #include "llvm/Support/BranchProbability.h"
100 #include "llvm/Support/Casting.h"
101 #include "llvm/Support/CodeGen.h"
102 #include "llvm/Support/CommandLine.h"
103 #include "llvm/Support/Compiler.h"
104 #include "llvm/Support/Debug.h"
105 #include "llvm/Support/ErrorHandling.h"
106 #include "llvm/Support/MachineValueType.h"
107 #include "llvm/Support/MathExtras.h"
108 #include "llvm/Support/raw_ostream.h"
109 #include "llvm/Target/TargetIntrinsicInfo.h"
110 #include "llvm/Target/TargetMachine.h"
111 #include "llvm/Target/TargetOptions.h"
112 #include <algorithm>
113 #include <cassert>
114 #include <cstddef>
115 #include <cstdint>
116 #include <cstring>
117 #include <iterator>
118 #include <limits>
119 #include <numeric>
120 #include <tuple>
121 #include <utility>
122 #include <vector>
123 
124 using namespace llvm;
125 using namespace PatternMatch;
126 
127 #define DEBUG_TYPE "isel"
128 
129 /// LimitFloatPrecision - Generate low-precision inline sequences for
130 /// some float libcalls (6, 8 or 12 bits).
131 static unsigned LimitFloatPrecision;
132 
133 static cl::opt<unsigned, true>
134     LimitFPPrecision("limit-float-precision",
135                      cl::desc("Generate low-precision inline sequences "
136                               "for some float libcalls"),
137                      cl::location(LimitFloatPrecision), cl::Hidden,
138                      cl::init(0));
139 
140 static cl::opt<unsigned> SwitchPeelThreshold(
141     "switch-peel-threshold", cl::Hidden, cl::init(66),
142     cl::desc("Set the case probability threshold for peeling the case from a "
143              "switch statement. A value greater than 100 will void this "
144              "optimization"));
145 
146 // Limit the width of DAG chains. This is important in general to prevent
147 // DAG-based analysis from blowing up. For example, alias analysis and
148 // load clustering may not complete in reasonable time. It is difficult to
149 // recognize and avoid this situation within each individual analysis, and
150 // future analyses are likely to have the same behavior. Limiting DAG width is
151 // the safe approach and will be especially important with global DAGs.
152 //
153 // MaxParallelChains default is arbitrarily high to avoid affecting
154 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
155 // sequence over this should have been converted to llvm.memcpy by the
156 // frontend. It is easy to induce this behavior with .ll code such as:
157 // %buffer = alloca [4096 x i8]
158 // %data = load [4096 x i8]* %argPtr
159 // store [4096 x i8] %data, [4096 x i8]* %buffer
160 static const unsigned MaxParallelChains = 64;
161 
162 // Return the calling convention if the Value passed requires ABI mangling as it
163 // is a parameter to a function or a return value from a function which is not
164 // an intrinsic.
165 static Optional<CallingConv::ID> getABIRegCopyCC(const Value *V) {
166   if (auto *R = dyn_cast<ReturnInst>(V))
167     return R->getParent()->getParent()->getCallingConv();
168 
169   if (auto *CI = dyn_cast<CallInst>(V)) {
170     const bool IsInlineAsm = CI->isInlineAsm();
171     const bool IsIndirectFunctionCall =
172         !IsInlineAsm && !CI->getCalledFunction();
173 
174     // It is possible that the call instruction is an inline asm statement or an
175     // indirect function call in which case the return value of
176     // getCalledFunction() would be nullptr.
177     const bool IsInstrinsicCall =
178         !IsInlineAsm && !IsIndirectFunctionCall &&
179         CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
180 
181     if (!IsInlineAsm && !IsInstrinsicCall)
182       return CI->getCallingConv();
183   }
184 
185   return None;
186 }
187 
188 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
189                                       const SDValue *Parts, unsigned NumParts,
190                                       MVT PartVT, EVT ValueVT, const Value *V,
191                                       Optional<CallingConv::ID> CC);
192 
193 /// getCopyFromParts - Create a value that contains the specified legal parts
194 /// combined into the value they represent.  If the parts combine to a type
195 /// larger than ValueVT then AssertOp can be used to specify whether the extra
196 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
197 /// (ISD::AssertSext).
198 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
199                                 const SDValue *Parts, unsigned NumParts,
200                                 MVT PartVT, EVT ValueVT, const Value *V,
201                                 Optional<CallingConv::ID> CC = None,
202                                 Optional<ISD::NodeType> AssertOp = None) {
203   if (ValueVT.isVector())
204     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
205                                   CC);
206 
207   assert(NumParts > 0 && "No parts to assemble!");
208   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
209   SDValue Val = Parts[0];
210 
211   if (NumParts > 1) {
212     // Assemble the value from multiple parts.
213     if (ValueVT.isInteger()) {
214       unsigned PartBits = PartVT.getSizeInBits();
215       unsigned ValueBits = ValueVT.getSizeInBits();
216 
217       // Assemble the power of 2 part.
218       unsigned RoundParts = NumParts & (NumParts - 1) ?
219         1 << Log2_32(NumParts) : NumParts;
220       unsigned RoundBits = PartBits * RoundParts;
221       EVT RoundVT = RoundBits == ValueBits ?
222         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
223       SDValue Lo, Hi;
224 
225       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
226 
227       if (RoundParts > 2) {
228         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
229                               PartVT, HalfVT, V);
230         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
231                               RoundParts / 2, PartVT, HalfVT, V);
232       } else {
233         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
234         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
235       }
236 
237       if (DAG.getDataLayout().isBigEndian())
238         std::swap(Lo, Hi);
239 
240       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
241 
242       if (RoundParts < NumParts) {
243         // Assemble the trailing non-power-of-2 part.
244         unsigned OddParts = NumParts - RoundParts;
245         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
246         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
247                               OddVT, V, CC);
248 
249         // Combine the round and odd parts.
250         Lo = Val;
251         if (DAG.getDataLayout().isBigEndian())
252           std::swap(Lo, Hi);
253         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
254         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
255         Hi =
256             DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
257                         DAG.getConstant(Lo.getValueSizeInBits(), DL,
258                                         TLI.getPointerTy(DAG.getDataLayout())));
259         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
260         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
261       }
262     } else if (PartVT.isFloatingPoint()) {
263       // FP split into multiple FP parts (for ppcf128)
264       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
265              "Unexpected split");
266       SDValue Lo, Hi;
267       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
268       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
269       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
270         std::swap(Lo, Hi);
271       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
272     } else {
273       // FP split into integer parts (soft fp)
274       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
275              !PartVT.isVector() && "Unexpected split");
276       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
277       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
278     }
279   }
280 
281   // There is now one part, held in Val.  Correct it to match ValueVT.
282   // PartEVT is the type of the register class that holds the value.
283   // ValueVT is the type of the inline asm operation.
284   EVT PartEVT = Val.getValueType();
285 
286   if (PartEVT == ValueVT)
287     return Val;
288 
289   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
290       ValueVT.bitsLT(PartEVT)) {
291     // For an FP value in an integer part, we need to truncate to the right
292     // width first.
293     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
294     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
295   }
296 
297   // Handle types that have the same size.
298   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
299     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
300 
301   // Handle types with different sizes.
302   if (PartEVT.isInteger() && ValueVT.isInteger()) {
303     if (ValueVT.bitsLT(PartEVT)) {
304       // For a truncate, see if we have any information to
305       // indicate whether the truncated bits will always be
306       // zero or sign-extension.
307       if (AssertOp.hasValue())
308         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
309                           DAG.getValueType(ValueVT));
310       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
311     }
312     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
313   }
314 
315   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
316     // FP_ROUND's are always exact here.
317     if (ValueVT.bitsLT(Val.getValueType()))
318       return DAG.getNode(
319           ISD::FP_ROUND, DL, ValueVT, Val,
320           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
321 
322     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
323   }
324 
325   llvm_unreachable("Unknown mismatch!");
326 }
327 
328 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
329                                               const Twine &ErrMsg) {
330   const Instruction *I = dyn_cast_or_null<Instruction>(V);
331   if (!V)
332     return Ctx.emitError(ErrMsg);
333 
334   const char *AsmError = ", possible invalid constraint for vector type";
335   if (const CallInst *CI = dyn_cast<CallInst>(I))
336     if (isa<InlineAsm>(CI->getCalledValue()))
337       return Ctx.emitError(I, ErrMsg + AsmError);
338 
339   return Ctx.emitError(I, ErrMsg);
340 }
341 
342 /// getCopyFromPartsVector - Create a value that contains the specified legal
343 /// parts combined into the value they represent.  If the parts combine to a
344 /// type larger than ValueVT then AssertOp can be used to specify whether the
345 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
346 /// ValueVT (ISD::AssertSext).
347 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
348                                       const SDValue *Parts, unsigned NumParts,
349                                       MVT PartVT, EVT ValueVT, const Value *V,
350                                       Optional<CallingConv::ID> CallConv) {
351   assert(ValueVT.isVector() && "Not a vector value");
352   assert(NumParts > 0 && "No parts to assemble!");
353   const bool IsABIRegCopy = CallConv.hasValue();
354 
355   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
356   SDValue Val = Parts[0];
357 
358   // Handle a multi-element vector.
359   if (NumParts > 1) {
360     EVT IntermediateVT;
361     MVT RegisterVT;
362     unsigned NumIntermediates;
363     unsigned NumRegs;
364 
365     if (IsABIRegCopy) {
366       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
367           *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
368           NumIntermediates, RegisterVT);
369     } else {
370       NumRegs =
371           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
372                                      NumIntermediates, RegisterVT);
373     }
374 
375     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
376     NumParts = NumRegs; // Silence a compiler warning.
377     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
378     assert(RegisterVT.getSizeInBits() ==
379            Parts[0].getSimpleValueType().getSizeInBits() &&
380            "Part type sizes don't match!");
381 
382     // Assemble the parts into intermediate operands.
383     SmallVector<SDValue, 8> Ops(NumIntermediates);
384     if (NumIntermediates == NumParts) {
385       // If the register was not expanded, truncate or copy the value,
386       // as appropriate.
387       for (unsigned i = 0; i != NumParts; ++i)
388         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
389                                   PartVT, IntermediateVT, V);
390     } else if (NumParts > 0) {
391       // If the intermediate type was expanded, build the intermediate
392       // operands from the parts.
393       assert(NumParts % NumIntermediates == 0 &&
394              "Must expand into a divisible number of parts!");
395       unsigned Factor = NumParts / NumIntermediates;
396       for (unsigned i = 0; i != NumIntermediates; ++i)
397         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
398                                   PartVT, IntermediateVT, V);
399     }
400 
401     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
402     // intermediate operands.
403     EVT BuiltVectorTy =
404         EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
405                          (IntermediateVT.isVector()
406                               ? IntermediateVT.getVectorNumElements() * NumParts
407                               : NumIntermediates));
408     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
409                                                 : ISD::BUILD_VECTOR,
410                       DL, BuiltVectorTy, Ops);
411   }
412 
413   // There is now one part, held in Val.  Correct it to match ValueVT.
414   EVT PartEVT = Val.getValueType();
415 
416   if (PartEVT == ValueVT)
417     return Val;
418 
419   if (PartEVT.isVector()) {
420     // If the element type of the source/dest vectors are the same, but the
421     // parts vector has more elements than the value vector, then we have a
422     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
423     // elements we want.
424     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
425       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
426              "Cannot narrow, it would be a lossy transformation");
427       return DAG.getNode(
428           ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
429           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
430     }
431 
432     // Vector/Vector bitcast.
433     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
434       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435 
436     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
437       "Cannot handle this kind of promotion");
438     // Promoted vector extract
439     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
440 
441   }
442 
443   // Trivial bitcast if the types are the same size and the destination
444   // vector type is legal.
445   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
446       TLI.isTypeLegal(ValueVT))
447     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
448 
449   if (ValueVT.getVectorNumElements() != 1) {
450      // Certain ABIs require that vectors are passed as integers. For vectors
451      // are the same size, this is an obvious bitcast.
452      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
453        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
454      } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
455        // Bitcast Val back the original type and extract the corresponding
456        // vector we want.
457        unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
458        EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
459                                            ValueVT.getVectorElementType(), Elts);
460        Val = DAG.getBitcast(WiderVecType, Val);
461        return DAG.getNode(
462            ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
463            DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
464      }
465 
466      diagnosePossiblyInvalidConstraint(
467          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
468      return DAG.getUNDEF(ValueVT);
469   }
470 
471   // Handle cases such as i8 -> <1 x i1>
472   EVT ValueSVT = ValueVT.getVectorElementType();
473   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
474     Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
475                                     : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
476 
477   return DAG.getBuildVector(ValueVT, DL, Val);
478 }
479 
480 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
481                                  SDValue Val, SDValue *Parts, unsigned NumParts,
482                                  MVT PartVT, const Value *V,
483                                  Optional<CallingConv::ID> CallConv);
484 
485 /// getCopyToParts - Create a series of nodes that contain the specified value
486 /// split into legal parts.  If the parts contain more bits than Val, then, for
487 /// integers, ExtendKind can be used to specify how to generate the extra bits.
488 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
489                            SDValue *Parts, unsigned NumParts, MVT PartVT,
490                            const Value *V,
491                            Optional<CallingConv::ID> CallConv = None,
492                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
493   EVT ValueVT = Val.getValueType();
494 
495   // Handle the vector case separately.
496   if (ValueVT.isVector())
497     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
498                                 CallConv);
499 
500   unsigned PartBits = PartVT.getSizeInBits();
501   unsigned OrigNumParts = NumParts;
502   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
503          "Copying to an illegal type!");
504 
505   if (NumParts == 0)
506     return;
507 
508   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
509   EVT PartEVT = PartVT;
510   if (PartEVT == ValueVT) {
511     assert(NumParts == 1 && "No-op copy with multiple parts!");
512     Parts[0] = Val;
513     return;
514   }
515 
516   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
517     // If the parts cover more bits than the value has, promote the value.
518     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
519       assert(NumParts == 1 && "Do not know what to promote to!");
520       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
521     } else {
522       if (ValueVT.isFloatingPoint()) {
523         // FP values need to be bitcast, then extended if they are being put
524         // into a larger container.
525         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
526         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
527       }
528       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529              ValueVT.isInteger() &&
530              "Unknown mismatch!");
531       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
533       if (PartVT == MVT::x86mmx)
534         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535     }
536   } else if (PartBits == ValueVT.getSizeInBits()) {
537     // Different types of the same size.
538     assert(NumParts == 1 && PartEVT != ValueVT);
539     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
540   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
541     // If the parts cover less bits than value has, truncate the value.
542     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
543            ValueVT.isInteger() &&
544            "Unknown mismatch!");
545     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
546     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
547     if (PartVT == MVT::x86mmx)
548       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
549   }
550 
551   // The value may have changed - recompute ValueVT.
552   ValueVT = Val.getValueType();
553   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
554          "Failed to tile the value with PartVT!");
555 
556   if (NumParts == 1) {
557     if (PartEVT != ValueVT) {
558       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
559                                         "scalar-to-vector conversion failed");
560       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561     }
562 
563     Parts[0] = Val;
564     return;
565   }
566 
567   // Expand the value into multiple parts.
568   if (NumParts & (NumParts - 1)) {
569     // The number of parts is not a power of 2.  Split off and copy the tail.
570     assert(PartVT.isInteger() && ValueVT.isInteger() &&
571            "Do not know what to expand to!");
572     unsigned RoundParts = 1 << Log2_32(NumParts);
573     unsigned RoundBits = RoundParts * PartBits;
574     unsigned OddParts = NumParts - RoundParts;
575     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
576                                  DAG.getIntPtrConstant(RoundBits, DL));
577     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
578                    CallConv);
579 
580     if (DAG.getDataLayout().isBigEndian())
581       // The odd parts were reversed by getCopyToParts - unreverse them.
582       std::reverse(Parts + RoundParts, Parts + NumParts);
583 
584     NumParts = RoundParts;
585     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
586     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
587   }
588 
589   // The number of parts is a power of 2.  Repeatedly bisect the value using
590   // EXTRACT_ELEMENT.
591   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
592                          EVT::getIntegerVT(*DAG.getContext(),
593                                            ValueVT.getSizeInBits()),
594                          Val);
595 
596   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
597     for (unsigned i = 0; i < NumParts; i += StepSize) {
598       unsigned ThisBits = StepSize * PartBits / 2;
599       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
600       SDValue &Part0 = Parts[i];
601       SDValue &Part1 = Parts[i+StepSize/2];
602 
603       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
605       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
606                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
607 
608       if (ThisBits == PartBits && ThisVT != PartVT) {
609         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
610         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
611       }
612     }
613   }
614 
615   if (DAG.getDataLayout().isBigEndian())
616     std::reverse(Parts, Parts + OrigNumParts);
617 }
618 
619 static SDValue widenVectorToPartType(SelectionDAG &DAG,
620                                      SDValue Val, const SDLoc &DL, EVT PartVT) {
621   if (!PartVT.isVector())
622     return SDValue();
623 
624   EVT ValueVT = Val.getValueType();
625   unsigned PartNumElts = PartVT.getVectorNumElements();
626   unsigned ValueNumElts = ValueVT.getVectorNumElements();
627   if (PartNumElts > ValueNumElts &&
628       PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
629     EVT ElementVT = PartVT.getVectorElementType();
630     // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
631     // undef elements.
632     SmallVector<SDValue, 16> Ops;
633     DAG.ExtractVectorElements(Val, Ops);
634     SDValue EltUndef = DAG.getUNDEF(ElementVT);
635     for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
636       Ops.push_back(EltUndef);
637 
638     // FIXME: Use CONCAT for 2x -> 4x.
639     return DAG.getBuildVector(PartVT, DL, Ops);
640   }
641 
642   return SDValue();
643 }
644 
645 /// getCopyToPartsVector - Create a series of nodes that contain the specified
646 /// value split into legal parts.
647 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
648                                  SDValue Val, SDValue *Parts, unsigned NumParts,
649                                  MVT PartVT, const Value *V,
650                                  Optional<CallingConv::ID> CallConv) {
651   EVT ValueVT = Val.getValueType();
652   assert(ValueVT.isVector() && "Not a vector");
653   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
654   const bool IsABIRegCopy = CallConv.hasValue();
655 
656   if (NumParts == 1) {
657     EVT PartEVT = PartVT;
658     if (PartEVT == ValueVT) {
659       // Nothing to do.
660     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
661       // Bitconvert vector->vector case.
662       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
663     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
664       Val = Widened;
665     } else if (PartVT.isVector() &&
666                PartEVT.getVectorElementType().bitsGE(
667                  ValueVT.getVectorElementType()) &&
668                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
669 
670       // Promoted vector extract
671       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
672     } else {
673       if (ValueVT.getVectorNumElements() == 1) {
674         Val = DAG.getNode(
675             ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
676             DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
677       } else {
678         assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
679                "lossy conversion of vector to scalar type");
680         EVT IntermediateType =
681             EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
682         Val = DAG.getBitcast(IntermediateType, Val);
683         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
684       }
685     }
686 
687     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
688     Parts[0] = Val;
689     return;
690   }
691 
692   // Handle a multi-element vector.
693   EVT IntermediateVT;
694   MVT RegisterVT;
695   unsigned NumIntermediates;
696   unsigned NumRegs;
697   if (IsABIRegCopy) {
698     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
699         *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
700         NumIntermediates, RegisterVT);
701   } else {
702     NumRegs =
703         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
704                                    NumIntermediates, RegisterVT);
705   }
706 
707   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
708   NumParts = NumRegs; // Silence a compiler warning.
709   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
710 
711   unsigned IntermediateNumElts = IntermediateVT.isVector() ?
712     IntermediateVT.getVectorNumElements() : 1;
713 
714   // Convert the vector to the appropiate type if necessary.
715   unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
716 
717   EVT BuiltVectorTy = EVT::getVectorVT(
718       *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
719   MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
720   if (ValueVT != BuiltVectorTy) {
721     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
722       Val = Widened;
723 
724     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
725   }
726 
727   // Split the vector into intermediate operands.
728   SmallVector<SDValue, 8> Ops(NumIntermediates);
729   for (unsigned i = 0; i != NumIntermediates; ++i) {
730     if (IntermediateVT.isVector()) {
731       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
732                            DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
733     } else {
734       Ops[i] = DAG.getNode(
735           ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
736           DAG.getConstant(i, DL, IdxVT));
737     }
738   }
739 
740   // Split the intermediate operands into legal parts.
741   if (NumParts == NumIntermediates) {
742     // If the register was not expanded, promote or copy the value,
743     // as appropriate.
744     for (unsigned i = 0; i != NumParts; ++i)
745       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
746   } else if (NumParts > 0) {
747     // If the intermediate type was expanded, split each the value into
748     // legal parts.
749     assert(NumIntermediates != 0 && "division by zero");
750     assert(NumParts % NumIntermediates == 0 &&
751            "Must expand into a divisible number of parts!");
752     unsigned Factor = NumParts / NumIntermediates;
753     for (unsigned i = 0; i != NumIntermediates; ++i)
754       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
755                      CallConv);
756   }
757 }
758 
759 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
760                            EVT valuevt, Optional<CallingConv::ID> CC)
761     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
762       RegCount(1, regs.size()), CallConv(CC) {}
763 
764 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
765                            const DataLayout &DL, unsigned Reg, Type *Ty,
766                            Optional<CallingConv::ID> CC) {
767   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
768 
769   CallConv = CC;
770 
771   for (EVT ValueVT : ValueVTs) {
772     unsigned NumRegs =
773         isABIMangled()
774             ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
775             : TLI.getNumRegisters(Context, ValueVT);
776     MVT RegisterVT =
777         isABIMangled()
778             ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
779             : TLI.getRegisterType(Context, ValueVT);
780     for (unsigned i = 0; i != NumRegs; ++i)
781       Regs.push_back(Reg + i);
782     RegVTs.push_back(RegisterVT);
783     RegCount.push_back(NumRegs);
784     Reg += NumRegs;
785   }
786 }
787 
788 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
789                                       FunctionLoweringInfo &FuncInfo,
790                                       const SDLoc &dl, SDValue &Chain,
791                                       SDValue *Flag, const Value *V) const {
792   // A Value with type {} or [0 x %t] needs no registers.
793   if (ValueVTs.empty())
794     return SDValue();
795 
796   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
797 
798   // Assemble the legal parts into the final values.
799   SmallVector<SDValue, 4> Values(ValueVTs.size());
800   SmallVector<SDValue, 8> Parts;
801   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
802     // Copy the legal parts from the registers.
803     EVT ValueVT = ValueVTs[Value];
804     unsigned NumRegs = RegCount[Value];
805     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
806                                           *DAG.getContext(),
807                                           CallConv.getValue(), RegVTs[Value])
808                                     : RegVTs[Value];
809 
810     Parts.resize(NumRegs);
811     for (unsigned i = 0; i != NumRegs; ++i) {
812       SDValue P;
813       if (!Flag) {
814         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
815       } else {
816         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
817         *Flag = P.getValue(2);
818       }
819 
820       Chain = P.getValue(1);
821       Parts[i] = P;
822 
823       // If the source register was virtual and if we know something about it,
824       // add an assert node.
825       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
826           !RegisterVT.isInteger())
827         continue;
828 
829       const FunctionLoweringInfo::LiveOutInfo *LOI =
830         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
831       if (!LOI)
832         continue;
833 
834       unsigned RegSize = RegisterVT.getScalarSizeInBits();
835       unsigned NumSignBits = LOI->NumSignBits;
836       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
837 
838       if (NumZeroBits == RegSize) {
839         // The current value is a zero.
840         // Explicitly express that as it would be easier for
841         // optimizations to kick in.
842         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
843         continue;
844       }
845 
846       // FIXME: We capture more information than the dag can represent.  For
847       // now, just use the tightest assertzext/assertsext possible.
848       bool isSExt;
849       EVT FromVT(MVT::Other);
850       if (NumZeroBits) {
851         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
852         isSExt = false;
853       } else if (NumSignBits > 1) {
854         FromVT =
855             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
856         isSExt = true;
857       } else {
858         continue;
859       }
860       // Add an assertion node.
861       assert(FromVT != MVT::Other);
862       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
863                              RegisterVT, P, DAG.getValueType(FromVT));
864     }
865 
866     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
867                                      RegisterVT, ValueVT, V, CallConv);
868     Part += NumRegs;
869     Parts.clear();
870   }
871 
872   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
873 }
874 
875 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
876                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
877                                  const Value *V,
878                                  ISD::NodeType PreferredExtendType) const {
879   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
880   ISD::NodeType ExtendKind = PreferredExtendType;
881 
882   // Get the list of the values's legal parts.
883   unsigned NumRegs = Regs.size();
884   SmallVector<SDValue, 8> Parts(NumRegs);
885   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
886     unsigned NumParts = RegCount[Value];
887 
888     MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
889                                           *DAG.getContext(),
890                                           CallConv.getValue(), RegVTs[Value])
891                                     : RegVTs[Value];
892 
893     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
894       ExtendKind = ISD::ZERO_EXTEND;
895 
896     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
897                    NumParts, RegisterVT, V, CallConv, ExtendKind);
898     Part += NumParts;
899   }
900 
901   // Copy the parts into the registers.
902   SmallVector<SDValue, 8> Chains(NumRegs);
903   for (unsigned i = 0; i != NumRegs; ++i) {
904     SDValue Part;
905     if (!Flag) {
906       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
907     } else {
908       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
909       *Flag = Part.getValue(1);
910     }
911 
912     Chains[i] = Part.getValue(0);
913   }
914 
915   if (NumRegs == 1 || Flag)
916     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
917     // flagged to it. That is the CopyToReg nodes and the user are considered
918     // a single scheduling unit. If we create a TokenFactor and return it as
919     // chain, then the TokenFactor is both a predecessor (operand) of the
920     // user as well as a successor (the TF operands are flagged to the user).
921     // c1, f1 = CopyToReg
922     // c2, f2 = CopyToReg
923     // c3     = TokenFactor c1, c2
924     // ...
925     //        = op c3, ..., f2
926     Chain = Chains[NumRegs-1];
927   else
928     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
929 }
930 
931 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
932                                         unsigned MatchingIdx, const SDLoc &dl,
933                                         SelectionDAG &DAG,
934                                         std::vector<SDValue> &Ops) const {
935   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
936 
937   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
938   if (HasMatching)
939     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
940   else if (!Regs.empty() &&
941            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
942     // Put the register class of the virtual registers in the flag word.  That
943     // way, later passes can recompute register class constraints for inline
944     // assembly as well as normal instructions.
945     // Don't do this for tied operands that can use the regclass information
946     // from the def.
947     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
948     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
949     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
950   }
951 
952   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
953   Ops.push_back(Res);
954 
955   if (Code == InlineAsm::Kind_Clobber) {
956     // Clobbers should always have a 1:1 mapping with registers, and may
957     // reference registers that have illegal (e.g. vector) types. Hence, we
958     // shouldn't try to apply any sort of splitting logic to them.
959     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
960            "No 1:1 mapping from clobbers to regs?");
961     unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
962     (void)SP;
963     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
964       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
965       assert(
966           (Regs[I] != SP ||
967            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
968           "If we clobbered the stack pointer, MFI should know about it.");
969     }
970     return;
971   }
972 
973   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
974     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
975     MVT RegisterVT = RegVTs[Value];
976     for (unsigned i = 0; i != NumRegs; ++i) {
977       assert(Reg < Regs.size() && "Mismatch in # registers expected");
978       unsigned TheReg = Regs[Reg++];
979       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
980     }
981   }
982 }
983 
984 SmallVector<std::pair<unsigned, unsigned>, 4>
985 RegsForValue::getRegsAndSizes() const {
986   SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
987   unsigned I = 0;
988   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
989     unsigned RegCount = std::get<0>(CountAndVT);
990     MVT RegisterVT = std::get<1>(CountAndVT);
991     unsigned RegisterSize = RegisterVT.getSizeInBits();
992     for (unsigned E = I + RegCount; I != E; ++I)
993       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
994   }
995   return OutVec;
996 }
997 
998 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
999                                const TargetLibraryInfo *li) {
1000   AA = aa;
1001   GFI = gfi;
1002   LibInfo = li;
1003   DL = &DAG.getDataLayout();
1004   Context = DAG.getContext();
1005   LPadToCallSiteMap.clear();
1006 }
1007 
1008 void SelectionDAGBuilder::clear() {
1009   NodeMap.clear();
1010   UnusedArgNodeMap.clear();
1011   PendingLoads.clear();
1012   PendingExports.clear();
1013   CurInst = nullptr;
1014   HasTailCall = false;
1015   SDNodeOrder = LowestSDNodeOrder;
1016   StatepointLowering.clear();
1017 }
1018 
1019 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1020   DanglingDebugInfoMap.clear();
1021 }
1022 
1023 SDValue SelectionDAGBuilder::getRoot() {
1024   if (PendingLoads.empty())
1025     return DAG.getRoot();
1026 
1027   if (PendingLoads.size() == 1) {
1028     SDValue Root = PendingLoads[0];
1029     DAG.setRoot(Root);
1030     PendingLoads.clear();
1031     return Root;
1032   }
1033 
1034   // Otherwise, we have to make a token factor node.
1035   // If we have >= 2^16 loads then split across multiple token factors as
1036   // there's a 64k limit on the number of SDNode operands.
1037   SDValue Root;
1038   size_t Limit = (1 << 16) - 1;
1039   while (PendingLoads.size() > Limit) {
1040     unsigned SliceIdx = PendingLoads.size() - Limit;
1041     auto ExtractedTFs = ArrayRef<SDValue>(PendingLoads).slice(SliceIdx, Limit);
1042     SDValue NewTF =
1043         DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, ExtractedTFs);
1044     PendingLoads.erase(PendingLoads.begin() + SliceIdx, PendingLoads.end());
1045     PendingLoads.emplace_back(NewTF);
1046   }
1047   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads);
1048   PendingLoads.clear();
1049   DAG.setRoot(Root);
1050   return Root;
1051 }
1052 
1053 SDValue SelectionDAGBuilder::getControlRoot() {
1054   SDValue Root = DAG.getRoot();
1055 
1056   if (PendingExports.empty())
1057     return Root;
1058 
1059   // Turn all of the CopyToReg chains into one factored node.
1060   if (Root.getOpcode() != ISD::EntryToken) {
1061     unsigned i = 0, e = PendingExports.size();
1062     for (; i != e; ++i) {
1063       assert(PendingExports[i].getNode()->getNumOperands() > 1);
1064       if (PendingExports[i].getNode()->getOperand(0) == Root)
1065         break;  // Don't add the root if we already indirectly depend on it.
1066     }
1067 
1068     if (i == e)
1069       PendingExports.push_back(Root);
1070   }
1071 
1072   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1073                      PendingExports);
1074   PendingExports.clear();
1075   DAG.setRoot(Root);
1076   return Root;
1077 }
1078 
1079 void SelectionDAGBuilder::visit(const Instruction &I) {
1080   // Set up outgoing PHI node register values before emitting the terminator.
1081   if (I.isTerminator()) {
1082     HandlePHINodesInSuccessorBlocks(I.getParent());
1083   }
1084 
1085   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1086   if (!isa<DbgInfoIntrinsic>(I))
1087     ++SDNodeOrder;
1088 
1089   CurInst = &I;
1090 
1091   visit(I.getOpcode(), I);
1092 
1093   if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1094     // Propagate the fast-math-flags of this IR instruction to the DAG node that
1095     // maps to this instruction.
1096     // TODO: We could handle all flags (nsw, etc) here.
1097     // TODO: If an IR instruction maps to >1 node, only the final node will have
1098     //       flags set.
1099     if (SDNode *Node = getNodeForIRValue(&I)) {
1100       SDNodeFlags IncomingFlags;
1101       IncomingFlags.copyFMF(*FPMO);
1102       if (!Node->getFlags().isDefined())
1103         Node->setFlags(IncomingFlags);
1104       else
1105         Node->intersectFlagsWith(IncomingFlags);
1106     }
1107   }
1108 
1109   if (!I.isTerminator() && !HasTailCall &&
1110       !isStatepoint(&I)) // statepoints handle their exports internally
1111     CopyToExportRegsIfNeeded(&I);
1112 
1113   CurInst = nullptr;
1114 }
1115 
1116 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1117   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1118 }
1119 
1120 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1121   // Note: this doesn't use InstVisitor, because it has to work with
1122   // ConstantExpr's in addition to instructions.
1123   switch (Opcode) {
1124   default: llvm_unreachable("Unknown instruction type encountered!");
1125     // Build the switch statement using the Instruction.def file.
1126 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1127     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1128 #include "llvm/IR/Instruction.def"
1129   }
1130 }
1131 
1132 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1133                                                 const DIExpression *Expr) {
1134   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1135     const DbgValueInst *DI = DDI.getDI();
1136     DIVariable *DanglingVariable = DI->getVariable();
1137     DIExpression *DanglingExpr = DI->getExpression();
1138     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1139       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1140       return true;
1141     }
1142     return false;
1143   };
1144 
1145   for (auto &DDIMI : DanglingDebugInfoMap) {
1146     DanglingDebugInfoVector &DDIV = DDIMI.second;
1147     DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1148   }
1149 }
1150 
1151 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1152 // generate the debug data structures now that we've seen its definition.
1153 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1154                                                    SDValue Val) {
1155   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1156   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1157     return;
1158 
1159   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1160   for (auto &DDI : DDIV) {
1161     const DbgValueInst *DI = DDI.getDI();
1162     assert(DI && "Ill-formed DanglingDebugInfo");
1163     DebugLoc dl = DDI.getdl();
1164     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1165     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1166     DILocalVariable *Variable = DI->getVariable();
1167     DIExpression *Expr = DI->getExpression();
1168     assert(Variable->isValidLocationForIntrinsic(dl) &&
1169            "Expected inlined-at fields to agree");
1170     SDDbgValue *SDV;
1171     if (Val.getNode()) {
1172       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1173         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1174                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1175         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1176         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1177         // inserted after the definition of Val when emitting the instructions
1178         // after ISel. An alternative could be to teach
1179         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1180         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1181                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1182                    << ValSDNodeOrder << "\n");
1183         SDV = getDbgValue(Val, Variable, Expr, dl,
1184                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1185         DAG.AddDbgValue(SDV, Val.getNode(), false);
1186       } else
1187         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1188                           << "in EmitFuncArgumentDbgValue\n");
1189     } else
1190       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1191   }
1192   DDIV.clear();
1193 }
1194 
1195 /// getCopyFromRegs - If there was virtual register allocated for the value V
1196 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1197 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1198   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1199   SDValue Result;
1200 
1201   if (It != FuncInfo.ValueMap.end()) {
1202     unsigned InReg = It->second;
1203 
1204     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1205                      DAG.getDataLayout(), InReg, Ty,
1206                      None); // This is not an ABI copy.
1207     SDValue Chain = DAG.getEntryNode();
1208     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1209                                  V);
1210     resolveDanglingDebugInfo(V, Result);
1211   }
1212 
1213   return Result;
1214 }
1215 
1216 /// getValue - Return an SDValue for the given Value.
1217 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1218   // If we already have an SDValue for this value, use it. It's important
1219   // to do this first, so that we don't create a CopyFromReg if we already
1220   // have a regular SDValue.
1221   SDValue &N = NodeMap[V];
1222   if (N.getNode()) return N;
1223 
1224   // If there's a virtual register allocated and initialized for this
1225   // value, use it.
1226   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1227     return copyFromReg;
1228 
1229   // Otherwise create a new SDValue and remember it.
1230   SDValue Val = getValueImpl(V);
1231   NodeMap[V] = Val;
1232   resolveDanglingDebugInfo(V, Val);
1233   return Val;
1234 }
1235 
1236 // Return true if SDValue exists for the given Value
1237 bool SelectionDAGBuilder::findValue(const Value *V) const {
1238   return (NodeMap.find(V) != NodeMap.end()) ||
1239     (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1240 }
1241 
1242 /// getNonRegisterValue - Return an SDValue for the given Value, but
1243 /// don't look in FuncInfo.ValueMap for a virtual register.
1244 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1245   // If we already have an SDValue for this value, use it.
1246   SDValue &N = NodeMap[V];
1247   if (N.getNode()) {
1248     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1249       // Remove the debug location from the node as the node is about to be used
1250       // in a location which may differ from the original debug location.  This
1251       // is relevant to Constant and ConstantFP nodes because they can appear
1252       // as constant expressions inside PHI nodes.
1253       N->setDebugLoc(DebugLoc());
1254     }
1255     return N;
1256   }
1257 
1258   // Otherwise create a new SDValue and remember it.
1259   SDValue Val = getValueImpl(V);
1260   NodeMap[V] = Val;
1261   resolveDanglingDebugInfo(V, Val);
1262   return Val;
1263 }
1264 
1265 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1266 /// Create an SDValue for the given value.
1267 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1268   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1269 
1270   if (const Constant *C = dyn_cast<Constant>(V)) {
1271     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1272 
1273     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1274       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1275 
1276     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1277       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1278 
1279     if (isa<ConstantPointerNull>(C)) {
1280       unsigned AS = V->getType()->getPointerAddressSpace();
1281       return DAG.getConstant(0, getCurSDLoc(),
1282                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1283     }
1284 
1285     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1286       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1287 
1288     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1289       return DAG.getUNDEF(VT);
1290 
1291     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1292       visit(CE->getOpcode(), *CE);
1293       SDValue N1 = NodeMap[V];
1294       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1295       return N1;
1296     }
1297 
1298     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1299       SmallVector<SDValue, 4> Constants;
1300       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1301            OI != OE; ++OI) {
1302         SDNode *Val = getValue(*OI).getNode();
1303         // If the operand is an empty aggregate, there are no values.
1304         if (!Val) continue;
1305         // Add each leaf value from the operand to the Constants list
1306         // to form a flattened list of all the values.
1307         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1308           Constants.push_back(SDValue(Val, i));
1309       }
1310 
1311       return DAG.getMergeValues(Constants, getCurSDLoc());
1312     }
1313 
1314     if (const ConstantDataSequential *CDS =
1315           dyn_cast<ConstantDataSequential>(C)) {
1316       SmallVector<SDValue, 4> Ops;
1317       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1318         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1319         // Add each leaf value from the operand to the Constants list
1320         // to form a flattened list of all the values.
1321         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1322           Ops.push_back(SDValue(Val, i));
1323       }
1324 
1325       if (isa<ArrayType>(CDS->getType()))
1326         return DAG.getMergeValues(Ops, getCurSDLoc());
1327       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1328     }
1329 
1330     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1331       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1332              "Unknown struct or array constant!");
1333 
1334       SmallVector<EVT, 4> ValueVTs;
1335       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1336       unsigned NumElts = ValueVTs.size();
1337       if (NumElts == 0)
1338         return SDValue(); // empty struct
1339       SmallVector<SDValue, 4> Constants(NumElts);
1340       for (unsigned i = 0; i != NumElts; ++i) {
1341         EVT EltVT = ValueVTs[i];
1342         if (isa<UndefValue>(C))
1343           Constants[i] = DAG.getUNDEF(EltVT);
1344         else if (EltVT.isFloatingPoint())
1345           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1346         else
1347           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1348       }
1349 
1350       return DAG.getMergeValues(Constants, getCurSDLoc());
1351     }
1352 
1353     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1354       return DAG.getBlockAddress(BA, VT);
1355 
1356     VectorType *VecTy = cast<VectorType>(V->getType());
1357     unsigned NumElements = VecTy->getNumElements();
1358 
1359     // Now that we know the number and type of the elements, get that number of
1360     // elements into the Ops array based on what kind of constant it is.
1361     SmallVector<SDValue, 16> Ops;
1362     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1363       for (unsigned i = 0; i != NumElements; ++i)
1364         Ops.push_back(getValue(CV->getOperand(i)));
1365     } else {
1366       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1367       EVT EltVT =
1368           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1369 
1370       SDValue Op;
1371       if (EltVT.isFloatingPoint())
1372         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1373       else
1374         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1375       Ops.assign(NumElements, Op);
1376     }
1377 
1378     // Create a BUILD_VECTOR node.
1379     return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1380   }
1381 
1382   // If this is a static alloca, generate it as the frameindex instead of
1383   // computation.
1384   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1385     DenseMap<const AllocaInst*, int>::iterator SI =
1386       FuncInfo.StaticAllocaMap.find(AI);
1387     if (SI != FuncInfo.StaticAllocaMap.end())
1388       return DAG.getFrameIndex(SI->second,
1389                                TLI.getFrameIndexTy(DAG.getDataLayout()));
1390   }
1391 
1392   // If this is an instruction which fast-isel has deferred, select it now.
1393   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1394     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1395 
1396     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1397                      Inst->getType(), getABIRegCopyCC(V));
1398     SDValue Chain = DAG.getEntryNode();
1399     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1400   }
1401 
1402   llvm_unreachable("Can't get register for value!");
1403 }
1404 
1405 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1406   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1407   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1408   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1409   bool IsSEH = isAsynchronousEHPersonality(Pers);
1410   bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1411   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1412   if (!IsSEH)
1413     CatchPadMBB->setIsEHScopeEntry();
1414   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1415   if (IsMSVCCXX || IsCoreCLR)
1416     CatchPadMBB->setIsEHFuncletEntry();
1417   // Wasm does not need catchpads anymore
1418   if (!IsWasmCXX)
1419     DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1420                             getControlRoot()));
1421 }
1422 
1423 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1424   // Update machine-CFG edge.
1425   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1426   FuncInfo.MBB->addSuccessor(TargetMBB);
1427 
1428   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1429   bool IsSEH = isAsynchronousEHPersonality(Pers);
1430   if (IsSEH) {
1431     // If this is not a fall-through branch or optimizations are switched off,
1432     // emit the branch.
1433     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1434         TM.getOptLevel() == CodeGenOpt::None)
1435       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1436                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1437     return;
1438   }
1439 
1440   // Figure out the funclet membership for the catchret's successor.
1441   // This will be used by the FuncletLayout pass to determine how to order the
1442   // BB's.
1443   // A 'catchret' returns to the outer scope's color.
1444   Value *ParentPad = I.getCatchSwitchParentPad();
1445   const BasicBlock *SuccessorColor;
1446   if (isa<ConstantTokenNone>(ParentPad))
1447     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1448   else
1449     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1450   assert(SuccessorColor && "No parent funclet for catchret!");
1451   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1452   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1453 
1454   // Create the terminator node.
1455   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1456                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1457                             DAG.getBasicBlock(SuccessorColorMBB));
1458   DAG.setRoot(Ret);
1459 }
1460 
1461 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1462   // Don't emit any special code for the cleanuppad instruction. It just marks
1463   // the start of an EH scope/funclet.
1464   FuncInfo.MBB->setIsEHScopeEntry();
1465   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1466   if (Pers != EHPersonality::Wasm_CXX) {
1467     FuncInfo.MBB->setIsEHFuncletEntry();
1468     FuncInfo.MBB->setIsCleanupFuncletEntry();
1469   }
1470 }
1471 
1472 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1473 /// many places it could ultimately go. In the IR, we have a single unwind
1474 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1475 /// This function skips over imaginary basic blocks that hold catchswitch
1476 /// instructions, and finds all the "real" machine
1477 /// basic block destinations. As those destinations may not be successors of
1478 /// EHPadBB, here we also calculate the edge probability to those destinations.
1479 /// The passed-in Prob is the edge probability to EHPadBB.
1480 static void findUnwindDestinations(
1481     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1482     BranchProbability Prob,
1483     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1484         &UnwindDests) {
1485   EHPersonality Personality =
1486     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1487   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1488   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1489   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1490   bool IsSEH = isAsynchronousEHPersonality(Personality);
1491 
1492   while (EHPadBB) {
1493     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1494     BasicBlock *NewEHPadBB = nullptr;
1495     if (isa<LandingPadInst>(Pad)) {
1496       // Stop on landingpads. They are not funclets.
1497       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1498       break;
1499     } else if (isa<CleanupPadInst>(Pad)) {
1500       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1501       // personalities.
1502       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1503       UnwindDests.back().first->setIsEHScopeEntry();
1504       if (!IsWasmCXX)
1505         UnwindDests.back().first->setIsEHFuncletEntry();
1506       break;
1507     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1508       // Add the catchpad handlers to the possible destinations.
1509       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1510         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1511         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1512         if (IsMSVCCXX || IsCoreCLR)
1513           UnwindDests.back().first->setIsEHFuncletEntry();
1514         if (!IsSEH)
1515           UnwindDests.back().first->setIsEHScopeEntry();
1516       }
1517       NewEHPadBB = CatchSwitch->getUnwindDest();
1518     } else {
1519       continue;
1520     }
1521 
1522     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1523     if (BPI && NewEHPadBB)
1524       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1525     EHPadBB = NewEHPadBB;
1526   }
1527 }
1528 
1529 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1530   // Update successor info.
1531   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1532   auto UnwindDest = I.getUnwindDest();
1533   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1534   BranchProbability UnwindDestProb =
1535       (BPI && UnwindDest)
1536           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1537           : BranchProbability::getZero();
1538   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1539   for (auto &UnwindDest : UnwindDests) {
1540     UnwindDest.first->setIsEHPad();
1541     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1542   }
1543   FuncInfo.MBB->normalizeSuccProbs();
1544 
1545   // Create the terminator node.
1546   SDValue Ret =
1547       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1548   DAG.setRoot(Ret);
1549 }
1550 
1551 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1552   report_fatal_error("visitCatchSwitch not yet implemented!");
1553 }
1554 
1555 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1556   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1557   auto &DL = DAG.getDataLayout();
1558   SDValue Chain = getControlRoot();
1559   SmallVector<ISD::OutputArg, 8> Outs;
1560   SmallVector<SDValue, 8> OutVals;
1561 
1562   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1563   // lower
1564   //
1565   //   %val = call <ty> @llvm.experimental.deoptimize()
1566   //   ret <ty> %val
1567   //
1568   // differently.
1569   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1570     LowerDeoptimizingReturn();
1571     return;
1572   }
1573 
1574   if (!FuncInfo.CanLowerReturn) {
1575     unsigned DemoteReg = FuncInfo.DemoteRegister;
1576     const Function *F = I.getParent()->getParent();
1577 
1578     // Emit a store of the return value through the virtual register.
1579     // Leave Outs empty so that LowerReturn won't try to load return
1580     // registers the usual way.
1581     SmallVector<EVT, 1> PtrValueVTs;
1582     ComputeValueVTs(TLI, DL,
1583                     F->getReturnType()->getPointerTo(
1584                         DAG.getDataLayout().getAllocaAddrSpace()),
1585                     PtrValueVTs);
1586 
1587     SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1588                                         DemoteReg, PtrValueVTs[0]);
1589     SDValue RetOp = getValue(I.getOperand(0));
1590 
1591     SmallVector<EVT, 4> ValueVTs;
1592     SmallVector<uint64_t, 4> Offsets;
1593     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1594     unsigned NumValues = ValueVTs.size();
1595 
1596     SmallVector<SDValue, 4> Chains(NumValues);
1597     for (unsigned i = 0; i != NumValues; ++i) {
1598       // An aggregate return value cannot wrap around the address space, so
1599       // offsets to its parts don't wrap either.
1600       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1601       Chains[i] = DAG.getStore(
1602           Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1603           // FIXME: better loc info would be nice.
1604           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1605     }
1606 
1607     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1608                         MVT::Other, Chains);
1609   } else if (I.getNumOperands() != 0) {
1610     SmallVector<EVT, 4> ValueVTs;
1611     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1612     unsigned NumValues = ValueVTs.size();
1613     if (NumValues) {
1614       SDValue RetOp = getValue(I.getOperand(0));
1615 
1616       const Function *F = I.getParent()->getParent();
1617 
1618       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1619       if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1620                                           Attribute::SExt))
1621         ExtendKind = ISD::SIGN_EXTEND;
1622       else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1623                                                Attribute::ZExt))
1624         ExtendKind = ISD::ZERO_EXTEND;
1625 
1626       LLVMContext &Context = F->getContext();
1627       bool RetInReg = F->getAttributes().hasAttribute(
1628           AttributeList::ReturnIndex, Attribute::InReg);
1629 
1630       for (unsigned j = 0; j != NumValues; ++j) {
1631         EVT VT = ValueVTs[j];
1632 
1633         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1634           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1635 
1636         CallingConv::ID CC = F->getCallingConv();
1637 
1638         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1639         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1640         SmallVector<SDValue, 4> Parts(NumParts);
1641         getCopyToParts(DAG, getCurSDLoc(),
1642                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1643                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1644 
1645         // 'inreg' on function refers to return value
1646         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1647         if (RetInReg)
1648           Flags.setInReg();
1649 
1650         // Propagate extension type if any
1651         if (ExtendKind == ISD::SIGN_EXTEND)
1652           Flags.setSExt();
1653         else if (ExtendKind == ISD::ZERO_EXTEND)
1654           Flags.setZExt();
1655 
1656         for (unsigned i = 0; i < NumParts; ++i) {
1657           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1658                                         VT, /*isfixed=*/true, 0, 0));
1659           OutVals.push_back(Parts[i]);
1660         }
1661       }
1662     }
1663   }
1664 
1665   // Push in swifterror virtual register as the last element of Outs. This makes
1666   // sure swifterror virtual register will be returned in the swifterror
1667   // physical register.
1668   const Function *F = I.getParent()->getParent();
1669   if (TLI.supportSwiftError() &&
1670       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1671     assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1672     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1673     Flags.setSwiftError();
1674     Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1675                                   EVT(TLI.getPointerTy(DL)) /*argvt*/,
1676                                   true /*isfixed*/, 1 /*origidx*/,
1677                                   0 /*partOffs*/));
1678     // Create SDNode for the swifterror virtual register.
1679     OutVals.push_back(
1680         DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1681                             &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1682                         EVT(TLI.getPointerTy(DL))));
1683   }
1684 
1685   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1686   CallingConv::ID CallConv =
1687     DAG.getMachineFunction().getFunction().getCallingConv();
1688   Chain = DAG.getTargetLoweringInfo().LowerReturn(
1689       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1690 
1691   // Verify that the target's LowerReturn behaved as expected.
1692   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1693          "LowerReturn didn't return a valid chain!");
1694 
1695   // Update the DAG with the new chain value resulting from return lowering.
1696   DAG.setRoot(Chain);
1697 }
1698 
1699 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1700 /// created for it, emit nodes to copy the value into the virtual
1701 /// registers.
1702 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1703   // Skip empty types
1704   if (V->getType()->isEmptyTy())
1705     return;
1706 
1707   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1708   if (VMI != FuncInfo.ValueMap.end()) {
1709     assert(!V->use_empty() && "Unused value assigned virtual registers!");
1710     CopyValueToVirtualRegister(V, VMI->second);
1711   }
1712 }
1713 
1714 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1715 /// the current basic block, add it to ValueMap now so that we'll get a
1716 /// CopyTo/FromReg.
1717 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1718   // No need to export constants.
1719   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1720 
1721   // Already exported?
1722   if (FuncInfo.isExportedInst(V)) return;
1723 
1724   unsigned Reg = FuncInfo.InitializeRegForValue(V);
1725   CopyValueToVirtualRegister(V, Reg);
1726 }
1727 
1728 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1729                                                      const BasicBlock *FromBB) {
1730   // The operands of the setcc have to be in this block.  We don't know
1731   // how to export them from some other block.
1732   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1733     // Can export from current BB.
1734     if (VI->getParent() == FromBB)
1735       return true;
1736 
1737     // Is already exported, noop.
1738     return FuncInfo.isExportedInst(V);
1739   }
1740 
1741   // If this is an argument, we can export it if the BB is the entry block or
1742   // if it is already exported.
1743   if (isa<Argument>(V)) {
1744     if (FromBB == &FromBB->getParent()->getEntryBlock())
1745       return true;
1746 
1747     // Otherwise, can only export this if it is already exported.
1748     return FuncInfo.isExportedInst(V);
1749   }
1750 
1751   // Otherwise, constants can always be exported.
1752   return true;
1753 }
1754 
1755 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1756 BranchProbability
1757 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1758                                         const MachineBasicBlock *Dst) const {
1759   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1760   const BasicBlock *SrcBB = Src->getBasicBlock();
1761   const BasicBlock *DstBB = Dst->getBasicBlock();
1762   if (!BPI) {
1763     // If BPI is not available, set the default probability as 1 / N, where N is
1764     // the number of successors.
1765     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1766     return BranchProbability(1, SuccSize);
1767   }
1768   return BPI->getEdgeProbability(SrcBB, DstBB);
1769 }
1770 
1771 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1772                                                MachineBasicBlock *Dst,
1773                                                BranchProbability Prob) {
1774   if (!FuncInfo.BPI)
1775     Src->addSuccessorWithoutProb(Dst);
1776   else {
1777     if (Prob.isUnknown())
1778       Prob = getEdgeProbability(Src, Dst);
1779     Src->addSuccessor(Dst, Prob);
1780   }
1781 }
1782 
1783 static bool InBlock(const Value *V, const BasicBlock *BB) {
1784   if (const Instruction *I = dyn_cast<Instruction>(V))
1785     return I->getParent() == BB;
1786   return true;
1787 }
1788 
1789 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1790 /// This function emits a branch and is used at the leaves of an OR or an
1791 /// AND operator tree.
1792 void
1793 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1794                                                   MachineBasicBlock *TBB,
1795                                                   MachineBasicBlock *FBB,
1796                                                   MachineBasicBlock *CurBB,
1797                                                   MachineBasicBlock *SwitchBB,
1798                                                   BranchProbability TProb,
1799                                                   BranchProbability FProb,
1800                                                   bool InvertCond) {
1801   const BasicBlock *BB = CurBB->getBasicBlock();
1802 
1803   // If the leaf of the tree is a comparison, merge the condition into
1804   // the caseblock.
1805   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1806     // The operands of the cmp have to be in this block.  We don't know
1807     // how to export them from some other block.  If this is the first block
1808     // of the sequence, no exporting is needed.
1809     if (CurBB == SwitchBB ||
1810         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1811          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1812       ISD::CondCode Condition;
1813       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1814         ICmpInst::Predicate Pred =
1815             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1816         Condition = getICmpCondCode(Pred);
1817       } else {
1818         const FCmpInst *FC = cast<FCmpInst>(Cond);
1819         FCmpInst::Predicate Pred =
1820             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1821         Condition = getFCmpCondCode(Pred);
1822         if (TM.Options.NoNaNsFPMath)
1823           Condition = getFCmpCodeWithoutNaN(Condition);
1824       }
1825 
1826       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1827                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1828       SwitchCases.push_back(CB);
1829       return;
1830     }
1831   }
1832 
1833   // Create a CaseBlock record representing this branch.
1834   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1835   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1836                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1837   SwitchCases.push_back(CB);
1838 }
1839 
1840 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1841                                                MachineBasicBlock *TBB,
1842                                                MachineBasicBlock *FBB,
1843                                                MachineBasicBlock *CurBB,
1844                                                MachineBasicBlock *SwitchBB,
1845                                                Instruction::BinaryOps Opc,
1846                                                BranchProbability TProb,
1847                                                BranchProbability FProb,
1848                                                bool InvertCond) {
1849   // Skip over not part of the tree and remember to invert op and operands at
1850   // next level.
1851   Value *NotCond;
1852   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
1853       InBlock(NotCond, CurBB->getBasicBlock())) {
1854     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1855                          !InvertCond);
1856     return;
1857   }
1858 
1859   const Instruction *BOp = dyn_cast<Instruction>(Cond);
1860   // Compute the effective opcode for Cond, taking into account whether it needs
1861   // to be inverted, e.g.
1862   //   and (not (or A, B)), C
1863   // gets lowered as
1864   //   and (and (not A, not B), C)
1865   unsigned BOpc = 0;
1866   if (BOp) {
1867     BOpc = BOp->getOpcode();
1868     if (InvertCond) {
1869       if (BOpc == Instruction::And)
1870         BOpc = Instruction::Or;
1871       else if (BOpc == Instruction::Or)
1872         BOpc = Instruction::And;
1873     }
1874   }
1875 
1876   // If this node is not part of the or/and tree, emit it as a branch.
1877   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1878       BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1879       BOp->getParent() != CurBB->getBasicBlock() ||
1880       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1881       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1882     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1883                                  TProb, FProb, InvertCond);
1884     return;
1885   }
1886 
1887   //  Create TmpBB after CurBB.
1888   MachineFunction::iterator BBI(CurBB);
1889   MachineFunction &MF = DAG.getMachineFunction();
1890   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1891   CurBB->getParent()->insert(++BBI, TmpBB);
1892 
1893   if (Opc == Instruction::Or) {
1894     // Codegen X | Y as:
1895     // BB1:
1896     //   jmp_if_X TBB
1897     //   jmp TmpBB
1898     // TmpBB:
1899     //   jmp_if_Y TBB
1900     //   jmp FBB
1901     //
1902 
1903     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1904     // The requirement is that
1905     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1906     //     = TrueProb for original BB.
1907     // Assuming the original probabilities are A and B, one choice is to set
1908     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1909     // A/(1+B) and 2B/(1+B). This choice assumes that
1910     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1911     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1912     // TmpBB, but the math is more complicated.
1913 
1914     auto NewTrueProb = TProb / 2;
1915     auto NewFalseProb = TProb / 2 + FProb;
1916     // Emit the LHS condition.
1917     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1918                          NewTrueProb, NewFalseProb, InvertCond);
1919 
1920     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1921     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1922     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1923     // Emit the RHS condition into TmpBB.
1924     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1925                          Probs[0], Probs[1], InvertCond);
1926   } else {
1927     assert(Opc == Instruction::And && "Unknown merge op!");
1928     // Codegen X & Y as:
1929     // BB1:
1930     //   jmp_if_X TmpBB
1931     //   jmp FBB
1932     // TmpBB:
1933     //   jmp_if_Y TBB
1934     //   jmp FBB
1935     //
1936     //  This requires creation of TmpBB after CurBB.
1937 
1938     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1939     // The requirement is that
1940     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1941     //     = FalseProb for original BB.
1942     // Assuming the original probabilities are A and B, one choice is to set
1943     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1944     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1945     // TrueProb for BB1 * FalseProb for TmpBB.
1946 
1947     auto NewTrueProb = TProb + FProb / 2;
1948     auto NewFalseProb = FProb / 2;
1949     // Emit the LHS condition.
1950     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1951                          NewTrueProb, NewFalseProb, InvertCond);
1952 
1953     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1954     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1955     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1956     // Emit the RHS condition into TmpBB.
1957     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1958                          Probs[0], Probs[1], InvertCond);
1959   }
1960 }
1961 
1962 /// If the set of cases should be emitted as a series of branches, return true.
1963 /// If we should emit this as a bunch of and/or'd together conditions, return
1964 /// false.
1965 bool
1966 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1967   if (Cases.size() != 2) return true;
1968 
1969   // If this is two comparisons of the same values or'd or and'd together, they
1970   // will get folded into a single comparison, so don't emit two blocks.
1971   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1972        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1973       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1974        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1975     return false;
1976   }
1977 
1978   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1979   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1980   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1981       Cases[0].CC == Cases[1].CC &&
1982       isa<Constant>(Cases[0].CmpRHS) &&
1983       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1984     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1985       return false;
1986     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1987       return false;
1988   }
1989 
1990   return true;
1991 }
1992 
1993 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1994   MachineBasicBlock *BrMBB = FuncInfo.MBB;
1995 
1996   // Update machine-CFG edges.
1997   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1998 
1999   if (I.isUnconditional()) {
2000     // Update machine-CFG edges.
2001     BrMBB->addSuccessor(Succ0MBB);
2002 
2003     // If this is not a fall-through branch or optimizations are switched off,
2004     // emit the branch.
2005     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2006       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2007                               MVT::Other, getControlRoot(),
2008                               DAG.getBasicBlock(Succ0MBB)));
2009 
2010     return;
2011   }
2012 
2013   // If this condition is one of the special cases we handle, do special stuff
2014   // now.
2015   const Value *CondVal = I.getCondition();
2016   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2017 
2018   // If this is a series of conditions that are or'd or and'd together, emit
2019   // this as a sequence of branches instead of setcc's with and/or operations.
2020   // As long as jumps are not expensive, this should improve performance.
2021   // For example, instead of something like:
2022   //     cmp A, B
2023   //     C = seteq
2024   //     cmp D, E
2025   //     F = setle
2026   //     or C, F
2027   //     jnz foo
2028   // Emit:
2029   //     cmp A, B
2030   //     je foo
2031   //     cmp D, E
2032   //     jle foo
2033   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2034     Instruction::BinaryOps Opcode = BOp->getOpcode();
2035     if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2036         !I.getMetadata(LLVMContext::MD_unpredictable) &&
2037         (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2038       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2039                            Opcode,
2040                            getEdgeProbability(BrMBB, Succ0MBB),
2041                            getEdgeProbability(BrMBB, Succ1MBB),
2042                            /*InvertCond=*/false);
2043       // If the compares in later blocks need to use values not currently
2044       // exported from this block, export them now.  This block should always
2045       // be the first entry.
2046       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2047 
2048       // Allow some cases to be rejected.
2049       if (ShouldEmitAsBranches(SwitchCases)) {
2050         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2051           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2052           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2053         }
2054 
2055         // Emit the branch for this block.
2056         visitSwitchCase(SwitchCases[0], BrMBB);
2057         SwitchCases.erase(SwitchCases.begin());
2058         return;
2059       }
2060 
2061       // Okay, we decided not to do this, remove any inserted MBB's and clear
2062       // SwitchCases.
2063       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2064         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2065 
2066       SwitchCases.clear();
2067     }
2068   }
2069 
2070   // Create a CaseBlock record representing this branch.
2071   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2072                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2073 
2074   // Use visitSwitchCase to actually insert the fast branch sequence for this
2075   // cond branch.
2076   visitSwitchCase(CB, BrMBB);
2077 }
2078 
2079 /// visitSwitchCase - Emits the necessary code to represent a single node in
2080 /// the binary search tree resulting from lowering a switch instruction.
2081 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2082                                           MachineBasicBlock *SwitchBB) {
2083   SDValue Cond;
2084   SDValue CondLHS = getValue(CB.CmpLHS);
2085   SDLoc dl = CB.DL;
2086 
2087   // Build the setcc now.
2088   if (!CB.CmpMHS) {
2089     // Fold "(X == true)" to X and "(X == false)" to !X to
2090     // handle common cases produced by branch lowering.
2091     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2092         CB.CC == ISD::SETEQ)
2093       Cond = CondLHS;
2094     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2095              CB.CC == ISD::SETEQ) {
2096       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2097       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2098     } else
2099       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2100   } else {
2101     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2102 
2103     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2104     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2105 
2106     SDValue CmpOp = getValue(CB.CmpMHS);
2107     EVT VT = CmpOp.getValueType();
2108 
2109     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2110       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2111                           ISD::SETLE);
2112     } else {
2113       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2114                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2115       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2116                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2117     }
2118   }
2119 
2120   // Update successor info
2121   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2122   // TrueBB and FalseBB are always different unless the incoming IR is
2123   // degenerate. This only happens when running llc on weird IR.
2124   if (CB.TrueBB != CB.FalseBB)
2125     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2126   SwitchBB->normalizeSuccProbs();
2127 
2128   // If the lhs block is the next block, invert the condition so that we can
2129   // fall through to the lhs instead of the rhs block.
2130   if (CB.TrueBB == NextBlock(SwitchBB)) {
2131     std::swap(CB.TrueBB, CB.FalseBB);
2132     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2133     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2134   }
2135 
2136   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2137                                MVT::Other, getControlRoot(), Cond,
2138                                DAG.getBasicBlock(CB.TrueBB));
2139 
2140   // Insert the false branch. Do this even if it's a fall through branch,
2141   // this makes it easier to do DAG optimizations which require inverting
2142   // the branch condition.
2143   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2144                        DAG.getBasicBlock(CB.FalseBB));
2145 
2146   DAG.setRoot(BrCond);
2147 }
2148 
2149 /// visitJumpTable - Emit JumpTable node in the current MBB
2150 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
2151   // Emit the code for the jump table
2152   assert(JT.Reg != -1U && "Should lower JT Header first!");
2153   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2154   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2155                                      JT.Reg, PTy);
2156   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2157   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2158                                     MVT::Other, Index.getValue(1),
2159                                     Table, Index);
2160   DAG.setRoot(BrJumpTable);
2161 }
2162 
2163 /// visitJumpTableHeader - This function emits necessary code to produce index
2164 /// in the JumpTable from switch case.
2165 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
2166                                                JumpTableHeader &JTH,
2167                                                MachineBasicBlock *SwitchBB) {
2168   SDLoc dl = getCurSDLoc();
2169 
2170   // Subtract the lowest switch case value from the value being switched on and
2171   // conditional branch to default mbb if the result is greater than the
2172   // difference between smallest and largest cases.
2173   SDValue SwitchOp = getValue(JTH.SValue);
2174   EVT VT = SwitchOp.getValueType();
2175   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2176                             DAG.getConstant(JTH.First, dl, VT));
2177 
2178   // The SDNode we just created, which holds the value being switched on minus
2179   // the smallest case value, needs to be copied to a virtual register so it
2180   // can be used as an index into the jump table in a subsequent basic block.
2181   // This value may be smaller or larger than the target's pointer type, and
2182   // therefore require extension or truncating.
2183   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2184   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2185 
2186   unsigned JumpTableReg =
2187       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2188   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2189                                     JumpTableReg, SwitchOp);
2190   JT.Reg = JumpTableReg;
2191 
2192   // Emit the range check for the jump table, and branch to the default block
2193   // for the switch statement if the value being switched on exceeds the largest
2194   // case in the switch.
2195   SDValue CMP = DAG.getSetCC(
2196       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2197                                  Sub.getValueType()),
2198       Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2199 
2200   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2201                                MVT::Other, CopyTo, CMP,
2202                                DAG.getBasicBlock(JT.Default));
2203 
2204   // Avoid emitting unnecessary branches to the next block.
2205   if (JT.MBB != NextBlock(SwitchBB))
2206     BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2207                          DAG.getBasicBlock(JT.MBB));
2208 
2209   DAG.setRoot(BrCond);
2210 }
2211 
2212 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2213 /// variable if there exists one.
2214 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2215                                  SDValue &Chain) {
2216   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2217   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2218   MachineFunction &MF = DAG.getMachineFunction();
2219   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2220   MachineSDNode *Node =
2221       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2222   if (Global) {
2223     MachinePointerInfo MPInfo(Global);
2224     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2225                  MachineMemOperand::MODereferenceable;
2226     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2227         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2228     DAG.setNodeMemRefs(Node, {MemRef});
2229   }
2230   return SDValue(Node, 0);
2231 }
2232 
2233 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2234 /// tail spliced into a stack protector check success bb.
2235 ///
2236 /// For a high level explanation of how this fits into the stack protector
2237 /// generation see the comment on the declaration of class
2238 /// StackProtectorDescriptor.
2239 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2240                                                   MachineBasicBlock *ParentBB) {
2241 
2242   // First create the loads to the guard/stack slot for the comparison.
2243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2244   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2245 
2246   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2247   int FI = MFI.getStackProtectorIndex();
2248 
2249   SDValue Guard;
2250   SDLoc dl = getCurSDLoc();
2251   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2252   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2253   unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2254 
2255   // Generate code to load the content of the guard slot.
2256   SDValue GuardVal = DAG.getLoad(
2257       PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2258       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2259       MachineMemOperand::MOVolatile);
2260 
2261   if (TLI.useStackGuardXorFP())
2262     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2263 
2264   // Retrieve guard check function, nullptr if instrumentation is inlined.
2265   if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2266     // The target provides a guard check function to validate the guard value.
2267     // Generate a call to that function with the content of the guard slot as
2268     // argument.
2269     auto *Fn = cast<Function>(GuardCheck);
2270     FunctionType *FnTy = Fn->getFunctionType();
2271     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2272 
2273     TargetLowering::ArgListTy Args;
2274     TargetLowering::ArgListEntry Entry;
2275     Entry.Node = GuardVal;
2276     Entry.Ty = FnTy->getParamType(0);
2277     if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2278       Entry.IsInReg = true;
2279     Args.push_back(Entry);
2280 
2281     TargetLowering::CallLoweringInfo CLI(DAG);
2282     CLI.setDebugLoc(getCurSDLoc())
2283       .setChain(DAG.getEntryNode())
2284       .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2285                  getValue(GuardCheck), std::move(Args));
2286 
2287     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2288     DAG.setRoot(Result.second);
2289     return;
2290   }
2291 
2292   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2293   // Otherwise, emit a volatile load to retrieve the stack guard value.
2294   SDValue Chain = DAG.getEntryNode();
2295   if (TLI.useLoadStackGuardNode()) {
2296     Guard = getLoadStackGuard(DAG, dl, Chain);
2297   } else {
2298     const Value *IRGuard = TLI.getSDagStackGuard(M);
2299     SDValue GuardPtr = getValue(IRGuard);
2300 
2301     Guard =
2302         DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2303                     Align, MachineMemOperand::MOVolatile);
2304   }
2305 
2306   // Perform the comparison via a subtract/getsetcc.
2307   EVT VT = Guard.getValueType();
2308   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2309 
2310   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2311                                                         *DAG.getContext(),
2312                                                         Sub.getValueType()),
2313                              Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2314 
2315   // If the sub is not 0, then we know the guard/stackslot do not equal, so
2316   // branch to failure MBB.
2317   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2318                                MVT::Other, GuardVal.getOperand(0),
2319                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2320   // Otherwise branch to success MBB.
2321   SDValue Br = DAG.getNode(ISD::BR, dl,
2322                            MVT::Other, BrCond,
2323                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2324 
2325   DAG.setRoot(Br);
2326 }
2327 
2328 /// Codegen the failure basic block for a stack protector check.
2329 ///
2330 /// A failure stack protector machine basic block consists simply of a call to
2331 /// __stack_chk_fail().
2332 ///
2333 /// For a high level explanation of how this fits into the stack protector
2334 /// generation see the comment on the declaration of class
2335 /// StackProtectorDescriptor.
2336 void
2337 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2338   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2339   SDValue Chain =
2340       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2341                       None, false, getCurSDLoc(), false, false).second;
2342   DAG.setRoot(Chain);
2343 }
2344 
2345 /// visitBitTestHeader - This function emits necessary code to produce value
2346 /// suitable for "bit tests"
2347 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2348                                              MachineBasicBlock *SwitchBB) {
2349   SDLoc dl = getCurSDLoc();
2350 
2351   // Subtract the minimum value
2352   SDValue SwitchOp = getValue(B.SValue);
2353   EVT VT = SwitchOp.getValueType();
2354   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2355                             DAG.getConstant(B.First, dl, VT));
2356 
2357   // Check range
2358   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2359   SDValue RangeCmp = DAG.getSetCC(
2360       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2361                                  Sub.getValueType()),
2362       Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2363 
2364   // Determine the type of the test operands.
2365   bool UsePtrType = false;
2366   if (!TLI.isTypeLegal(VT))
2367     UsePtrType = true;
2368   else {
2369     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2370       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2371         // Switch table case range are encoded into series of masks.
2372         // Just use pointer type, it's guaranteed to fit.
2373         UsePtrType = true;
2374         break;
2375       }
2376   }
2377   if (UsePtrType) {
2378     VT = TLI.getPointerTy(DAG.getDataLayout());
2379     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2380   }
2381 
2382   B.RegVT = VT.getSimpleVT();
2383   B.Reg = FuncInfo.CreateReg(B.RegVT);
2384   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2385 
2386   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2387 
2388   addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2389   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2390   SwitchBB->normalizeSuccProbs();
2391 
2392   SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2393                                 MVT::Other, CopyTo, RangeCmp,
2394                                 DAG.getBasicBlock(B.Default));
2395 
2396   // Avoid emitting unnecessary branches to the next block.
2397   if (MBB != NextBlock(SwitchBB))
2398     BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2399                           DAG.getBasicBlock(MBB));
2400 
2401   DAG.setRoot(BrRange);
2402 }
2403 
2404 /// visitBitTestCase - this function produces one "bit test"
2405 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2406                                            MachineBasicBlock* NextMBB,
2407                                            BranchProbability BranchProbToNext,
2408                                            unsigned Reg,
2409                                            BitTestCase &B,
2410                                            MachineBasicBlock *SwitchBB) {
2411   SDLoc dl = getCurSDLoc();
2412   MVT VT = BB.RegVT;
2413   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2414   SDValue Cmp;
2415   unsigned PopCount = countPopulation(B.Mask);
2416   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2417   if (PopCount == 1) {
2418     // Testing for a single bit; just compare the shift count with what it
2419     // would need to be to shift a 1 bit in that position.
2420     Cmp = DAG.getSetCC(
2421         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2422         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2423         ISD::SETEQ);
2424   } else if (PopCount == BB.Range) {
2425     // There is only one zero bit in the range, test for it directly.
2426     Cmp = DAG.getSetCC(
2427         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2428         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2429         ISD::SETNE);
2430   } else {
2431     // Make desired shift
2432     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2433                                     DAG.getConstant(1, dl, VT), ShiftOp);
2434 
2435     // Emit bit tests and jumps
2436     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2437                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2438     Cmp = DAG.getSetCC(
2439         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2440         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2441   }
2442 
2443   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2444   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2445   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2446   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2447   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2448   // one as they are relative probabilities (and thus work more like weights),
2449   // and hence we need to normalize them to let the sum of them become one.
2450   SwitchBB->normalizeSuccProbs();
2451 
2452   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2453                               MVT::Other, getControlRoot(),
2454                               Cmp, DAG.getBasicBlock(B.TargetBB));
2455 
2456   // Avoid emitting unnecessary branches to the next block.
2457   if (NextMBB != NextBlock(SwitchBB))
2458     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2459                         DAG.getBasicBlock(NextMBB));
2460 
2461   DAG.setRoot(BrAnd);
2462 }
2463 
2464 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2465   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2466 
2467   // Retrieve successors. Look through artificial IR level blocks like
2468   // catchswitch for successors.
2469   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2470   const BasicBlock *EHPadBB = I.getSuccessor(1);
2471 
2472   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2473   // have to do anything here to lower funclet bundles.
2474   assert(!I.hasOperandBundlesOtherThan(
2475              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2476          "Cannot lower invokes with arbitrary operand bundles yet!");
2477 
2478   const Value *Callee(I.getCalledValue());
2479   const Function *Fn = dyn_cast<Function>(Callee);
2480   if (isa<InlineAsm>(Callee))
2481     visitInlineAsm(&I);
2482   else if (Fn && Fn->isIntrinsic()) {
2483     switch (Fn->getIntrinsicID()) {
2484     default:
2485       llvm_unreachable("Cannot invoke this intrinsic");
2486     case Intrinsic::donothing:
2487       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2488       break;
2489     case Intrinsic::experimental_patchpoint_void:
2490     case Intrinsic::experimental_patchpoint_i64:
2491       visitPatchpoint(&I, EHPadBB);
2492       break;
2493     case Intrinsic::experimental_gc_statepoint:
2494       LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2495       break;
2496     }
2497   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2498     // Currently we do not lower any intrinsic calls with deopt operand bundles.
2499     // Eventually we will support lowering the @llvm.experimental.deoptimize
2500     // intrinsic, and right now there are no plans to support other intrinsics
2501     // with deopt state.
2502     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2503   } else {
2504     LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2505   }
2506 
2507   // If the value of the invoke is used outside of its defining block, make it
2508   // available as a virtual register.
2509   // We already took care of the exported value for the statepoint instruction
2510   // during call to the LowerStatepoint.
2511   if (!isStatepoint(I)) {
2512     CopyToExportRegsIfNeeded(&I);
2513   }
2514 
2515   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2516   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2517   BranchProbability EHPadBBProb =
2518       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2519           : BranchProbability::getZero();
2520   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2521 
2522   // Update successor info.
2523   addSuccessorWithProb(InvokeMBB, Return);
2524   for (auto &UnwindDest : UnwindDests) {
2525     UnwindDest.first->setIsEHPad();
2526     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2527   }
2528   InvokeMBB->normalizeSuccProbs();
2529 
2530   // Drop into normal successor.
2531   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2532                           MVT::Other, getControlRoot(),
2533                           DAG.getBasicBlock(Return)));
2534 }
2535 
2536 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2537   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2538 }
2539 
2540 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2541   assert(FuncInfo.MBB->isEHPad() &&
2542          "Call to landingpad not in landing pad!");
2543 
2544   // If there aren't registers to copy the values into (e.g., during SjLj
2545   // exceptions), then don't bother to create these DAG nodes.
2546   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2547   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2548   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2549       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2550     return;
2551 
2552   // If landingpad's return type is token type, we don't create DAG nodes
2553   // for its exception pointer and selector value. The extraction of exception
2554   // pointer or selector value from token type landingpads is not currently
2555   // supported.
2556   if (LP.getType()->isTokenTy())
2557     return;
2558 
2559   SmallVector<EVT, 2> ValueVTs;
2560   SDLoc dl = getCurSDLoc();
2561   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2562   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2563 
2564   // Get the two live-in registers as SDValues. The physregs have already been
2565   // copied into virtual registers.
2566   SDValue Ops[2];
2567   if (FuncInfo.ExceptionPointerVirtReg) {
2568     Ops[0] = DAG.getZExtOrTrunc(
2569         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2570                            FuncInfo.ExceptionPointerVirtReg,
2571                            TLI.getPointerTy(DAG.getDataLayout())),
2572         dl, ValueVTs[0]);
2573   } else {
2574     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2575   }
2576   Ops[1] = DAG.getZExtOrTrunc(
2577       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2578                          FuncInfo.ExceptionSelectorVirtReg,
2579                          TLI.getPointerTy(DAG.getDataLayout())),
2580       dl, ValueVTs[1]);
2581 
2582   // Merge into one.
2583   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2584                             DAG.getVTList(ValueVTs), Ops);
2585   setValue(&LP, Res);
2586 }
2587 
2588 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2589 #ifndef NDEBUG
2590   for (const CaseCluster &CC : Clusters)
2591     assert(CC.Low == CC.High && "Input clusters must be single-case");
2592 #endif
2593 
2594   llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) {
2595     return a.Low->getValue().slt(b.Low->getValue());
2596   });
2597 
2598   // Merge adjacent clusters with the same destination.
2599   const unsigned N = Clusters.size();
2600   unsigned DstIndex = 0;
2601   for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2602     CaseCluster &CC = Clusters[SrcIndex];
2603     const ConstantInt *CaseVal = CC.Low;
2604     MachineBasicBlock *Succ = CC.MBB;
2605 
2606     if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2607         (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2608       // If this case has the same successor and is a neighbour, merge it into
2609       // the previous cluster.
2610       Clusters[DstIndex - 1].High = CaseVal;
2611       Clusters[DstIndex - 1].Prob += CC.Prob;
2612     } else {
2613       std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2614                    sizeof(Clusters[SrcIndex]));
2615     }
2616   }
2617   Clusters.resize(DstIndex);
2618 }
2619 
2620 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2621                                            MachineBasicBlock *Last) {
2622   // Update JTCases.
2623   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2624     if (JTCases[i].first.HeaderBB == First)
2625       JTCases[i].first.HeaderBB = Last;
2626 
2627   // Update BitTestCases.
2628   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2629     if (BitTestCases[i].Parent == First)
2630       BitTestCases[i].Parent = Last;
2631 }
2632 
2633 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2634   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2635 
2636   // Update machine-CFG edges with unique successors.
2637   SmallSet<BasicBlock*, 32> Done;
2638   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2639     BasicBlock *BB = I.getSuccessor(i);
2640     bool Inserted = Done.insert(BB).second;
2641     if (!Inserted)
2642         continue;
2643 
2644     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2645     addSuccessorWithProb(IndirectBrMBB, Succ);
2646   }
2647   IndirectBrMBB->normalizeSuccProbs();
2648 
2649   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2650                           MVT::Other, getControlRoot(),
2651                           getValue(I.getAddress())));
2652 }
2653 
2654 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2655   if (!DAG.getTarget().Options.TrapUnreachable)
2656     return;
2657 
2658   // We may be able to ignore unreachable behind a noreturn call.
2659   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
2660     const BasicBlock &BB = *I.getParent();
2661     if (&I != &BB.front()) {
2662       BasicBlock::const_iterator PredI =
2663         std::prev(BasicBlock::const_iterator(&I));
2664       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2665         if (Call->doesNotReturn())
2666           return;
2667       }
2668     }
2669   }
2670 
2671   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2672 }
2673 
2674 void SelectionDAGBuilder::visitFSub(const User &I) {
2675   // -0.0 - X --> fneg
2676   Type *Ty = I.getType();
2677   if (isa<Constant>(I.getOperand(0)) &&
2678       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2679     SDValue Op2 = getValue(I.getOperand(1));
2680     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2681                              Op2.getValueType(), Op2));
2682     return;
2683   }
2684 
2685   visitBinary(I, ISD::FSUB);
2686 }
2687 
2688 /// Checks if the given instruction performs a vector reduction, in which case
2689 /// we have the freedom to alter the elements in the result as long as the
2690 /// reduction of them stays unchanged.
2691 static bool isVectorReductionOp(const User *I) {
2692   const Instruction *Inst = dyn_cast<Instruction>(I);
2693   if (!Inst || !Inst->getType()->isVectorTy())
2694     return false;
2695 
2696   auto OpCode = Inst->getOpcode();
2697   switch (OpCode) {
2698   case Instruction::Add:
2699   case Instruction::Mul:
2700   case Instruction::And:
2701   case Instruction::Or:
2702   case Instruction::Xor:
2703     break;
2704   case Instruction::FAdd:
2705   case Instruction::FMul:
2706     if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2707       if (FPOp->getFastMathFlags().isFast())
2708         break;
2709     LLVM_FALLTHROUGH;
2710   default:
2711     return false;
2712   }
2713 
2714   unsigned ElemNum = Inst->getType()->getVectorNumElements();
2715   // Ensure the reduction size is a power of 2.
2716   if (!isPowerOf2_32(ElemNum))
2717     return false;
2718 
2719   unsigned ElemNumToReduce = ElemNum;
2720 
2721   // Do DFS search on the def-use chain from the given instruction. We only
2722   // allow four kinds of operations during the search until we reach the
2723   // instruction that extracts the first element from the vector:
2724   //
2725   //   1. The reduction operation of the same opcode as the given instruction.
2726   //
2727   //   2. PHI node.
2728   //
2729   //   3. ShuffleVector instruction together with a reduction operation that
2730   //      does a partial reduction.
2731   //
2732   //   4. ExtractElement that extracts the first element from the vector, and we
2733   //      stop searching the def-use chain here.
2734   //
2735   // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2736   // from 1-3 to the stack to continue the DFS. The given instruction is not
2737   // a reduction operation if we meet any other instructions other than those
2738   // listed above.
2739 
2740   SmallVector<const User *, 16> UsersToVisit{Inst};
2741   SmallPtrSet<const User *, 16> Visited;
2742   bool ReduxExtracted = false;
2743 
2744   while (!UsersToVisit.empty()) {
2745     auto User = UsersToVisit.back();
2746     UsersToVisit.pop_back();
2747     if (!Visited.insert(User).second)
2748       continue;
2749 
2750     for (const auto &U : User->users()) {
2751       auto Inst = dyn_cast<Instruction>(U);
2752       if (!Inst)
2753         return false;
2754 
2755       if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2756         if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2757           if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2758             return false;
2759         UsersToVisit.push_back(U);
2760       } else if (const ShuffleVectorInst *ShufInst =
2761                      dyn_cast<ShuffleVectorInst>(U)) {
2762         // Detect the following pattern: A ShuffleVector instruction together
2763         // with a reduction that do partial reduction on the first and second
2764         // ElemNumToReduce / 2 elements, and store the result in
2765         // ElemNumToReduce / 2 elements in another vector.
2766 
2767         unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2768         if (ResultElements < ElemNum)
2769           return false;
2770 
2771         if (ElemNumToReduce == 1)
2772           return false;
2773         if (!isa<UndefValue>(U->getOperand(1)))
2774           return false;
2775         for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2776           if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2777             return false;
2778         for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2779           if (ShufInst->getMaskValue(i) != -1)
2780             return false;
2781 
2782         // There is only one user of this ShuffleVector instruction, which
2783         // must be a reduction operation.
2784         if (!U->hasOneUse())
2785           return false;
2786 
2787         auto U2 = dyn_cast<Instruction>(*U->user_begin());
2788         if (!U2 || U2->getOpcode() != OpCode)
2789           return false;
2790 
2791         // Check operands of the reduction operation.
2792         if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2793             (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2794           UsersToVisit.push_back(U2);
2795           ElemNumToReduce /= 2;
2796         } else
2797           return false;
2798       } else if (isa<ExtractElementInst>(U)) {
2799         // At this moment we should have reduced all elements in the vector.
2800         if (ElemNumToReduce != 1)
2801           return false;
2802 
2803         const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2804         if (!Val || !Val->isZero())
2805           return false;
2806 
2807         ReduxExtracted = true;
2808       } else
2809         return false;
2810     }
2811   }
2812   return ReduxExtracted;
2813 }
2814 
2815 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
2816   SDNodeFlags Flags;
2817 
2818   SDValue Op = getValue(I.getOperand(0));
2819   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
2820                                     Op, Flags);
2821   setValue(&I, UnNodeValue);
2822 }
2823 
2824 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2825   SDNodeFlags Flags;
2826   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2827     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2828     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2829   }
2830   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2831     Flags.setExact(ExactOp->isExact());
2832   }
2833   if (isVectorReductionOp(&I)) {
2834     Flags.setVectorReduction(true);
2835     LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2836   }
2837 
2838   SDValue Op1 = getValue(I.getOperand(0));
2839   SDValue Op2 = getValue(I.getOperand(1));
2840   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2841                                      Op1, Op2, Flags);
2842   setValue(&I, BinNodeValue);
2843 }
2844 
2845 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2846   SDValue Op1 = getValue(I.getOperand(0));
2847   SDValue Op2 = getValue(I.getOperand(1));
2848 
2849   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2850       Op1.getValueType(), DAG.getDataLayout());
2851 
2852   // Coerce the shift amount to the right type if we can.
2853   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2854     unsigned ShiftSize = ShiftTy.getSizeInBits();
2855     unsigned Op2Size = Op2.getValueSizeInBits();
2856     SDLoc DL = getCurSDLoc();
2857 
2858     // If the operand is smaller than the shift count type, promote it.
2859     if (ShiftSize > Op2Size)
2860       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2861 
2862     // If the operand is larger than the shift count type but the shift
2863     // count type has enough bits to represent any shift value, truncate
2864     // it now. This is a common case and it exposes the truncate to
2865     // optimization early.
2866     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2867       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2868     // Otherwise we'll need to temporarily settle for some other convenient
2869     // type.  Type legalization will make adjustments once the shiftee is split.
2870     else
2871       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2872   }
2873 
2874   bool nuw = false;
2875   bool nsw = false;
2876   bool exact = false;
2877 
2878   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2879 
2880     if (const OverflowingBinaryOperator *OFBinOp =
2881             dyn_cast<const OverflowingBinaryOperator>(&I)) {
2882       nuw = OFBinOp->hasNoUnsignedWrap();
2883       nsw = OFBinOp->hasNoSignedWrap();
2884     }
2885     if (const PossiblyExactOperator *ExactOp =
2886             dyn_cast<const PossiblyExactOperator>(&I))
2887       exact = ExactOp->isExact();
2888   }
2889   SDNodeFlags Flags;
2890   Flags.setExact(exact);
2891   Flags.setNoSignedWrap(nsw);
2892   Flags.setNoUnsignedWrap(nuw);
2893   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2894                             Flags);
2895   setValue(&I, Res);
2896 }
2897 
2898 void SelectionDAGBuilder::visitSDiv(const User &I) {
2899   SDValue Op1 = getValue(I.getOperand(0));
2900   SDValue Op2 = getValue(I.getOperand(1));
2901 
2902   SDNodeFlags Flags;
2903   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2904                  cast<PossiblyExactOperator>(&I)->isExact());
2905   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2906                            Op2, Flags));
2907 }
2908 
2909 void SelectionDAGBuilder::visitICmp(const User &I) {
2910   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2911   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2912     predicate = IC->getPredicate();
2913   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2914     predicate = ICmpInst::Predicate(IC->getPredicate());
2915   SDValue Op1 = getValue(I.getOperand(0));
2916   SDValue Op2 = getValue(I.getOperand(1));
2917   ISD::CondCode Opcode = getICmpCondCode(predicate);
2918 
2919   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2920                                                         I.getType());
2921   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2922 }
2923 
2924 void SelectionDAGBuilder::visitFCmp(const User &I) {
2925   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2926   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2927     predicate = FC->getPredicate();
2928   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2929     predicate = FCmpInst::Predicate(FC->getPredicate());
2930   SDValue Op1 = getValue(I.getOperand(0));
2931   SDValue Op2 = getValue(I.getOperand(1));
2932 
2933   ISD::CondCode Condition = getFCmpCondCode(predicate);
2934   auto *FPMO = dyn_cast<FPMathOperator>(&I);
2935   if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2936     Condition = getFCmpCodeWithoutNaN(Condition);
2937 
2938   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2939                                                         I.getType());
2940   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2941 }
2942 
2943 // Check if the condition of the select has one use or two users that are both
2944 // selects with the same condition.
2945 static bool hasOnlySelectUsers(const Value *Cond) {
2946   return llvm::all_of(Cond->users(), [](const Value *V) {
2947     return isa<SelectInst>(V);
2948   });
2949 }
2950 
2951 void SelectionDAGBuilder::visitSelect(const User &I) {
2952   SmallVector<EVT, 4> ValueVTs;
2953   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2954                   ValueVTs);
2955   unsigned NumValues = ValueVTs.size();
2956   if (NumValues == 0) return;
2957 
2958   SmallVector<SDValue, 4> Values(NumValues);
2959   SDValue Cond     = getValue(I.getOperand(0));
2960   SDValue LHSVal   = getValue(I.getOperand(1));
2961   SDValue RHSVal   = getValue(I.getOperand(2));
2962   auto BaseOps = {Cond};
2963   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2964     ISD::VSELECT : ISD::SELECT;
2965 
2966   // Min/max matching is only viable if all output VTs are the same.
2967   if (is_splat(ValueVTs)) {
2968     EVT VT = ValueVTs[0];
2969     LLVMContext &Ctx = *DAG.getContext();
2970     auto &TLI = DAG.getTargetLoweringInfo();
2971 
2972     // We care about the legality of the operation after it has been type
2973     // legalized.
2974     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2975            VT != TLI.getTypeToTransformTo(Ctx, VT))
2976       VT = TLI.getTypeToTransformTo(Ctx, VT);
2977 
2978     // If the vselect is legal, assume we want to leave this as a vector setcc +
2979     // vselect. Otherwise, if this is going to be scalarized, we want to see if
2980     // min/max is legal on the scalar type.
2981     bool UseScalarMinMax = VT.isVector() &&
2982       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2983 
2984     Value *LHS, *RHS;
2985     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2986     ISD::NodeType Opc = ISD::DELETED_NODE;
2987     switch (SPR.Flavor) {
2988     case SPF_UMAX:    Opc = ISD::UMAX; break;
2989     case SPF_UMIN:    Opc = ISD::UMIN; break;
2990     case SPF_SMAX:    Opc = ISD::SMAX; break;
2991     case SPF_SMIN:    Opc = ISD::SMIN; break;
2992     case SPF_FMINNUM:
2993       switch (SPR.NaNBehavior) {
2994       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2995       case SPNB_RETURNS_NAN:   Opc = ISD::FMINIMUM; break;
2996       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2997       case SPNB_RETURNS_ANY: {
2998         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2999           Opc = ISD::FMINNUM;
3000         else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3001           Opc = ISD::FMINIMUM;
3002         else if (UseScalarMinMax)
3003           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3004             ISD::FMINNUM : ISD::FMINIMUM;
3005         break;
3006       }
3007       }
3008       break;
3009     case SPF_FMAXNUM:
3010       switch (SPR.NaNBehavior) {
3011       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3012       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXIMUM; break;
3013       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3014       case SPNB_RETURNS_ANY:
3015 
3016         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3017           Opc = ISD::FMAXNUM;
3018         else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3019           Opc = ISD::FMAXIMUM;
3020         else if (UseScalarMinMax)
3021           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3022             ISD::FMAXNUM : ISD::FMAXIMUM;
3023         break;
3024       }
3025       break;
3026     default: break;
3027     }
3028 
3029     if (Opc != ISD::DELETED_NODE &&
3030         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3031          (UseScalarMinMax &&
3032           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3033         // If the underlying comparison instruction is used by any other
3034         // instruction, the consumed instructions won't be destroyed, so it is
3035         // not profitable to convert to a min/max.
3036         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3037       OpCode = Opc;
3038       LHSVal = getValue(LHS);
3039       RHSVal = getValue(RHS);
3040       BaseOps = {};
3041     }
3042   }
3043 
3044   for (unsigned i = 0; i != NumValues; ++i) {
3045     SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3046     Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3047     Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3048     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3049                             LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3050                             Ops);
3051   }
3052 
3053   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3054                            DAG.getVTList(ValueVTs), Values));
3055 }
3056 
3057 void SelectionDAGBuilder::visitTrunc(const User &I) {
3058   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3059   SDValue N = getValue(I.getOperand(0));
3060   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3061                                                         I.getType());
3062   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3063 }
3064 
3065 void SelectionDAGBuilder::visitZExt(const User &I) {
3066   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3067   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3068   SDValue N = getValue(I.getOperand(0));
3069   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3070                                                         I.getType());
3071   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3072 }
3073 
3074 void SelectionDAGBuilder::visitSExt(const User &I) {
3075   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3076   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3077   SDValue N = getValue(I.getOperand(0));
3078   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3079                                                         I.getType());
3080   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3081 }
3082 
3083 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3084   // FPTrunc is never a no-op cast, no need to check
3085   SDValue N = getValue(I.getOperand(0));
3086   SDLoc dl = getCurSDLoc();
3087   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3088   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3089   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3090                            DAG.getTargetConstant(
3091                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3092 }
3093 
3094 void SelectionDAGBuilder::visitFPExt(const User &I) {
3095   // FPExt is never a no-op cast, no need to check
3096   SDValue N = getValue(I.getOperand(0));
3097   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3098                                                         I.getType());
3099   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3100 }
3101 
3102 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3103   // FPToUI is never a no-op cast, no need to check
3104   SDValue N = getValue(I.getOperand(0));
3105   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3106                                                         I.getType());
3107   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3108 }
3109 
3110 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3111   // FPToSI is never a no-op cast, no need to check
3112   SDValue N = getValue(I.getOperand(0));
3113   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3114                                                         I.getType());
3115   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3116 }
3117 
3118 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3119   // UIToFP is never a no-op cast, no need to check
3120   SDValue N = getValue(I.getOperand(0));
3121   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3122                                                         I.getType());
3123   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3124 }
3125 
3126 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3127   // SIToFP is never a no-op cast, no need to check
3128   SDValue N = getValue(I.getOperand(0));
3129   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3130                                                         I.getType());
3131   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3132 }
3133 
3134 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3135   // What to do depends on the size of the integer and the size of the pointer.
3136   // We can either truncate, zero extend, or no-op, accordingly.
3137   SDValue N = getValue(I.getOperand(0));
3138   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3139                                                         I.getType());
3140   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3141 }
3142 
3143 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3144   // What to do depends on the size of the integer and the size of the pointer.
3145   // We can either truncate, zero extend, or no-op, accordingly.
3146   SDValue N = getValue(I.getOperand(0));
3147   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3148                                                         I.getType());
3149   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3150 }
3151 
3152 void SelectionDAGBuilder::visitBitCast(const User &I) {
3153   SDValue N = getValue(I.getOperand(0));
3154   SDLoc dl = getCurSDLoc();
3155   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3156                                                         I.getType());
3157 
3158   // BitCast assures us that source and destination are the same size so this is
3159   // either a BITCAST or a no-op.
3160   if (DestVT != N.getValueType())
3161     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3162                              DestVT, N)); // convert types.
3163   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3164   // might fold any kind of constant expression to an integer constant and that
3165   // is not what we are looking for. Only recognize a bitcast of a genuine
3166   // constant integer as an opaque constant.
3167   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3168     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3169                                  /*isOpaque*/true));
3170   else
3171     setValue(&I, N);            // noop cast.
3172 }
3173 
3174 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3175   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3176   const Value *SV = I.getOperand(0);
3177   SDValue N = getValue(SV);
3178   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3179 
3180   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3181   unsigned DestAS = I.getType()->getPointerAddressSpace();
3182 
3183   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3184     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3185 
3186   setValue(&I, N);
3187 }
3188 
3189 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3190   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3191   SDValue InVec = getValue(I.getOperand(0));
3192   SDValue InVal = getValue(I.getOperand(1));
3193   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3194                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3195   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3196                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3197                            InVec, InVal, InIdx));
3198 }
3199 
3200 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3201   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3202   SDValue InVec = getValue(I.getOperand(0));
3203   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3204                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3205   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3206                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3207                            InVec, InIdx));
3208 }
3209 
3210 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3211   SDValue Src1 = getValue(I.getOperand(0));
3212   SDValue Src2 = getValue(I.getOperand(1));
3213   SDLoc DL = getCurSDLoc();
3214 
3215   SmallVector<int, 8> Mask;
3216   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3217   unsigned MaskNumElts = Mask.size();
3218 
3219   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3220   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3221   EVT SrcVT = Src1.getValueType();
3222   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3223 
3224   if (SrcNumElts == MaskNumElts) {
3225     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3226     return;
3227   }
3228 
3229   // Normalize the shuffle vector since mask and vector length don't match.
3230   if (SrcNumElts < MaskNumElts) {
3231     // Mask is longer than the source vectors. We can use concatenate vector to
3232     // make the mask and vectors lengths match.
3233 
3234     if (MaskNumElts % SrcNumElts == 0) {
3235       // Mask length is a multiple of the source vector length.
3236       // Check if the shuffle is some kind of concatenation of the input
3237       // vectors.
3238       unsigned NumConcat = MaskNumElts / SrcNumElts;
3239       bool IsConcat = true;
3240       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3241       for (unsigned i = 0; i != MaskNumElts; ++i) {
3242         int Idx = Mask[i];
3243         if (Idx < 0)
3244           continue;
3245         // Ensure the indices in each SrcVT sized piece are sequential and that
3246         // the same source is used for the whole piece.
3247         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3248             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3249              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3250           IsConcat = false;
3251           break;
3252         }
3253         // Remember which source this index came from.
3254         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3255       }
3256 
3257       // The shuffle is concatenating multiple vectors together. Just emit
3258       // a CONCAT_VECTORS operation.
3259       if (IsConcat) {
3260         SmallVector<SDValue, 8> ConcatOps;
3261         for (auto Src : ConcatSrcs) {
3262           if (Src < 0)
3263             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3264           else if (Src == 0)
3265             ConcatOps.push_back(Src1);
3266           else
3267             ConcatOps.push_back(Src2);
3268         }
3269         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3270         return;
3271       }
3272     }
3273 
3274     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3275     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3276     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3277                                     PaddedMaskNumElts);
3278 
3279     // Pad both vectors with undefs to make them the same length as the mask.
3280     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3281 
3282     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3283     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3284     MOps1[0] = Src1;
3285     MOps2[0] = Src2;
3286 
3287     Src1 = Src1.isUndef()
3288                ? DAG.getUNDEF(PaddedVT)
3289                : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3290     Src2 = Src2.isUndef()
3291                ? DAG.getUNDEF(PaddedVT)
3292                : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3293 
3294     // Readjust mask for new input vector length.
3295     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3296     for (unsigned i = 0; i != MaskNumElts; ++i) {
3297       int Idx = Mask[i];
3298       if (Idx >= (int)SrcNumElts)
3299         Idx -= SrcNumElts - PaddedMaskNumElts;
3300       MappedOps[i] = Idx;
3301     }
3302 
3303     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3304 
3305     // If the concatenated vector was padded, extract a subvector with the
3306     // correct number of elements.
3307     if (MaskNumElts != PaddedMaskNumElts)
3308       Result = DAG.getNode(
3309           ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3310           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3311 
3312     setValue(&I, Result);
3313     return;
3314   }
3315 
3316   if (SrcNumElts > MaskNumElts) {
3317     // Analyze the access pattern of the vector to see if we can extract
3318     // two subvectors and do the shuffle.
3319     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3320     bool CanExtract = true;
3321     for (int Idx : Mask) {
3322       unsigned Input = 0;
3323       if (Idx < 0)
3324         continue;
3325 
3326       if (Idx >= (int)SrcNumElts) {
3327         Input = 1;
3328         Idx -= SrcNumElts;
3329       }
3330 
3331       // If all the indices come from the same MaskNumElts sized portion of
3332       // the sources we can use extract. Also make sure the extract wouldn't
3333       // extract past the end of the source.
3334       int NewStartIdx = alignDown(Idx, MaskNumElts);
3335       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3336           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3337         CanExtract = false;
3338       // Make sure we always update StartIdx as we use it to track if all
3339       // elements are undef.
3340       StartIdx[Input] = NewStartIdx;
3341     }
3342 
3343     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3344       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3345       return;
3346     }
3347     if (CanExtract) {
3348       // Extract appropriate subvector and generate a vector shuffle
3349       for (unsigned Input = 0; Input < 2; ++Input) {
3350         SDValue &Src = Input == 0 ? Src1 : Src2;
3351         if (StartIdx[Input] < 0)
3352           Src = DAG.getUNDEF(VT);
3353         else {
3354           Src = DAG.getNode(
3355               ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3356               DAG.getConstant(StartIdx[Input], DL,
3357                               TLI.getVectorIdxTy(DAG.getDataLayout())));
3358         }
3359       }
3360 
3361       // Calculate new mask.
3362       SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3363       for (int &Idx : MappedOps) {
3364         if (Idx >= (int)SrcNumElts)
3365           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3366         else if (Idx >= 0)
3367           Idx -= StartIdx[0];
3368       }
3369 
3370       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3371       return;
3372     }
3373   }
3374 
3375   // We can't use either concat vectors or extract subvectors so fall back to
3376   // replacing the shuffle with extract and build vector.
3377   // to insert and build vector.
3378   EVT EltVT = VT.getVectorElementType();
3379   EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3380   SmallVector<SDValue,8> Ops;
3381   for (int Idx : Mask) {
3382     SDValue Res;
3383 
3384     if (Idx < 0) {
3385       Res = DAG.getUNDEF(EltVT);
3386     } else {
3387       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3388       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3389 
3390       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3391                         EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3392     }
3393 
3394     Ops.push_back(Res);
3395   }
3396 
3397   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3398 }
3399 
3400 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3401   ArrayRef<unsigned> Indices;
3402   if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3403     Indices = IV->getIndices();
3404   else
3405     Indices = cast<ConstantExpr>(&I)->getIndices();
3406 
3407   const Value *Op0 = I.getOperand(0);
3408   const Value *Op1 = I.getOperand(1);
3409   Type *AggTy = I.getType();
3410   Type *ValTy = Op1->getType();
3411   bool IntoUndef = isa<UndefValue>(Op0);
3412   bool FromUndef = isa<UndefValue>(Op1);
3413 
3414   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3415 
3416   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3417   SmallVector<EVT, 4> AggValueVTs;
3418   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3419   SmallVector<EVT, 4> ValValueVTs;
3420   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3421 
3422   unsigned NumAggValues = AggValueVTs.size();
3423   unsigned NumValValues = ValValueVTs.size();
3424   SmallVector<SDValue, 4> Values(NumAggValues);
3425 
3426   // Ignore an insertvalue that produces an empty object
3427   if (!NumAggValues) {
3428     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3429     return;
3430   }
3431 
3432   SDValue Agg = getValue(Op0);
3433   unsigned i = 0;
3434   // Copy the beginning value(s) from the original aggregate.
3435   for (; i != LinearIndex; ++i)
3436     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3437                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3438   // Copy values from the inserted value(s).
3439   if (NumValValues) {
3440     SDValue Val = getValue(Op1);
3441     for (; i != LinearIndex + NumValValues; ++i)
3442       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3443                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3444   }
3445   // Copy remaining value(s) from the original aggregate.
3446   for (; i != NumAggValues; ++i)
3447     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3448                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3449 
3450   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3451                            DAG.getVTList(AggValueVTs), Values));
3452 }
3453 
3454 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3455   ArrayRef<unsigned> Indices;
3456   if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3457     Indices = EV->getIndices();
3458   else
3459     Indices = cast<ConstantExpr>(&I)->getIndices();
3460 
3461   const Value *Op0 = I.getOperand(0);
3462   Type *AggTy = Op0->getType();
3463   Type *ValTy = I.getType();
3464   bool OutOfUndef = isa<UndefValue>(Op0);
3465 
3466   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3467 
3468   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3469   SmallVector<EVT, 4> ValValueVTs;
3470   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3471 
3472   unsigned NumValValues = ValValueVTs.size();
3473 
3474   // Ignore a extractvalue that produces an empty object
3475   if (!NumValValues) {
3476     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3477     return;
3478   }
3479 
3480   SmallVector<SDValue, 4> Values(NumValValues);
3481 
3482   SDValue Agg = getValue(Op0);
3483   // Copy out the selected value(s).
3484   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3485     Values[i - LinearIndex] =
3486       OutOfUndef ?
3487         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3488         SDValue(Agg.getNode(), Agg.getResNo() + i);
3489 
3490   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3491                            DAG.getVTList(ValValueVTs), Values));
3492 }
3493 
3494 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3495   Value *Op0 = I.getOperand(0);
3496   // Note that the pointer operand may be a vector of pointers. Take the scalar
3497   // element which holds a pointer.
3498   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3499   SDValue N = getValue(Op0);
3500   SDLoc dl = getCurSDLoc();
3501 
3502   // Normalize Vector GEP - all scalar operands should be converted to the
3503   // splat vector.
3504   unsigned VectorWidth = I.getType()->isVectorTy() ?
3505     cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3506 
3507   if (VectorWidth && !N.getValueType().isVector()) {
3508     LLVMContext &Context = *DAG.getContext();
3509     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3510     N = DAG.getSplatBuildVector(VT, dl, N);
3511   }
3512 
3513   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3514        GTI != E; ++GTI) {
3515     const Value *Idx = GTI.getOperand();
3516     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3517       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3518       if (Field) {
3519         // N = N + Offset
3520         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3521 
3522         // In an inbounds GEP with an offset that is nonnegative even when
3523         // interpreted as signed, assume there is no unsigned overflow.
3524         SDNodeFlags Flags;
3525         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3526           Flags.setNoUnsignedWrap(true);
3527 
3528         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3529                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3530       }
3531     } else {
3532       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3533       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3534       APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3535 
3536       // If this is a scalar constant or a splat vector of constants,
3537       // handle it quickly.
3538       const auto *CI = dyn_cast<ConstantInt>(Idx);
3539       if (!CI && isa<ConstantDataVector>(Idx) &&
3540           cast<ConstantDataVector>(Idx)->getSplatValue())
3541         CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3542 
3543       if (CI) {
3544         if (CI->isZero())
3545           continue;
3546         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3547         LLVMContext &Context = *DAG.getContext();
3548         SDValue OffsVal = VectorWidth ?
3549           DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3550           DAG.getConstant(Offs, dl, IdxTy);
3551 
3552         // In an inbouds GEP with an offset that is nonnegative even when
3553         // interpreted as signed, assume there is no unsigned overflow.
3554         SDNodeFlags Flags;
3555         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3556           Flags.setNoUnsignedWrap(true);
3557 
3558         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3559         continue;
3560       }
3561 
3562       // N = N + Idx * ElementSize;
3563       SDValue IdxN = getValue(Idx);
3564 
3565       if (!IdxN.getValueType().isVector() && VectorWidth) {
3566         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3567         IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3568       }
3569 
3570       // If the index is smaller or larger than intptr_t, truncate or extend
3571       // it.
3572       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3573 
3574       // If this is a multiply by a power of two, turn it into a shl
3575       // immediately.  This is a very common case.
3576       if (ElementSize != 1) {
3577         if (ElementSize.isPowerOf2()) {
3578           unsigned Amt = ElementSize.logBase2();
3579           IdxN = DAG.getNode(ISD::SHL, dl,
3580                              N.getValueType(), IdxN,
3581                              DAG.getConstant(Amt, dl, IdxN.getValueType()));
3582         } else {
3583           SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3584           IdxN = DAG.getNode(ISD::MUL, dl,
3585                              N.getValueType(), IdxN, Scale);
3586         }
3587       }
3588 
3589       N = DAG.getNode(ISD::ADD, dl,
3590                       N.getValueType(), N, IdxN);
3591     }
3592   }
3593 
3594   setValue(&I, N);
3595 }
3596 
3597 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3598   // If this is a fixed sized alloca in the entry block of the function,
3599   // allocate it statically on the stack.
3600   if (FuncInfo.StaticAllocaMap.count(&I))
3601     return;   // getValue will auto-populate this.
3602 
3603   SDLoc dl = getCurSDLoc();
3604   Type *Ty = I.getAllocatedType();
3605   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3606   auto &DL = DAG.getDataLayout();
3607   uint64_t TySize = DL.getTypeAllocSize(Ty);
3608   unsigned Align =
3609       std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3610 
3611   SDValue AllocSize = getValue(I.getArraySize());
3612 
3613   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3614   if (AllocSize.getValueType() != IntPtr)
3615     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3616 
3617   AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3618                           AllocSize,
3619                           DAG.getConstant(TySize, dl, IntPtr));
3620 
3621   // Handle alignment.  If the requested alignment is less than or equal to
3622   // the stack alignment, ignore it.  If the size is greater than or equal to
3623   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3624   unsigned StackAlign =
3625       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3626   if (Align <= StackAlign)
3627     Align = 0;
3628 
3629   // Round the size of the allocation up to the stack alignment size
3630   // by add SA-1 to the size. This doesn't overflow because we're computing
3631   // an address inside an alloca.
3632   SDNodeFlags Flags;
3633   Flags.setNoUnsignedWrap(true);
3634   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3635                           DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3636 
3637   // Mask out the low bits for alignment purposes.
3638   AllocSize =
3639       DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3640                   DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3641 
3642   SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3643   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3644   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3645   setValue(&I, DSA);
3646   DAG.setRoot(DSA.getValue(1));
3647 
3648   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3649 }
3650 
3651 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3652   if (I.isAtomic())
3653     return visitAtomicLoad(I);
3654 
3655   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3656   const Value *SV = I.getOperand(0);
3657   if (TLI.supportSwiftError()) {
3658     // Swifterror values can come from either a function parameter with
3659     // swifterror attribute or an alloca with swifterror attribute.
3660     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3661       if (Arg->hasSwiftErrorAttr())
3662         return visitLoadFromSwiftError(I);
3663     }
3664 
3665     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3666       if (Alloca->isSwiftError())
3667         return visitLoadFromSwiftError(I);
3668     }
3669   }
3670 
3671   SDValue Ptr = getValue(SV);
3672 
3673   Type *Ty = I.getType();
3674 
3675   bool isVolatile = I.isVolatile();
3676   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3677   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3678   bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3679   unsigned Alignment = I.getAlignment();
3680 
3681   AAMDNodes AAInfo;
3682   I.getAAMetadata(AAInfo);
3683   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3684 
3685   SmallVector<EVT, 4> ValueVTs;
3686   SmallVector<uint64_t, 4> Offsets;
3687   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3688   unsigned NumValues = ValueVTs.size();
3689   if (NumValues == 0)
3690     return;
3691 
3692   SDValue Root;
3693   bool ConstantMemory = false;
3694   if (isVolatile || NumValues > MaxParallelChains)
3695     // Serialize volatile loads with other side effects.
3696     Root = getRoot();
3697   else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3698                SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3699     // Do not serialize (non-volatile) loads of constant memory with anything.
3700     Root = DAG.getEntryNode();
3701     ConstantMemory = true;
3702   } else {
3703     // Do not serialize non-volatile loads against each other.
3704     Root = DAG.getRoot();
3705   }
3706 
3707   SDLoc dl = getCurSDLoc();
3708 
3709   if (isVolatile)
3710     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3711 
3712   // An aggregate load cannot wrap around the address space, so offsets to its
3713   // parts don't wrap either.
3714   SDNodeFlags Flags;
3715   Flags.setNoUnsignedWrap(true);
3716 
3717   SmallVector<SDValue, 4> Values(NumValues);
3718   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3719   EVT PtrVT = Ptr.getValueType();
3720   unsigned ChainI = 0;
3721   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3722     // Serializing loads here may result in excessive register pressure, and
3723     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3724     // could recover a bit by hoisting nodes upward in the chain by recognizing
3725     // they are side-effect free or do not alias. The optimizer should really
3726     // avoid this case by converting large object/array copies to llvm.memcpy
3727     // (MaxParallelChains should always remain as failsafe).
3728     if (ChainI == MaxParallelChains) {
3729       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3730       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3731                                   makeArrayRef(Chains.data(), ChainI));
3732       Root = Chain;
3733       ChainI = 0;
3734     }
3735     SDValue A = DAG.getNode(ISD::ADD, dl,
3736                             PtrVT, Ptr,
3737                             DAG.getConstant(Offsets[i], dl, PtrVT),
3738                             Flags);
3739     auto MMOFlags = MachineMemOperand::MONone;
3740     if (isVolatile)
3741       MMOFlags |= MachineMemOperand::MOVolatile;
3742     if (isNonTemporal)
3743       MMOFlags |= MachineMemOperand::MONonTemporal;
3744     if (isInvariant)
3745       MMOFlags |= MachineMemOperand::MOInvariant;
3746     if (isDereferenceable)
3747       MMOFlags |= MachineMemOperand::MODereferenceable;
3748     MMOFlags |= TLI.getMMOFlags(I);
3749 
3750     SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3751                             MachinePointerInfo(SV, Offsets[i]), Alignment,
3752                             MMOFlags, AAInfo, Ranges);
3753 
3754     Values[i] = L;
3755     Chains[ChainI] = L.getValue(1);
3756   }
3757 
3758   if (!ConstantMemory) {
3759     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3760                                 makeArrayRef(Chains.data(), ChainI));
3761     if (isVolatile)
3762       DAG.setRoot(Chain);
3763     else
3764       PendingLoads.push_back(Chain);
3765   }
3766 
3767   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3768                            DAG.getVTList(ValueVTs), Values));
3769 }
3770 
3771 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3772   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3773          "call visitStoreToSwiftError when backend supports swifterror");
3774 
3775   SmallVector<EVT, 4> ValueVTs;
3776   SmallVector<uint64_t, 4> Offsets;
3777   const Value *SrcV = I.getOperand(0);
3778   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3779                   SrcV->getType(), ValueVTs, &Offsets);
3780   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3781          "expect a single EVT for swifterror");
3782 
3783   SDValue Src = getValue(SrcV);
3784   // Create a virtual register, then update the virtual register.
3785   unsigned VReg; bool CreatedVReg;
3786   std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3787   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3788   // Chain can be getRoot or getControlRoot.
3789   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3790                                       SDValue(Src.getNode(), Src.getResNo()));
3791   DAG.setRoot(CopyNode);
3792   if (CreatedVReg)
3793     FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3794 }
3795 
3796 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3797   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3798          "call visitLoadFromSwiftError when backend supports swifterror");
3799 
3800   assert(!I.isVolatile() &&
3801          I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3802          I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
3803          "Support volatile, non temporal, invariant for load_from_swift_error");
3804 
3805   const Value *SV = I.getOperand(0);
3806   Type *Ty = I.getType();
3807   AAMDNodes AAInfo;
3808   I.getAAMetadata(AAInfo);
3809   assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
3810              SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
3811          "load_from_swift_error should not be constant memory");
3812 
3813   SmallVector<EVT, 4> ValueVTs;
3814   SmallVector<uint64_t, 4> Offsets;
3815   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3816                   ValueVTs, &Offsets);
3817   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3818          "expect a single EVT for swifterror");
3819 
3820   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3821   SDValue L = DAG.getCopyFromReg(
3822       getRoot(), getCurSDLoc(),
3823       FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3824       ValueVTs[0]);
3825 
3826   setValue(&I, L);
3827 }
3828 
3829 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3830   if (I.isAtomic())
3831     return visitAtomicStore(I);
3832 
3833   const Value *SrcV = I.getOperand(0);
3834   const Value *PtrV = I.getOperand(1);
3835 
3836   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3837   if (TLI.supportSwiftError()) {
3838     // Swifterror values can come from either a function parameter with
3839     // swifterror attribute or an alloca with swifterror attribute.
3840     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3841       if (Arg->hasSwiftErrorAttr())
3842         return visitStoreToSwiftError(I);
3843     }
3844 
3845     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3846       if (Alloca->isSwiftError())
3847         return visitStoreToSwiftError(I);
3848     }
3849   }
3850 
3851   SmallVector<EVT, 4> ValueVTs;
3852   SmallVector<uint64_t, 4> Offsets;
3853   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3854                   SrcV->getType(), ValueVTs, &Offsets);
3855   unsigned NumValues = ValueVTs.size();
3856   if (NumValues == 0)
3857     return;
3858 
3859   // Get the lowered operands. Note that we do this after
3860   // checking if NumResults is zero, because with zero results
3861   // the operands won't have values in the map.
3862   SDValue Src = getValue(SrcV);
3863   SDValue Ptr = getValue(PtrV);
3864 
3865   SDValue Root = getRoot();
3866   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3867   SDLoc dl = getCurSDLoc();
3868   EVT PtrVT = Ptr.getValueType();
3869   unsigned Alignment = I.getAlignment();
3870   AAMDNodes AAInfo;
3871   I.getAAMetadata(AAInfo);
3872 
3873   auto MMOFlags = MachineMemOperand::MONone;
3874   if (I.isVolatile())
3875     MMOFlags |= MachineMemOperand::MOVolatile;
3876   if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3877     MMOFlags |= MachineMemOperand::MONonTemporal;
3878   MMOFlags |= TLI.getMMOFlags(I);
3879 
3880   // An aggregate load cannot wrap around the address space, so offsets to its
3881   // parts don't wrap either.
3882   SDNodeFlags Flags;
3883   Flags.setNoUnsignedWrap(true);
3884 
3885   unsigned ChainI = 0;
3886   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3887     // See visitLoad comments.
3888     if (ChainI == MaxParallelChains) {
3889       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3890                                   makeArrayRef(Chains.data(), ChainI));
3891       Root = Chain;
3892       ChainI = 0;
3893     }
3894     SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3895                               DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3896     SDValue St = DAG.getStore(
3897         Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3898         MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3899     Chains[ChainI] = St;
3900   }
3901 
3902   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3903                                   makeArrayRef(Chains.data(), ChainI));
3904   DAG.setRoot(StoreNode);
3905 }
3906 
3907 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3908                                            bool IsCompressing) {
3909   SDLoc sdl = getCurSDLoc();
3910 
3911   auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3912                            unsigned& Alignment) {
3913     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3914     Src0 = I.getArgOperand(0);
3915     Ptr = I.getArgOperand(1);
3916     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3917     Mask = I.getArgOperand(3);
3918   };
3919   auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3920                            unsigned& Alignment) {
3921     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3922     Src0 = I.getArgOperand(0);
3923     Ptr = I.getArgOperand(1);
3924     Mask = I.getArgOperand(2);
3925     Alignment = 0;
3926   };
3927 
3928   Value  *PtrOperand, *MaskOperand, *Src0Operand;
3929   unsigned Alignment;
3930   if (IsCompressing)
3931     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3932   else
3933     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3934 
3935   SDValue Ptr = getValue(PtrOperand);
3936   SDValue Src0 = getValue(Src0Operand);
3937   SDValue Mask = getValue(MaskOperand);
3938 
3939   EVT VT = Src0.getValueType();
3940   if (!Alignment)
3941     Alignment = DAG.getEVTAlignment(VT);
3942 
3943   AAMDNodes AAInfo;
3944   I.getAAMetadata(AAInfo);
3945 
3946   MachineMemOperand *MMO =
3947     DAG.getMachineFunction().
3948     getMachineMemOperand(MachinePointerInfo(PtrOperand),
3949                           MachineMemOperand::MOStore,  VT.getStoreSize(),
3950                           Alignment, AAInfo);
3951   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3952                                          MMO, false /* Truncating */,
3953                                          IsCompressing);
3954   DAG.setRoot(StoreNode);
3955   setValue(&I, StoreNode);
3956 }
3957 
3958 // Get a uniform base for the Gather/Scatter intrinsic.
3959 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3960 // We try to represent it as a base pointer + vector of indices.
3961 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3962 // The first operand of the GEP may be a single pointer or a vector of pointers
3963 // Example:
3964 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3965 //  or
3966 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
3967 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3968 //
3969 // When the first GEP operand is a single pointer - it is the uniform base we
3970 // are looking for. If first operand of the GEP is a splat vector - we
3971 // extract the splat value and use it as a uniform base.
3972 // In all other cases the function returns 'false'.
3973 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3974                            SDValue &Scale, SelectionDAGBuilder* SDB) {
3975   SelectionDAG& DAG = SDB->DAG;
3976   LLVMContext &Context = *DAG.getContext();
3977 
3978   assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3979   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3980   if (!GEP)
3981     return false;
3982 
3983   const Value *GEPPtr = GEP->getPointerOperand();
3984   if (!GEPPtr->getType()->isVectorTy())
3985     Ptr = GEPPtr;
3986   else if (!(Ptr = getSplatValue(GEPPtr)))
3987     return false;
3988 
3989   unsigned FinalIndex = GEP->getNumOperands() - 1;
3990   Value *IndexVal = GEP->getOperand(FinalIndex);
3991 
3992   // Ensure all the other indices are 0.
3993   for (unsigned i = 1; i < FinalIndex; ++i) {
3994     auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
3995     if (!C || !C->isZero())
3996       return false;
3997   }
3998 
3999   // The operands of the GEP may be defined in another basic block.
4000   // In this case we'll not find nodes for the operands.
4001   if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
4002     return false;
4003 
4004   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4005   const DataLayout &DL = DAG.getDataLayout();
4006   Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4007                                 SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4008   Base = SDB->getValue(Ptr);
4009   Index = SDB->getValue(IndexVal);
4010 
4011   if (!Index.getValueType().isVector()) {
4012     unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4013     EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4014     Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4015   }
4016   return true;
4017 }
4018 
4019 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4020   SDLoc sdl = getCurSDLoc();
4021 
4022   // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4023   const Value *Ptr = I.getArgOperand(1);
4024   SDValue Src0 = getValue(I.getArgOperand(0));
4025   SDValue Mask = getValue(I.getArgOperand(3));
4026   EVT VT = Src0.getValueType();
4027   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4028   if (!Alignment)
4029     Alignment = DAG.getEVTAlignment(VT);
4030   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4031 
4032   AAMDNodes AAInfo;
4033   I.getAAMetadata(AAInfo);
4034 
4035   SDValue Base;
4036   SDValue Index;
4037   SDValue Scale;
4038   const Value *BasePtr = Ptr;
4039   bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4040 
4041   const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4042   MachineMemOperand *MMO = DAG.getMachineFunction().
4043     getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4044                          MachineMemOperand::MOStore,  VT.getStoreSize(),
4045                          Alignment, AAInfo);
4046   if (!UniformBase) {
4047     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4048     Index = getValue(Ptr);
4049     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4050   }
4051   SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4052   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4053                                          Ops, MMO);
4054   DAG.setRoot(Scatter);
4055   setValue(&I, Scatter);
4056 }
4057 
4058 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4059   SDLoc sdl = getCurSDLoc();
4060 
4061   auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4062                            unsigned& Alignment) {
4063     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4064     Ptr = I.getArgOperand(0);
4065     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4066     Mask = I.getArgOperand(2);
4067     Src0 = I.getArgOperand(3);
4068   };
4069   auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4070                            unsigned& Alignment) {
4071     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4072     Ptr = I.getArgOperand(0);
4073     Alignment = 0;
4074     Mask = I.getArgOperand(1);
4075     Src0 = I.getArgOperand(2);
4076   };
4077 
4078   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4079   unsigned Alignment;
4080   if (IsExpanding)
4081     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4082   else
4083     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4084 
4085   SDValue Ptr = getValue(PtrOperand);
4086   SDValue Src0 = getValue(Src0Operand);
4087   SDValue Mask = getValue(MaskOperand);
4088 
4089   EVT VT = Src0.getValueType();
4090   if (!Alignment)
4091     Alignment = DAG.getEVTAlignment(VT);
4092 
4093   AAMDNodes AAInfo;
4094   I.getAAMetadata(AAInfo);
4095   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4096 
4097   // Do not serialize masked loads of constant memory with anything.
4098   bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
4099       PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
4100   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4101 
4102   MachineMemOperand *MMO =
4103     DAG.getMachineFunction().
4104     getMachineMemOperand(MachinePointerInfo(PtrOperand),
4105                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
4106                           Alignment, AAInfo, Ranges);
4107 
4108   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4109                                    ISD::NON_EXTLOAD, IsExpanding);
4110   if (AddToChain)
4111     PendingLoads.push_back(Load.getValue(1));
4112   setValue(&I, Load);
4113 }
4114 
4115 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4116   SDLoc sdl = getCurSDLoc();
4117 
4118   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4119   const Value *Ptr = I.getArgOperand(0);
4120   SDValue Src0 = getValue(I.getArgOperand(3));
4121   SDValue Mask = getValue(I.getArgOperand(2));
4122 
4123   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4124   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4125   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4126   if (!Alignment)
4127     Alignment = DAG.getEVTAlignment(VT);
4128 
4129   AAMDNodes AAInfo;
4130   I.getAAMetadata(AAInfo);
4131   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4132 
4133   SDValue Root = DAG.getRoot();
4134   SDValue Base;
4135   SDValue Index;
4136   SDValue Scale;
4137   const Value *BasePtr = Ptr;
4138   bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4139   bool ConstantMemory = false;
4140   if (UniformBase &&
4141       AA && AA->pointsToConstantMemory(MemoryLocation(
4142           BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
4143           AAInfo))) {
4144     // Do not serialize (non-volatile) loads of constant memory with anything.
4145     Root = DAG.getEntryNode();
4146     ConstantMemory = true;
4147   }
4148 
4149   MachineMemOperand *MMO =
4150     DAG.getMachineFunction().
4151     getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4152                          MachineMemOperand::MOLoad,  VT.getStoreSize(),
4153                          Alignment, AAInfo, Ranges);
4154 
4155   if (!UniformBase) {
4156     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4157     Index = getValue(Ptr);
4158     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4159   }
4160   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4161   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4162                                        Ops, MMO);
4163 
4164   SDValue OutChain = Gather.getValue(1);
4165   if (!ConstantMemory)
4166     PendingLoads.push_back(OutChain);
4167   setValue(&I, Gather);
4168 }
4169 
4170 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4171   SDLoc dl = getCurSDLoc();
4172   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4173   AtomicOrdering FailureOrder = I.getFailureOrdering();
4174   SyncScope::ID SSID = I.getSyncScopeID();
4175 
4176   SDValue InChain = getRoot();
4177 
4178   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4179   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4180   SDValue L = DAG.getAtomicCmpSwap(
4181       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4182       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4183       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
4184       /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4185 
4186   SDValue OutChain = L.getValue(2);
4187 
4188   setValue(&I, L);
4189   DAG.setRoot(OutChain);
4190 }
4191 
4192 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4193   SDLoc dl = getCurSDLoc();
4194   ISD::NodeType NT;
4195   switch (I.getOperation()) {
4196   default: llvm_unreachable("Unknown atomicrmw operation");
4197   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4198   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4199   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4200   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4201   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4202   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4203   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4204   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4205   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4206   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4207   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4208   }
4209   AtomicOrdering Order = I.getOrdering();
4210   SyncScope::ID SSID = I.getSyncScopeID();
4211 
4212   SDValue InChain = getRoot();
4213 
4214   SDValue L =
4215     DAG.getAtomic(NT, dl,
4216                   getValue(I.getValOperand()).getSimpleValueType(),
4217                   InChain,
4218                   getValue(I.getPointerOperand()),
4219                   getValue(I.getValOperand()),
4220                   I.getPointerOperand(),
4221                   /* Alignment=*/ 0, Order, SSID);
4222 
4223   SDValue OutChain = L.getValue(1);
4224 
4225   setValue(&I, L);
4226   DAG.setRoot(OutChain);
4227 }
4228 
4229 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4230   SDLoc dl = getCurSDLoc();
4231   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4232   SDValue Ops[3];
4233   Ops[0] = getRoot();
4234   Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4235                            TLI.getFenceOperandTy(DAG.getDataLayout()));
4236   Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4237                            TLI.getFenceOperandTy(DAG.getDataLayout()));
4238   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4239 }
4240 
4241 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4242   SDLoc dl = getCurSDLoc();
4243   AtomicOrdering Order = I.getOrdering();
4244   SyncScope::ID SSID = I.getSyncScopeID();
4245 
4246   SDValue InChain = getRoot();
4247 
4248   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4249   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4250 
4251   if (!TLI.supportsUnalignedAtomics() &&
4252       I.getAlignment() < VT.getStoreSize())
4253     report_fatal_error("Cannot generate unaligned atomic load");
4254 
4255   MachineMemOperand *MMO =
4256       DAG.getMachineFunction().
4257       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4258                            MachineMemOperand::MOVolatile |
4259                            MachineMemOperand::MOLoad,
4260                            VT.getStoreSize(),
4261                            I.getAlignment() ? I.getAlignment() :
4262                                               DAG.getEVTAlignment(VT),
4263                            AAMDNodes(), nullptr, SSID, Order);
4264 
4265   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4266   SDValue L =
4267       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4268                     getValue(I.getPointerOperand()), MMO);
4269 
4270   SDValue OutChain = L.getValue(1);
4271 
4272   setValue(&I, L);
4273   DAG.setRoot(OutChain);
4274 }
4275 
4276 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4277   SDLoc dl = getCurSDLoc();
4278 
4279   AtomicOrdering Order = I.getOrdering();
4280   SyncScope::ID SSID = I.getSyncScopeID();
4281 
4282   SDValue InChain = getRoot();
4283 
4284   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4285   EVT VT =
4286       TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4287 
4288   if (I.getAlignment() < VT.getStoreSize())
4289     report_fatal_error("Cannot generate unaligned atomic store");
4290 
4291   SDValue OutChain =
4292     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4293                   InChain,
4294                   getValue(I.getPointerOperand()),
4295                   getValue(I.getValueOperand()),
4296                   I.getPointerOperand(), I.getAlignment(),
4297                   Order, SSID);
4298 
4299   DAG.setRoot(OutChain);
4300 }
4301 
4302 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4303 /// node.
4304 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4305                                                unsigned Intrinsic) {
4306   // Ignore the callsite's attributes. A specific call site may be marked with
4307   // readnone, but the lowering code will expect the chain based on the
4308   // definition.
4309   const Function *F = I.getCalledFunction();
4310   bool HasChain = !F->doesNotAccessMemory();
4311   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4312 
4313   // Build the operand list.
4314   SmallVector<SDValue, 8> Ops;
4315   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4316     if (OnlyLoad) {
4317       // We don't need to serialize loads against other loads.
4318       Ops.push_back(DAG.getRoot());
4319     } else {
4320       Ops.push_back(getRoot());
4321     }
4322   }
4323 
4324   // Info is set by getTgtMemInstrinsic
4325   TargetLowering::IntrinsicInfo Info;
4326   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4327   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4328                                                DAG.getMachineFunction(),
4329                                                Intrinsic);
4330 
4331   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4332   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4333       Info.opc == ISD::INTRINSIC_W_CHAIN)
4334     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4335                                         TLI.getPointerTy(DAG.getDataLayout())));
4336 
4337   // Add all operands of the call to the operand list.
4338   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4339     SDValue Op = getValue(I.getArgOperand(i));
4340     Ops.push_back(Op);
4341   }
4342 
4343   SmallVector<EVT, 4> ValueVTs;
4344   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4345 
4346   if (HasChain)
4347     ValueVTs.push_back(MVT::Other);
4348 
4349   SDVTList VTs = DAG.getVTList(ValueVTs);
4350 
4351   // Create the node.
4352   SDValue Result;
4353   if (IsTgtIntrinsic) {
4354     // This is target intrinsic that touches memory
4355     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4356       Ops, Info.memVT,
4357       MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4358       Info.flags, Info.size);
4359   } else if (!HasChain) {
4360     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4361   } else if (!I.getType()->isVoidTy()) {
4362     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4363   } else {
4364     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4365   }
4366 
4367   if (HasChain) {
4368     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4369     if (OnlyLoad)
4370       PendingLoads.push_back(Chain);
4371     else
4372       DAG.setRoot(Chain);
4373   }
4374 
4375   if (!I.getType()->isVoidTy()) {
4376     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4377       EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4378       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4379     } else
4380       Result = lowerRangeToAssertZExt(DAG, I, Result);
4381 
4382     setValue(&I, Result);
4383   }
4384 }
4385 
4386 /// GetSignificand - Get the significand and build it into a floating-point
4387 /// number with exponent of 1:
4388 ///
4389 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4390 ///
4391 /// where Op is the hexadecimal representation of floating point value.
4392 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4393   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4394                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4395   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4396                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4397   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4398 }
4399 
4400 /// GetExponent - Get the exponent:
4401 ///
4402 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4403 ///
4404 /// where Op is the hexadecimal representation of floating point value.
4405 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4406                            const TargetLowering &TLI, const SDLoc &dl) {
4407   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4408                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4409   SDValue t1 = DAG.getNode(
4410       ISD::SRL, dl, MVT::i32, t0,
4411       DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4412   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4413                            DAG.getConstant(127, dl, MVT::i32));
4414   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4415 }
4416 
4417 /// getF32Constant - Get 32-bit floating point constant.
4418 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4419                               const SDLoc &dl) {
4420   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4421                            MVT::f32);
4422 }
4423 
4424 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4425                                        SelectionDAG &DAG) {
4426   // TODO: What fast-math-flags should be set on the floating-point nodes?
4427 
4428   //   IntegerPartOfX = ((int32_t)(t0);
4429   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4430 
4431   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4432   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4433   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4434 
4435   //   IntegerPartOfX <<= 23;
4436   IntegerPartOfX = DAG.getNode(
4437       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4438       DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4439                                   DAG.getDataLayout())));
4440 
4441   SDValue TwoToFractionalPartOfX;
4442   if (LimitFloatPrecision <= 6) {
4443     // For floating-point precision of 6:
4444     //
4445     //   TwoToFractionalPartOfX =
4446     //     0.997535578f +
4447     //       (0.735607626f + 0.252464424f * x) * x;
4448     //
4449     // error 0.0144103317, which is 6 bits
4450     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4451                              getF32Constant(DAG, 0x3e814304, dl));
4452     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4453                              getF32Constant(DAG, 0x3f3c50c8, dl));
4454     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4455     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4456                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
4457   } else if (LimitFloatPrecision <= 12) {
4458     // For floating-point precision of 12:
4459     //
4460     //   TwoToFractionalPartOfX =
4461     //     0.999892986f +
4462     //       (0.696457318f +
4463     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4464     //
4465     // error 0.000107046256, which is 13 to 14 bits
4466     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4467                              getF32Constant(DAG, 0x3da235e3, dl));
4468     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4469                              getF32Constant(DAG, 0x3e65b8f3, dl));
4470     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4471     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4472                              getF32Constant(DAG, 0x3f324b07, dl));
4473     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4474     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4475                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
4476   } else { // LimitFloatPrecision <= 18
4477     // For floating-point precision of 18:
4478     //
4479     //   TwoToFractionalPartOfX =
4480     //     0.999999982f +
4481     //       (0.693148872f +
4482     //         (0.240227044f +
4483     //           (0.554906021e-1f +
4484     //             (0.961591928e-2f +
4485     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4486     // error 2.47208000*10^(-7), which is better than 18 bits
4487     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4488                              getF32Constant(DAG, 0x3924b03e, dl));
4489     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4490                              getF32Constant(DAG, 0x3ab24b87, dl));
4491     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4492     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4493                              getF32Constant(DAG, 0x3c1d8c17, dl));
4494     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4495     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4496                              getF32Constant(DAG, 0x3d634a1d, dl));
4497     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4498     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4499                              getF32Constant(DAG, 0x3e75fe14, dl));
4500     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4501     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4502                               getF32Constant(DAG, 0x3f317234, dl));
4503     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4504     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4505                                          getF32Constant(DAG, 0x3f800000, dl));
4506   }
4507 
4508   // Add the exponent into the result in integer domain.
4509   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4510   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4511                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4512 }
4513 
4514 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4515 /// limited-precision mode.
4516 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4517                          const TargetLowering &TLI) {
4518   if (Op.getValueType() == MVT::f32 &&
4519       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4520 
4521     // Put the exponent in the right bit position for later addition to the
4522     // final result:
4523     //
4524     //   #define LOG2OFe 1.4426950f
4525     //   t0 = Op * LOG2OFe
4526 
4527     // TODO: What fast-math-flags should be set here?
4528     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4529                              getF32Constant(DAG, 0x3fb8aa3b, dl));
4530     return getLimitedPrecisionExp2(t0, dl, DAG);
4531   }
4532 
4533   // No special expansion.
4534   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4535 }
4536 
4537 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4538 /// limited-precision mode.
4539 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4540                          const TargetLowering &TLI) {
4541   // TODO: What fast-math-flags should be set on the floating-point nodes?
4542 
4543   if (Op.getValueType() == MVT::f32 &&
4544       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4545     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4546 
4547     // Scale the exponent by log(2) [0.69314718f].
4548     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4549     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4550                                         getF32Constant(DAG, 0x3f317218, dl));
4551 
4552     // Get the significand and build it into a floating-point number with
4553     // exponent of 1.
4554     SDValue X = GetSignificand(DAG, Op1, dl);
4555 
4556     SDValue LogOfMantissa;
4557     if (LimitFloatPrecision <= 6) {
4558       // For floating-point precision of 6:
4559       //
4560       //   LogofMantissa =
4561       //     -1.1609546f +
4562       //       (1.4034025f - 0.23903021f * x) * x;
4563       //
4564       // error 0.0034276066, which is better than 8 bits
4565       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4566                                getF32Constant(DAG, 0xbe74c456, dl));
4567       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4568                                getF32Constant(DAG, 0x3fb3a2b1, dl));
4569       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4570       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4571                                   getF32Constant(DAG, 0x3f949a29, dl));
4572     } else if (LimitFloatPrecision <= 12) {
4573       // For floating-point precision of 12:
4574       //
4575       //   LogOfMantissa =
4576       //     -1.7417939f +
4577       //       (2.8212026f +
4578       //         (-1.4699568f +
4579       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4580       //
4581       // error 0.000061011436, which is 14 bits
4582       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4583                                getF32Constant(DAG, 0xbd67b6d6, dl));
4584       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4585                                getF32Constant(DAG, 0x3ee4f4b8, dl));
4586       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4587       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4588                                getF32Constant(DAG, 0x3fbc278b, dl));
4589       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4590       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4591                                getF32Constant(DAG, 0x40348e95, dl));
4592       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4593       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4594                                   getF32Constant(DAG, 0x3fdef31a, dl));
4595     } else { // LimitFloatPrecision <= 18
4596       // For floating-point precision of 18:
4597       //
4598       //   LogOfMantissa =
4599       //     -2.1072184f +
4600       //       (4.2372794f +
4601       //         (-3.7029485f +
4602       //           (2.2781945f +
4603       //             (-0.87823314f +
4604       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4605       //
4606       // error 0.0000023660568, which is better than 18 bits
4607       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4608                                getF32Constant(DAG, 0xbc91e5ac, dl));
4609       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4610                                getF32Constant(DAG, 0x3e4350aa, dl));
4611       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4612       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4613                                getF32Constant(DAG, 0x3f60d3e3, dl));
4614       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4615       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4616                                getF32Constant(DAG, 0x4011cdf0, dl));
4617       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4618       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4619                                getF32Constant(DAG, 0x406cfd1c, dl));
4620       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4621       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4622                                getF32Constant(DAG, 0x408797cb, dl));
4623       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4624       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4625                                   getF32Constant(DAG, 0x4006dcab, dl));
4626     }
4627 
4628     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4629   }
4630 
4631   // No special expansion.
4632   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4633 }
4634 
4635 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4636 /// limited-precision mode.
4637 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4638                           const TargetLowering &TLI) {
4639   // TODO: What fast-math-flags should be set on the floating-point nodes?
4640 
4641   if (Op.getValueType() == MVT::f32 &&
4642       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4643     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4644 
4645     // Get the exponent.
4646     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4647 
4648     // Get the significand and build it into a floating-point number with
4649     // exponent of 1.
4650     SDValue X = GetSignificand(DAG, Op1, dl);
4651 
4652     // Different possible minimax approximations of significand in
4653     // floating-point for various degrees of accuracy over [1,2].
4654     SDValue Log2ofMantissa;
4655     if (LimitFloatPrecision <= 6) {
4656       // For floating-point precision of 6:
4657       //
4658       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4659       //
4660       // error 0.0049451742, which is more than 7 bits
4661       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4662                                getF32Constant(DAG, 0xbeb08fe0, dl));
4663       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4664                                getF32Constant(DAG, 0x40019463, dl));
4665       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4666       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4667                                    getF32Constant(DAG, 0x3fd6633d, dl));
4668     } else if (LimitFloatPrecision <= 12) {
4669       // For floating-point precision of 12:
4670       //
4671       //   Log2ofMantissa =
4672       //     -2.51285454f +
4673       //       (4.07009056f +
4674       //         (-2.12067489f +
4675       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4676       //
4677       // error 0.0000876136000, which is better than 13 bits
4678       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4679                                getF32Constant(DAG, 0xbda7262e, dl));
4680       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4681                                getF32Constant(DAG, 0x3f25280b, dl));
4682       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4683       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4684                                getF32Constant(DAG, 0x4007b923, dl));
4685       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4686       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4687                                getF32Constant(DAG, 0x40823e2f, dl));
4688       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4689       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4690                                    getF32Constant(DAG, 0x4020d29c, dl));
4691     } else { // LimitFloatPrecision <= 18
4692       // For floating-point precision of 18:
4693       //
4694       //   Log2ofMantissa =
4695       //     -3.0400495f +
4696       //       (6.1129976f +
4697       //         (-5.3420409f +
4698       //           (3.2865683f +
4699       //             (-1.2669343f +
4700       //               (0.27515199f -
4701       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4702       //
4703       // error 0.0000018516, which is better than 18 bits
4704       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4705                                getF32Constant(DAG, 0xbcd2769e, dl));
4706       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4707                                getF32Constant(DAG, 0x3e8ce0b9, dl));
4708       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4709       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4710                                getF32Constant(DAG, 0x3fa22ae7, dl));
4711       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4712       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4713                                getF32Constant(DAG, 0x40525723, dl));
4714       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4715       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4716                                getF32Constant(DAG, 0x40aaf200, dl));
4717       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4718       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4719                                getF32Constant(DAG, 0x40c39dad, dl));
4720       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4721       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4722                                    getF32Constant(DAG, 0x4042902c, dl));
4723     }
4724 
4725     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4726   }
4727 
4728   // No special expansion.
4729   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4730 }
4731 
4732 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4733 /// limited-precision mode.
4734 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4735                            const TargetLowering &TLI) {
4736   // TODO: What fast-math-flags should be set on the floating-point nodes?
4737 
4738   if (Op.getValueType() == MVT::f32 &&
4739       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4740     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4741 
4742     // Scale the exponent by log10(2) [0.30102999f].
4743     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4744     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4745                                         getF32Constant(DAG, 0x3e9a209a, dl));
4746 
4747     // Get the significand and build it into a floating-point number with
4748     // exponent of 1.
4749     SDValue X = GetSignificand(DAG, Op1, dl);
4750 
4751     SDValue Log10ofMantissa;
4752     if (LimitFloatPrecision <= 6) {
4753       // For floating-point precision of 6:
4754       //
4755       //   Log10ofMantissa =
4756       //     -0.50419619f +
4757       //       (0.60948995f - 0.10380950f * x) * x;
4758       //
4759       // error 0.0014886165, which is 6 bits
4760       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4761                                getF32Constant(DAG, 0xbdd49a13, dl));
4762       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4763                                getF32Constant(DAG, 0x3f1c0789, dl));
4764       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4765       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4766                                     getF32Constant(DAG, 0x3f011300, dl));
4767     } else if (LimitFloatPrecision <= 12) {
4768       // For floating-point precision of 12:
4769       //
4770       //   Log10ofMantissa =
4771       //     -0.64831180f +
4772       //       (0.91751397f +
4773       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4774       //
4775       // error 0.00019228036, which is better than 12 bits
4776       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4777                                getF32Constant(DAG, 0x3d431f31, dl));
4778       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4779                                getF32Constant(DAG, 0x3ea21fb2, dl));
4780       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4781       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4782                                getF32Constant(DAG, 0x3f6ae232, dl));
4783       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4784       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4785                                     getF32Constant(DAG, 0x3f25f7c3, dl));
4786     } else { // LimitFloatPrecision <= 18
4787       // For floating-point precision of 18:
4788       //
4789       //   Log10ofMantissa =
4790       //     -0.84299375f +
4791       //       (1.5327582f +
4792       //         (-1.0688956f +
4793       //           (0.49102474f +
4794       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4795       //
4796       // error 0.0000037995730, which is better than 18 bits
4797       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4798                                getF32Constant(DAG, 0x3c5d51ce, dl));
4799       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4800                                getF32Constant(DAG, 0x3e00685a, dl));
4801       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4802       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4803                                getF32Constant(DAG, 0x3efb6798, dl));
4804       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4805       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4806                                getF32Constant(DAG, 0x3f88d192, dl));
4807       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4808       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4809                                getF32Constant(DAG, 0x3fc4316c, dl));
4810       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4811       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4812                                     getF32Constant(DAG, 0x3f57ce70, dl));
4813     }
4814 
4815     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4816   }
4817 
4818   // No special expansion.
4819   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4820 }
4821 
4822 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4823 /// limited-precision mode.
4824 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4825                           const TargetLowering &TLI) {
4826   if (Op.getValueType() == MVT::f32 &&
4827       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4828     return getLimitedPrecisionExp2(Op, dl, DAG);
4829 
4830   // No special expansion.
4831   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4832 }
4833 
4834 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4835 /// limited-precision mode with x == 10.0f.
4836 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4837                          SelectionDAG &DAG, const TargetLowering &TLI) {
4838   bool IsExp10 = false;
4839   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4840       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4841     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4842       APFloat Ten(10.0f);
4843       IsExp10 = LHSC->isExactlyValue(Ten);
4844     }
4845   }
4846 
4847   // TODO: What fast-math-flags should be set on the FMUL node?
4848   if (IsExp10) {
4849     // Put the exponent in the right bit position for later addition to the
4850     // final result:
4851     //
4852     //   #define LOG2OF10 3.3219281f
4853     //   t0 = Op * LOG2OF10;
4854     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4855                              getF32Constant(DAG, 0x40549a78, dl));
4856     return getLimitedPrecisionExp2(t0, dl, DAG);
4857   }
4858 
4859   // No special expansion.
4860   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4861 }
4862 
4863 /// ExpandPowI - Expand a llvm.powi intrinsic.
4864 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4865                           SelectionDAG &DAG) {
4866   // If RHS is a constant, we can expand this out to a multiplication tree,
4867   // otherwise we end up lowering to a call to __powidf2 (for example).  When
4868   // optimizing for size, we only want to do this if the expansion would produce
4869   // a small number of multiplies, otherwise we do the full expansion.
4870   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4871     // Get the exponent as a positive value.
4872     unsigned Val = RHSC->getSExtValue();
4873     if ((int)Val < 0) Val = -Val;
4874 
4875     // powi(x, 0) -> 1.0
4876     if (Val == 0)
4877       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4878 
4879     const Function &F = DAG.getMachineFunction().getFunction();
4880     if (!F.optForSize() ||
4881         // If optimizing for size, don't insert too many multiplies.
4882         // This inserts up to 5 multiplies.
4883         countPopulation(Val) + Log2_32(Val) < 7) {
4884       // We use the simple binary decomposition method to generate the multiply
4885       // sequence.  There are more optimal ways to do this (for example,
4886       // powi(x,15) generates one more multiply than it should), but this has
4887       // the benefit of being both really simple and much better than a libcall.
4888       SDValue Res;  // Logically starts equal to 1.0
4889       SDValue CurSquare = LHS;
4890       // TODO: Intrinsics should have fast-math-flags that propagate to these
4891       // nodes.
4892       while (Val) {
4893         if (Val & 1) {
4894           if (Res.getNode())
4895             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4896           else
4897             Res = CurSquare;  // 1.0*CurSquare.
4898         }
4899 
4900         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4901                                 CurSquare, CurSquare);
4902         Val >>= 1;
4903       }
4904 
4905       // If the original was negative, invert the result, producing 1/(x*x*x).
4906       if (RHSC->getSExtValue() < 0)
4907         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4908                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4909       return Res;
4910     }
4911   }
4912 
4913   // Otherwise, expand to a libcall.
4914   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4915 }
4916 
4917 // getUnderlyingArgReg - Find underlying register used for a truncated or
4918 // bitcasted argument.
4919 static unsigned getUnderlyingArgReg(const SDValue &N) {
4920   switch (N.getOpcode()) {
4921   case ISD::CopyFromReg:
4922     return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4923   case ISD::BITCAST:
4924   case ISD::AssertZext:
4925   case ISD::AssertSext:
4926   case ISD::TRUNCATE:
4927     return getUnderlyingArgReg(N.getOperand(0));
4928   default:
4929     return 0;
4930   }
4931 }
4932 
4933 /// If the DbgValueInst is a dbg_value of a function argument, create the
4934 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
4935 /// instruction selection, they will be inserted to the entry BB.
4936 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4937     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4938     DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4939   const Argument *Arg = dyn_cast<Argument>(V);
4940   if (!Arg)
4941     return false;
4942 
4943   MachineFunction &MF = DAG.getMachineFunction();
4944   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4945 
4946   bool IsIndirect = false;
4947   Optional<MachineOperand> Op;
4948   // Some arguments' frame index is recorded during argument lowering.
4949   int FI = FuncInfo.getArgumentFrameIndex(Arg);
4950   if (FI != std::numeric_limits<int>::max())
4951     Op = MachineOperand::CreateFI(FI);
4952 
4953   if (!Op && N.getNode()) {
4954     unsigned Reg = getUnderlyingArgReg(N);
4955     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4956       MachineRegisterInfo &RegInfo = MF.getRegInfo();
4957       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4958       if (PR)
4959         Reg = PR;
4960     }
4961     if (Reg) {
4962       Op = MachineOperand::CreateReg(Reg, false);
4963       IsIndirect = IsDbgDeclare;
4964     }
4965   }
4966 
4967   if (!Op && N.getNode())
4968     // Check if frame index is available.
4969     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4970       if (FrameIndexSDNode *FINode =
4971           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4972         Op = MachineOperand::CreateFI(FINode->getIndex());
4973 
4974   if (!Op) {
4975     // Check if ValueMap has reg number.
4976     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4977     if (VMI != FuncInfo.ValueMap.end()) {
4978       const auto &TLI = DAG.getTargetLoweringInfo();
4979       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4980                        V->getType(), getABIRegCopyCC(V));
4981       if (RFV.occupiesMultipleRegs()) {
4982         unsigned Offset = 0;
4983         for (auto RegAndSize : RFV.getRegsAndSizes()) {
4984           Op = MachineOperand::CreateReg(RegAndSize.first, false);
4985           auto FragmentExpr = DIExpression::createFragmentExpression(
4986               Expr, Offset, RegAndSize.second);
4987           if (!FragmentExpr)
4988             continue;
4989           FuncInfo.ArgDbgValues.push_back(
4990               BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
4991                       Op->getReg(), Variable, *FragmentExpr));
4992           Offset += RegAndSize.second;
4993         }
4994         return true;
4995       }
4996       Op = MachineOperand::CreateReg(VMI->second, false);
4997       IsIndirect = IsDbgDeclare;
4998     }
4999   }
5000 
5001   if (!Op)
5002     return false;
5003 
5004   assert(Variable->isValidLocationForIntrinsic(DL) &&
5005          "Expected inlined-at fields to agree");
5006   IsIndirect = (Op->isReg()) ? IsIndirect : true;
5007   FuncInfo.ArgDbgValues.push_back(
5008       BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
5009               *Op, Variable, Expr));
5010 
5011   return true;
5012 }
5013 
5014 /// Return the appropriate SDDbgValue based on N.
5015 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5016                                              DILocalVariable *Variable,
5017                                              DIExpression *Expr,
5018                                              const DebugLoc &dl,
5019                                              unsigned DbgSDNodeOrder) {
5020   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5021     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5022     // stack slot locations.
5023     //
5024     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5025     // debug values here after optimization:
5026     //
5027     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5028     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5029     //
5030     // Both describe the direct values of their associated variables.
5031     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5032                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5033   }
5034   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5035                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5036 }
5037 
5038 // VisualStudio defines setjmp as _setjmp
5039 #if defined(_MSC_VER) && defined(setjmp) && \
5040                          !defined(setjmp_undefined_for_msvc)
5041 #  pragma push_macro("setjmp")
5042 #  undef setjmp
5043 #  define setjmp_undefined_for_msvc
5044 #endif
5045 
5046 /// Lower the call to the specified intrinsic function. If we want to emit this
5047 /// as a call to a named external function, return the name. Otherwise, lower it
5048 /// and return null.
5049 const char *
5050 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
5051   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5052   SDLoc sdl = getCurSDLoc();
5053   DebugLoc dl = getCurDebugLoc();
5054   SDValue Res;
5055 
5056   switch (Intrinsic) {
5057   default:
5058     // By default, turn this into a target intrinsic node.
5059     visitTargetIntrinsic(I, Intrinsic);
5060     return nullptr;
5061   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
5062   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
5063   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
5064   case Intrinsic::returnaddress:
5065     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5066                              TLI.getPointerTy(DAG.getDataLayout()),
5067                              getValue(I.getArgOperand(0))));
5068     return nullptr;
5069   case Intrinsic::addressofreturnaddress:
5070     setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5071                              TLI.getPointerTy(DAG.getDataLayout())));
5072     return nullptr;
5073   case Intrinsic::sponentry:
5074     setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl,
5075                              TLI.getPointerTy(DAG.getDataLayout())));
5076     return nullptr;
5077   case Intrinsic::frameaddress:
5078     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5079                              TLI.getPointerTy(DAG.getDataLayout()),
5080                              getValue(I.getArgOperand(0))));
5081     return nullptr;
5082   case Intrinsic::read_register: {
5083     Value *Reg = I.getArgOperand(0);
5084     SDValue Chain = getRoot();
5085     SDValue RegName =
5086         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5087     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5088     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5089       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5090     setValue(&I, Res);
5091     DAG.setRoot(Res.getValue(1));
5092     return nullptr;
5093   }
5094   case Intrinsic::write_register: {
5095     Value *Reg = I.getArgOperand(0);
5096     Value *RegValue = I.getArgOperand(1);
5097     SDValue Chain = getRoot();
5098     SDValue RegName =
5099         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5100     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5101                             RegName, getValue(RegValue)));
5102     return nullptr;
5103   }
5104   case Intrinsic::setjmp:
5105     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
5106   case Intrinsic::longjmp:
5107     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
5108   case Intrinsic::memcpy: {
5109     const auto &MCI = cast<MemCpyInst>(I);
5110     SDValue Op1 = getValue(I.getArgOperand(0));
5111     SDValue Op2 = getValue(I.getArgOperand(1));
5112     SDValue Op3 = getValue(I.getArgOperand(2));
5113     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5114     unsigned DstAlign = std::max<unsigned>(MCI.getDestAlignment(), 1);
5115     unsigned SrcAlign = std::max<unsigned>(MCI.getSourceAlignment(), 1);
5116     unsigned Align = MinAlign(DstAlign, SrcAlign);
5117     bool isVol = MCI.isVolatile();
5118     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5119     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5120     // node.
5121     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5122                                false, isTC,
5123                                MachinePointerInfo(I.getArgOperand(0)),
5124                                MachinePointerInfo(I.getArgOperand(1)));
5125     updateDAGForMaybeTailCall(MC);
5126     return nullptr;
5127   }
5128   case Intrinsic::memset: {
5129     const auto &MSI = cast<MemSetInst>(I);
5130     SDValue Op1 = getValue(I.getArgOperand(0));
5131     SDValue Op2 = getValue(I.getArgOperand(1));
5132     SDValue Op3 = getValue(I.getArgOperand(2));
5133     // @llvm.memset defines 0 and 1 to both mean no alignment.
5134     unsigned Align = std::max<unsigned>(MSI.getDestAlignment(), 1);
5135     bool isVol = MSI.isVolatile();
5136     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5137     SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5138                                isTC, MachinePointerInfo(I.getArgOperand(0)));
5139     updateDAGForMaybeTailCall(MS);
5140     return nullptr;
5141   }
5142   case Intrinsic::memmove: {
5143     const auto &MMI = cast<MemMoveInst>(I);
5144     SDValue Op1 = getValue(I.getArgOperand(0));
5145     SDValue Op2 = getValue(I.getArgOperand(1));
5146     SDValue Op3 = getValue(I.getArgOperand(2));
5147     // @llvm.memmove defines 0 and 1 to both mean no alignment.
5148     unsigned DstAlign = std::max<unsigned>(MMI.getDestAlignment(), 1);
5149     unsigned SrcAlign = std::max<unsigned>(MMI.getSourceAlignment(), 1);
5150     unsigned Align = MinAlign(DstAlign, SrcAlign);
5151     bool isVol = MMI.isVolatile();
5152     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5153     // FIXME: Support passing different dest/src alignments to the memmove DAG
5154     // node.
5155     SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5156                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
5157                                 MachinePointerInfo(I.getArgOperand(1)));
5158     updateDAGForMaybeTailCall(MM);
5159     return nullptr;
5160   }
5161   case Intrinsic::memcpy_element_unordered_atomic: {
5162     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5163     SDValue Dst = getValue(MI.getRawDest());
5164     SDValue Src = getValue(MI.getRawSource());
5165     SDValue Length = getValue(MI.getLength());
5166 
5167     unsigned DstAlign = MI.getDestAlignment();
5168     unsigned SrcAlign = MI.getSourceAlignment();
5169     Type *LengthTy = MI.getLength()->getType();
5170     unsigned ElemSz = MI.getElementSizeInBytes();
5171     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5172     SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5173                                      SrcAlign, Length, LengthTy, ElemSz, isTC,
5174                                      MachinePointerInfo(MI.getRawDest()),
5175                                      MachinePointerInfo(MI.getRawSource()));
5176     updateDAGForMaybeTailCall(MC);
5177     return nullptr;
5178   }
5179   case Intrinsic::memmove_element_unordered_atomic: {
5180     auto &MI = cast<AtomicMemMoveInst>(I);
5181     SDValue Dst = getValue(MI.getRawDest());
5182     SDValue Src = getValue(MI.getRawSource());
5183     SDValue Length = getValue(MI.getLength());
5184 
5185     unsigned DstAlign = MI.getDestAlignment();
5186     unsigned SrcAlign = MI.getSourceAlignment();
5187     Type *LengthTy = MI.getLength()->getType();
5188     unsigned ElemSz = MI.getElementSizeInBytes();
5189     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5190     SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5191                                       SrcAlign, Length, LengthTy, ElemSz, isTC,
5192                                       MachinePointerInfo(MI.getRawDest()),
5193                                       MachinePointerInfo(MI.getRawSource()));
5194     updateDAGForMaybeTailCall(MC);
5195     return nullptr;
5196   }
5197   case Intrinsic::memset_element_unordered_atomic: {
5198     auto &MI = cast<AtomicMemSetInst>(I);
5199     SDValue Dst = getValue(MI.getRawDest());
5200     SDValue Val = getValue(MI.getValue());
5201     SDValue Length = getValue(MI.getLength());
5202 
5203     unsigned DstAlign = MI.getDestAlignment();
5204     Type *LengthTy = MI.getLength()->getType();
5205     unsigned ElemSz = MI.getElementSizeInBytes();
5206     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5207     SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5208                                      LengthTy, ElemSz, isTC,
5209                                      MachinePointerInfo(MI.getRawDest()));
5210     updateDAGForMaybeTailCall(MC);
5211     return nullptr;
5212   }
5213   case Intrinsic::dbg_addr:
5214   case Intrinsic::dbg_declare: {
5215     const auto &DI = cast<DbgVariableIntrinsic>(I);
5216     DILocalVariable *Variable = DI.getVariable();
5217     DIExpression *Expression = DI.getExpression();
5218     dropDanglingDebugInfo(Variable, Expression);
5219     assert(Variable && "Missing variable");
5220 
5221     // Check if address has undef value.
5222     const Value *Address = DI.getVariableLocation();
5223     if (!Address || isa<UndefValue>(Address) ||
5224         (Address->use_empty() && !isa<Argument>(Address))) {
5225       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5226       return nullptr;
5227     }
5228 
5229     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
5230 
5231     // Check if this variable can be described by a frame index, typically
5232     // either as a static alloca or a byval parameter.
5233     int FI = std::numeric_limits<int>::max();
5234     if (const auto *AI =
5235             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
5236       if (AI->isStaticAlloca()) {
5237         auto I = FuncInfo.StaticAllocaMap.find(AI);
5238         if (I != FuncInfo.StaticAllocaMap.end())
5239           FI = I->second;
5240       }
5241     } else if (const auto *Arg = dyn_cast<Argument>(
5242                    Address->stripInBoundsConstantOffsets())) {
5243       FI = FuncInfo.getArgumentFrameIndex(Arg);
5244     }
5245 
5246     // llvm.dbg.addr is control dependent and always generates indirect
5247     // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
5248     // the MachineFunction variable table.
5249     if (FI != std::numeric_limits<int>::max()) {
5250       if (Intrinsic == Intrinsic::dbg_addr) {
5251         SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
5252             Variable, Expression, FI, /*IsIndirect*/ true, dl, SDNodeOrder);
5253         DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter);
5254       }
5255       return nullptr;
5256     }
5257 
5258     SDValue &N = NodeMap[Address];
5259     if (!N.getNode() && isa<Argument>(Address))
5260       // Check unused arguments map.
5261       N = UnusedArgNodeMap[Address];
5262     SDDbgValue *SDV;
5263     if (N.getNode()) {
5264       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
5265         Address = BCI->getOperand(0);
5266       // Parameters are handled specially.
5267       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
5268       if (isParameter && FINode) {
5269         // Byval parameter. We have a frame index at this point.
5270         SDV =
5271             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
5272                                       /*IsIndirect*/ true, dl, SDNodeOrder);
5273       } else if (isa<Argument>(Address)) {
5274         // Address is an argument, so try to emit its dbg value using
5275         // virtual register info from the FuncInfo.ValueMap.
5276         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N);
5277         return nullptr;
5278       } else {
5279         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5280                               true, dl, SDNodeOrder);
5281       }
5282       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5283     } else {
5284       // If Address is an argument then try to emit its dbg value using
5285       // virtual register info from the FuncInfo.ValueMap.
5286       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true,
5287                                     N)) {
5288         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5289       }
5290     }
5291     return nullptr;
5292   }
5293   case Intrinsic::dbg_label: {
5294     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
5295     DILabel *Label = DI.getLabel();
5296     assert(Label && "Missing label");
5297 
5298     SDDbgLabel *SDV;
5299     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
5300     DAG.AddDbgLabel(SDV);
5301     return nullptr;
5302   }
5303   case Intrinsic::dbg_value: {
5304     const DbgValueInst &DI = cast<DbgValueInst>(I);
5305     assert(DI.getVariable() && "Missing variable");
5306 
5307     DILocalVariable *Variable = DI.getVariable();
5308     DIExpression *Expression = DI.getExpression();
5309     dropDanglingDebugInfo(Variable, Expression);
5310     const Value *V = DI.getValue();
5311     if (!V)
5312       return nullptr;
5313 
5314     SDDbgValue *SDV;
5315     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
5316         isa<ConstantPointerNull>(V)) {
5317       SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder);
5318       DAG.AddDbgValue(SDV, nullptr, false);
5319       return nullptr;
5320     }
5321 
5322     // Do not use getValue() in here; we don't want to generate code at
5323     // this point if it hasn't been done yet.
5324     SDValue N = NodeMap[V];
5325     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
5326       N = UnusedArgNodeMap[V];
5327     if (N.getNode()) {
5328       if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N))
5329         return nullptr;
5330       SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder);
5331       DAG.AddDbgValue(SDV, N.getNode(), false);
5332       return nullptr;
5333     }
5334 
5335     // PHI nodes have already been selected, so we should know which VReg that
5336     // is assigns to already.
5337     if (isa<PHINode>(V)) {
5338       auto VMI = FuncInfo.ValueMap.find(V);
5339       if (VMI != FuncInfo.ValueMap.end()) {
5340         unsigned Reg = VMI->second;
5341         // The PHI node may be split up into several MI PHI nodes (in
5342         // FunctionLoweringInfo::set).
5343         RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
5344                          V->getType(), None);
5345         if (RFV.occupiesMultipleRegs()) {
5346           unsigned Offset = 0;
5347           unsigned BitsToDescribe = 0;
5348           if (auto VarSize = Variable->getSizeInBits())
5349             BitsToDescribe = *VarSize;
5350           if (auto Fragment = Expression->getFragmentInfo())
5351             BitsToDescribe = Fragment->SizeInBits;
5352           for (auto RegAndSize : RFV.getRegsAndSizes()) {
5353             unsigned RegisterSize = RegAndSize.second;
5354             // Bail out if all bits are described already.
5355             if (Offset >= BitsToDescribe)
5356               break;
5357             unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
5358                 ? BitsToDescribe - Offset
5359                 : RegisterSize;
5360             auto FragmentExpr = DIExpression::createFragmentExpression(
5361                 Expression, Offset, FragmentSize);
5362             if (!FragmentExpr)
5363                 continue;
5364             SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first,
5365                                       false, dl, SDNodeOrder);
5366             DAG.AddDbgValue(SDV, nullptr, false);
5367             Offset += RegisterSize;
5368           }
5369         } else {
5370           SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl,
5371                                     SDNodeOrder);
5372           DAG.AddDbgValue(SDV, nullptr, false);
5373         }
5374         return nullptr;
5375       }
5376     }
5377 
5378     // TODO: When we get here we will either drop the dbg.value completely, or
5379     // we try to move it forward by letting it dangle for awhile. So we should
5380     // probably add an extra DbgValue to the DAG here, with a reference to
5381     // "noreg", to indicate that we have lost the debug location for the
5382     // variable.
5383 
5384     if (!V->use_empty() ) {
5385       // Do not call getValue(V) yet, as we don't want to generate code.
5386       // Remember it for later.
5387       DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder);
5388       return nullptr;
5389     }
5390 
5391     LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
5392     LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
5393     return nullptr;
5394   }
5395 
5396   case Intrinsic::eh_typeid_for: {
5397     // Find the type id for the given typeinfo.
5398     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5399     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5400     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5401     setValue(&I, Res);
5402     return nullptr;
5403   }
5404 
5405   case Intrinsic::eh_return_i32:
5406   case Intrinsic::eh_return_i64:
5407     DAG.getMachineFunction().setCallsEHReturn(true);
5408     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5409                             MVT::Other,
5410                             getControlRoot(),
5411                             getValue(I.getArgOperand(0)),
5412                             getValue(I.getArgOperand(1))));
5413     return nullptr;
5414   case Intrinsic::eh_unwind_init:
5415     DAG.getMachineFunction().setCallsUnwindInit(true);
5416     return nullptr;
5417   case Intrinsic::eh_dwarf_cfa:
5418     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5419                              TLI.getPointerTy(DAG.getDataLayout()),
5420                              getValue(I.getArgOperand(0))));
5421     return nullptr;
5422   case Intrinsic::eh_sjlj_callsite: {
5423     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5424     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5425     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
5426     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
5427 
5428     MMI.setCurrentCallSite(CI->getZExtValue());
5429     return nullptr;
5430   }
5431   case Intrinsic::eh_sjlj_functioncontext: {
5432     // Get and store the index of the function context.
5433     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5434     AllocaInst *FnCtx =
5435       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5436     int FI = FuncInfo.StaticAllocaMap[FnCtx];
5437     MFI.setFunctionContextIndex(FI);
5438     return nullptr;
5439   }
5440   case Intrinsic::eh_sjlj_setjmp: {
5441     SDValue Ops[2];
5442     Ops[0] = getRoot();
5443     Ops[1] = getValue(I.getArgOperand(0));
5444     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5445                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
5446     setValue(&I, Op.getValue(0));
5447     DAG.setRoot(Op.getValue(1));
5448     return nullptr;
5449   }
5450   case Intrinsic::eh_sjlj_longjmp:
5451     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5452                             getRoot(), getValue(I.getArgOperand(0))));
5453     return nullptr;
5454   case Intrinsic::eh_sjlj_setup_dispatch:
5455     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5456                             getRoot()));
5457     return nullptr;
5458   case Intrinsic::masked_gather:
5459     visitMaskedGather(I);
5460     return nullptr;
5461   case Intrinsic::masked_load:
5462     visitMaskedLoad(I);
5463     return nullptr;
5464   case Intrinsic::masked_scatter:
5465     visitMaskedScatter(I);
5466     return nullptr;
5467   case Intrinsic::masked_store:
5468     visitMaskedStore(I);
5469     return nullptr;
5470   case Intrinsic::masked_expandload:
5471     visitMaskedLoad(I, true /* IsExpanding */);
5472     return nullptr;
5473   case Intrinsic::masked_compressstore:
5474     visitMaskedStore(I, true /* IsCompressing */);
5475     return nullptr;
5476   case Intrinsic::x86_mmx_pslli_w:
5477   case Intrinsic::x86_mmx_pslli_d:
5478   case Intrinsic::x86_mmx_pslli_q:
5479   case Intrinsic::x86_mmx_psrli_w:
5480   case Intrinsic::x86_mmx_psrli_d:
5481   case Intrinsic::x86_mmx_psrli_q:
5482   case Intrinsic::x86_mmx_psrai_w:
5483   case Intrinsic::x86_mmx_psrai_d: {
5484     SDValue ShAmt = getValue(I.getArgOperand(1));
5485     if (isa<ConstantSDNode>(ShAmt)) {
5486       visitTargetIntrinsic(I, Intrinsic);
5487       return nullptr;
5488     }
5489     unsigned NewIntrinsic = 0;
5490     EVT ShAmtVT = MVT::v2i32;
5491     switch (Intrinsic) {
5492     case Intrinsic::x86_mmx_pslli_w:
5493       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5494       break;
5495     case Intrinsic::x86_mmx_pslli_d:
5496       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5497       break;
5498     case Intrinsic::x86_mmx_pslli_q:
5499       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5500       break;
5501     case Intrinsic::x86_mmx_psrli_w:
5502       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5503       break;
5504     case Intrinsic::x86_mmx_psrli_d:
5505       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5506       break;
5507     case Intrinsic::x86_mmx_psrli_q:
5508       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5509       break;
5510     case Intrinsic::x86_mmx_psrai_w:
5511       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5512       break;
5513     case Intrinsic::x86_mmx_psrai_d:
5514       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5515       break;
5516     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
5517     }
5518 
5519     // The vector shift intrinsics with scalars uses 32b shift amounts but
5520     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5521     // to be zero.
5522     // We must do this early because v2i32 is not a legal type.
5523     SDValue ShOps[2];
5524     ShOps[0] = ShAmt;
5525     ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5526     ShAmt =  DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5527     EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5528     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5529     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5530                        DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5531                        getValue(I.getArgOperand(0)), ShAmt);
5532     setValue(&I, Res);
5533     return nullptr;
5534   }
5535   case Intrinsic::powi:
5536     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5537                             getValue(I.getArgOperand(1)), DAG));
5538     return nullptr;
5539   case Intrinsic::log:
5540     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5541     return nullptr;
5542   case Intrinsic::log2:
5543     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5544     return nullptr;
5545   case Intrinsic::log10:
5546     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5547     return nullptr;
5548   case Intrinsic::exp:
5549     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5550     return nullptr;
5551   case Intrinsic::exp2:
5552     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5553     return nullptr;
5554   case Intrinsic::pow:
5555     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5556                            getValue(I.getArgOperand(1)), DAG, TLI));
5557     return nullptr;
5558   case Intrinsic::sqrt:
5559   case Intrinsic::fabs:
5560   case Intrinsic::sin:
5561   case Intrinsic::cos:
5562   case Intrinsic::floor:
5563   case Intrinsic::ceil:
5564   case Intrinsic::trunc:
5565   case Intrinsic::rint:
5566   case Intrinsic::nearbyint:
5567   case Intrinsic::round:
5568   case Intrinsic::canonicalize: {
5569     unsigned Opcode;
5570     switch (Intrinsic) {
5571     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
5572     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
5573     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
5574     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
5575     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
5576     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
5577     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
5578     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
5579     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
5580     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5581     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
5582     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5583     }
5584 
5585     setValue(&I, DAG.getNode(Opcode, sdl,
5586                              getValue(I.getArgOperand(0)).getValueType(),
5587                              getValue(I.getArgOperand(0))));
5588     return nullptr;
5589   }
5590   case Intrinsic::minnum: {
5591     auto VT = getValue(I.getArgOperand(0)).getValueType();
5592     unsigned Opc =
5593         I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT)
5594             ? ISD::FMINIMUM
5595             : ISD::FMINNUM;
5596     setValue(&I, DAG.getNode(Opc, sdl, VT,
5597                              getValue(I.getArgOperand(0)),
5598                              getValue(I.getArgOperand(1))));
5599     return nullptr;
5600   }
5601   case Intrinsic::maxnum: {
5602     auto VT = getValue(I.getArgOperand(0)).getValueType();
5603     unsigned Opc =
5604         I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT)
5605             ? ISD::FMAXIMUM
5606             : ISD::FMAXNUM;
5607     setValue(&I, DAG.getNode(Opc, sdl, VT,
5608                              getValue(I.getArgOperand(0)),
5609                              getValue(I.getArgOperand(1))));
5610     return nullptr;
5611   }
5612   case Intrinsic::minimum:
5613     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
5614                              getValue(I.getArgOperand(0)).getValueType(),
5615                              getValue(I.getArgOperand(0)),
5616                              getValue(I.getArgOperand(1))));
5617     return nullptr;
5618   case Intrinsic::maximum:
5619     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
5620                              getValue(I.getArgOperand(0)).getValueType(),
5621                              getValue(I.getArgOperand(0)),
5622                              getValue(I.getArgOperand(1))));
5623     return nullptr;
5624   case Intrinsic::copysign:
5625     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5626                              getValue(I.getArgOperand(0)).getValueType(),
5627                              getValue(I.getArgOperand(0)),
5628                              getValue(I.getArgOperand(1))));
5629     return nullptr;
5630   case Intrinsic::fma:
5631     setValue(&I, DAG.getNode(ISD::FMA, sdl,
5632                              getValue(I.getArgOperand(0)).getValueType(),
5633                              getValue(I.getArgOperand(0)),
5634                              getValue(I.getArgOperand(1)),
5635                              getValue(I.getArgOperand(2))));
5636     return nullptr;
5637   case Intrinsic::experimental_constrained_fadd:
5638   case Intrinsic::experimental_constrained_fsub:
5639   case Intrinsic::experimental_constrained_fmul:
5640   case Intrinsic::experimental_constrained_fdiv:
5641   case Intrinsic::experimental_constrained_frem:
5642   case Intrinsic::experimental_constrained_fma:
5643   case Intrinsic::experimental_constrained_sqrt:
5644   case Intrinsic::experimental_constrained_pow:
5645   case Intrinsic::experimental_constrained_powi:
5646   case Intrinsic::experimental_constrained_sin:
5647   case Intrinsic::experimental_constrained_cos:
5648   case Intrinsic::experimental_constrained_exp:
5649   case Intrinsic::experimental_constrained_exp2:
5650   case Intrinsic::experimental_constrained_log:
5651   case Intrinsic::experimental_constrained_log10:
5652   case Intrinsic::experimental_constrained_log2:
5653   case Intrinsic::experimental_constrained_rint:
5654   case Intrinsic::experimental_constrained_nearbyint:
5655   case Intrinsic::experimental_constrained_maxnum:
5656   case Intrinsic::experimental_constrained_minnum:
5657   case Intrinsic::experimental_constrained_ceil:
5658   case Intrinsic::experimental_constrained_floor:
5659   case Intrinsic::experimental_constrained_round:
5660   case Intrinsic::experimental_constrained_trunc:
5661     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
5662     return nullptr;
5663   case Intrinsic::fmuladd: {
5664     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5665     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5666         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5667       setValue(&I, DAG.getNode(ISD::FMA, sdl,
5668                                getValue(I.getArgOperand(0)).getValueType(),
5669                                getValue(I.getArgOperand(0)),
5670                                getValue(I.getArgOperand(1)),
5671                                getValue(I.getArgOperand(2))));
5672     } else {
5673       // TODO: Intrinsic calls should have fast-math-flags.
5674       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5675                                 getValue(I.getArgOperand(0)).getValueType(),
5676                                 getValue(I.getArgOperand(0)),
5677                                 getValue(I.getArgOperand(1)));
5678       SDValue Add = DAG.getNode(ISD::FADD, sdl,
5679                                 getValue(I.getArgOperand(0)).getValueType(),
5680                                 Mul,
5681                                 getValue(I.getArgOperand(2)));
5682       setValue(&I, Add);
5683     }
5684     return nullptr;
5685   }
5686   case Intrinsic::convert_to_fp16:
5687     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5688                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5689                                          getValue(I.getArgOperand(0)),
5690                                          DAG.getTargetConstant(0, sdl,
5691                                                                MVT::i32))));
5692     return nullptr;
5693   case Intrinsic::convert_from_fp16:
5694     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5695                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5696                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5697                                          getValue(I.getArgOperand(0)))));
5698     return nullptr;
5699   case Intrinsic::pcmarker: {
5700     SDValue Tmp = getValue(I.getArgOperand(0));
5701     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5702     return nullptr;
5703   }
5704   case Intrinsic::readcyclecounter: {
5705     SDValue Op = getRoot();
5706     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5707                       DAG.getVTList(MVT::i64, MVT::Other), Op);
5708     setValue(&I, Res);
5709     DAG.setRoot(Res.getValue(1));
5710     return nullptr;
5711   }
5712   case Intrinsic::bitreverse:
5713     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5714                              getValue(I.getArgOperand(0)).getValueType(),
5715                              getValue(I.getArgOperand(0))));
5716     return nullptr;
5717   case Intrinsic::bswap:
5718     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5719                              getValue(I.getArgOperand(0)).getValueType(),
5720                              getValue(I.getArgOperand(0))));
5721     return nullptr;
5722   case Intrinsic::cttz: {
5723     SDValue Arg = getValue(I.getArgOperand(0));
5724     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5725     EVT Ty = Arg.getValueType();
5726     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5727                              sdl, Ty, Arg));
5728     return nullptr;
5729   }
5730   case Intrinsic::ctlz: {
5731     SDValue Arg = getValue(I.getArgOperand(0));
5732     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5733     EVT Ty = Arg.getValueType();
5734     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5735                              sdl, Ty, Arg));
5736     return nullptr;
5737   }
5738   case Intrinsic::ctpop: {
5739     SDValue Arg = getValue(I.getArgOperand(0));
5740     EVT Ty = Arg.getValueType();
5741     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5742     return nullptr;
5743   }
5744   case Intrinsic::fshl:
5745   case Intrinsic::fshr: {
5746     bool IsFSHL = Intrinsic == Intrinsic::fshl;
5747     SDValue X = getValue(I.getArgOperand(0));
5748     SDValue Y = getValue(I.getArgOperand(1));
5749     SDValue Z = getValue(I.getArgOperand(2));
5750     EVT VT = X.getValueType();
5751     SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT);
5752     SDValue Zero = DAG.getConstant(0, sdl, VT);
5753     SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC);
5754 
5755     auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
5756     if (TLI.isOperationLegalOrCustom(FunnelOpcode, VT)) {
5757       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
5758       return nullptr;
5759     }
5760 
5761     // When X == Y, this is rotate. If the data type has a power-of-2 size, we
5762     // avoid the select that is necessary in the general case to filter out
5763     // the 0-shift possibility that leads to UB.
5764     if (X == Y && isPowerOf2_32(VT.getScalarSizeInBits())) {
5765       // TODO: This should also be done if the operation is custom, but we have
5766       // to make sure targets are handling the modulo shift amount as expected.
5767       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
5768       if (TLI.isOperationLegal(RotateOpcode, VT)) {
5769         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
5770         return nullptr;
5771       }
5772 
5773       // Some targets only rotate one way. Try the opposite direction.
5774       RotateOpcode = IsFSHL ? ISD::ROTR : ISD::ROTL;
5775       if (TLI.isOperationLegal(RotateOpcode, VT)) {
5776         // Negate the shift amount because it is safe to ignore the high bits.
5777         SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
5778         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, NegShAmt));
5779         return nullptr;
5780       }
5781 
5782       // fshl (rotl): (X << (Z % BW)) | (X >> ((0 - Z) % BW))
5783       // fshr (rotr): (X << ((0 - Z) % BW)) | (X >> (Z % BW))
5784       SDValue NegZ = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
5785       SDValue NShAmt = DAG.getNode(ISD::UREM, sdl, VT, NegZ, BitWidthC);
5786       SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : NShAmt);
5787       SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, X, IsFSHL ? NShAmt : ShAmt);
5788       setValue(&I, DAG.getNode(ISD::OR, sdl, VT, ShX, ShY));
5789       return nullptr;
5790     }
5791 
5792     // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
5793     // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
5794     SDValue InvShAmt = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, ShAmt);
5795     SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt);
5796     SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt);
5797     SDValue Or = DAG.getNode(ISD::OR, sdl, VT, ShX, ShY);
5798 
5799     // If (Z % BW == 0), then the opposite direction shift is shift-by-bitwidth,
5800     // and that is undefined. We must compare and select to avoid UB.
5801     EVT CCVT = MVT::i1;
5802     if (VT.isVector())
5803       CCVT = EVT::getVectorVT(*Context, CCVT, VT.getVectorNumElements());
5804 
5805     // For fshl, 0-shift returns the 1st arg (X).
5806     // For fshr, 0-shift returns the 2nd arg (Y).
5807     SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ);
5808     setValue(&I, DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Or));
5809     return nullptr;
5810   }
5811   case Intrinsic::sadd_sat: {
5812     SDValue Op1 = getValue(I.getArgOperand(0));
5813     SDValue Op2 = getValue(I.getArgOperand(1));
5814     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
5815     return nullptr;
5816   }
5817   case Intrinsic::uadd_sat: {
5818     SDValue Op1 = getValue(I.getArgOperand(0));
5819     SDValue Op2 = getValue(I.getArgOperand(1));
5820     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
5821     return nullptr;
5822   }
5823   case Intrinsic::ssub_sat: {
5824     SDValue Op1 = getValue(I.getArgOperand(0));
5825     SDValue Op2 = getValue(I.getArgOperand(1));
5826     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
5827     return nullptr;
5828   }
5829   case Intrinsic::usub_sat: {
5830     SDValue Op1 = getValue(I.getArgOperand(0));
5831     SDValue Op2 = getValue(I.getArgOperand(1));
5832     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
5833     return nullptr;
5834   }
5835   case Intrinsic::stacksave: {
5836     SDValue Op = getRoot();
5837     Res = DAG.getNode(
5838         ISD::STACKSAVE, sdl,
5839         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5840     setValue(&I, Res);
5841     DAG.setRoot(Res.getValue(1));
5842     return nullptr;
5843   }
5844   case Intrinsic::stackrestore:
5845     Res = getValue(I.getArgOperand(0));
5846     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5847     return nullptr;
5848   case Intrinsic::get_dynamic_area_offset: {
5849     SDValue Op = getRoot();
5850     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5851     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5852     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5853     // target.
5854     if (PtrTy != ResTy)
5855       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5856                          " intrinsic!");
5857     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5858                       Op);
5859     DAG.setRoot(Op);
5860     setValue(&I, Res);
5861     return nullptr;
5862   }
5863   case Intrinsic::stackguard: {
5864     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5865     MachineFunction &MF = DAG.getMachineFunction();
5866     const Module &M = *MF.getFunction().getParent();
5867     SDValue Chain = getRoot();
5868     if (TLI.useLoadStackGuardNode()) {
5869       Res = getLoadStackGuard(DAG, sdl, Chain);
5870     } else {
5871       const Value *Global = TLI.getSDagStackGuard(M);
5872       unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5873       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5874                         MachinePointerInfo(Global, 0), Align,
5875                         MachineMemOperand::MOVolatile);
5876     }
5877     if (TLI.useStackGuardXorFP())
5878       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
5879     DAG.setRoot(Chain);
5880     setValue(&I, Res);
5881     return nullptr;
5882   }
5883   case Intrinsic::stackprotector: {
5884     // Emit code into the DAG to store the stack guard onto the stack.
5885     MachineFunction &MF = DAG.getMachineFunction();
5886     MachineFrameInfo &MFI = MF.getFrameInfo();
5887     EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5888     SDValue Src, Chain = getRoot();
5889 
5890     if (TLI.useLoadStackGuardNode())
5891       Src = getLoadStackGuard(DAG, sdl, Chain);
5892     else
5893       Src = getValue(I.getArgOperand(0));   // The guard's value.
5894 
5895     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5896 
5897     int FI = FuncInfo.StaticAllocaMap[Slot];
5898     MFI.setStackProtectorIndex(FI);
5899 
5900     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5901 
5902     // Store the stack protector onto the stack.
5903     Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5904                                                  DAG.getMachineFunction(), FI),
5905                        /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5906     setValue(&I, Res);
5907     DAG.setRoot(Res);
5908     return nullptr;
5909   }
5910   case Intrinsic::objectsize: {
5911     // If we don't know by now, we're never going to know.
5912     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5913 
5914     assert(CI && "Non-constant type in __builtin_object_size?");
5915 
5916     SDValue Arg = getValue(I.getCalledValue());
5917     EVT Ty = Arg.getValueType();
5918 
5919     if (CI->isZero())
5920       Res = DAG.getConstant(-1ULL, sdl, Ty);
5921     else
5922       Res = DAG.getConstant(0, sdl, Ty);
5923 
5924     setValue(&I, Res);
5925     return nullptr;
5926   }
5927 
5928   case Intrinsic::is_constant:
5929     // If this wasn't constant-folded away by now, then it's not a
5930     // constant.
5931     setValue(&I, DAG.getConstant(0, sdl, MVT::i1));
5932     return nullptr;
5933 
5934   case Intrinsic::annotation:
5935   case Intrinsic::ptr_annotation:
5936   case Intrinsic::launder_invariant_group:
5937   case Intrinsic::strip_invariant_group:
5938     // Drop the intrinsic, but forward the value
5939     setValue(&I, getValue(I.getOperand(0)));
5940     return nullptr;
5941   case Intrinsic::assume:
5942   case Intrinsic::var_annotation:
5943   case Intrinsic::sideeffect:
5944     // Discard annotate attributes, assumptions, and artificial side-effects.
5945     return nullptr;
5946 
5947   case Intrinsic::codeview_annotation: {
5948     // Emit a label associated with this metadata.
5949     MachineFunction &MF = DAG.getMachineFunction();
5950     MCSymbol *Label =
5951         MF.getMMI().getContext().createTempSymbol("annotation", true);
5952     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
5953     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
5954     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
5955     DAG.setRoot(Res);
5956     return nullptr;
5957   }
5958 
5959   case Intrinsic::init_trampoline: {
5960     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5961 
5962     SDValue Ops[6];
5963     Ops[0] = getRoot();
5964     Ops[1] = getValue(I.getArgOperand(0));
5965     Ops[2] = getValue(I.getArgOperand(1));
5966     Ops[3] = getValue(I.getArgOperand(2));
5967     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5968     Ops[5] = DAG.getSrcValue(F);
5969 
5970     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5971 
5972     DAG.setRoot(Res);
5973     return nullptr;
5974   }
5975   case Intrinsic::adjust_trampoline:
5976     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5977                              TLI.getPointerTy(DAG.getDataLayout()),
5978                              getValue(I.getArgOperand(0))));
5979     return nullptr;
5980   case Intrinsic::gcroot: {
5981     assert(DAG.getMachineFunction().getFunction().hasGC() &&
5982            "only valid in functions with gc specified, enforced by Verifier");
5983     assert(GFI && "implied by previous");
5984     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5985     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5986 
5987     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5988     GFI->addStackRoot(FI->getIndex(), TypeMap);
5989     return nullptr;
5990   }
5991   case Intrinsic::gcread:
5992   case Intrinsic::gcwrite:
5993     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
5994   case Intrinsic::flt_rounds:
5995     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5996     return nullptr;
5997 
5998   case Intrinsic::expect:
5999     // Just replace __builtin_expect(exp, c) with EXP.
6000     setValue(&I, getValue(I.getArgOperand(0)));
6001     return nullptr;
6002 
6003   case Intrinsic::debugtrap:
6004   case Intrinsic::trap: {
6005     StringRef TrapFuncName =
6006         I.getAttributes()
6007             .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
6008             .getValueAsString();
6009     if (TrapFuncName.empty()) {
6010       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
6011         ISD::TRAP : ISD::DEBUGTRAP;
6012       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
6013       return nullptr;
6014     }
6015     TargetLowering::ArgListTy Args;
6016 
6017     TargetLowering::CallLoweringInfo CLI(DAG);
6018     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6019         CallingConv::C, I.getType(),
6020         DAG.getExternalSymbol(TrapFuncName.data(),
6021                               TLI.getPointerTy(DAG.getDataLayout())),
6022         std::move(Args));
6023 
6024     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6025     DAG.setRoot(Result.second);
6026     return nullptr;
6027   }
6028 
6029   case Intrinsic::uadd_with_overflow:
6030   case Intrinsic::sadd_with_overflow:
6031   case Intrinsic::usub_with_overflow:
6032   case Intrinsic::ssub_with_overflow:
6033   case Intrinsic::umul_with_overflow:
6034   case Intrinsic::smul_with_overflow: {
6035     ISD::NodeType Op;
6036     switch (Intrinsic) {
6037     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6038     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6039     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6040     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6041     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6042     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6043     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6044     }
6045     SDValue Op1 = getValue(I.getArgOperand(0));
6046     SDValue Op2 = getValue(I.getArgOperand(1));
6047 
6048     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
6049     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6050     return nullptr;
6051   }
6052   case Intrinsic::prefetch: {
6053     SDValue Ops[5];
6054     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6055     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6056     Ops[0] = DAG.getRoot();
6057     Ops[1] = getValue(I.getArgOperand(0));
6058     Ops[2] = getValue(I.getArgOperand(1));
6059     Ops[3] = getValue(I.getArgOperand(2));
6060     Ops[4] = getValue(I.getArgOperand(3));
6061     SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
6062                                              DAG.getVTList(MVT::Other), Ops,
6063                                              EVT::getIntegerVT(*Context, 8),
6064                                              MachinePointerInfo(I.getArgOperand(0)),
6065                                              0, /* align */
6066                                              Flags);
6067 
6068     // Chain the prefetch in parallell with any pending loads, to stay out of
6069     // the way of later optimizations.
6070     PendingLoads.push_back(Result);
6071     Result = getRoot();
6072     DAG.setRoot(Result);
6073     return nullptr;
6074   }
6075   case Intrinsic::lifetime_start:
6076   case Intrinsic::lifetime_end: {
6077     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6078     // Stack coloring is not enabled in O0, discard region information.
6079     if (TM.getOptLevel() == CodeGenOpt::None)
6080       return nullptr;
6081 
6082     SmallVector<Value *, 4> Allocas;
6083     GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
6084 
6085     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
6086            E = Allocas.end(); Object != E; ++Object) {
6087       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
6088 
6089       // Could not find an Alloca.
6090       if (!LifetimeObject)
6091         continue;
6092 
6093       // First check that the Alloca is static, otherwise it won't have a
6094       // valid frame index.
6095       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6096       if (SI == FuncInfo.StaticAllocaMap.end())
6097         return nullptr;
6098 
6099       int FI = SI->second;
6100 
6101       SDValue Ops[2];
6102       Ops[0] = getRoot();
6103       Ops[1] =
6104           DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
6105       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
6106 
6107       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
6108       DAG.setRoot(Res);
6109     }
6110     return nullptr;
6111   }
6112   case Intrinsic::invariant_start:
6113     // Discard region information.
6114     setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
6115     return nullptr;
6116   case Intrinsic::invariant_end:
6117     // Discard region information.
6118     return nullptr;
6119   case Intrinsic::clear_cache:
6120     return TLI.getClearCacheBuiltinName();
6121   case Intrinsic::donothing:
6122     // ignore
6123     return nullptr;
6124   case Intrinsic::experimental_stackmap:
6125     visitStackmap(I);
6126     return nullptr;
6127   case Intrinsic::experimental_patchpoint_void:
6128   case Intrinsic::experimental_patchpoint_i64:
6129     visitPatchpoint(&I);
6130     return nullptr;
6131   case Intrinsic::experimental_gc_statepoint:
6132     LowerStatepoint(ImmutableStatepoint(&I));
6133     return nullptr;
6134   case Intrinsic::experimental_gc_result:
6135     visitGCResult(cast<GCResultInst>(I));
6136     return nullptr;
6137   case Intrinsic::experimental_gc_relocate:
6138     visitGCRelocate(cast<GCRelocateInst>(I));
6139     return nullptr;
6140   case Intrinsic::instrprof_increment:
6141     llvm_unreachable("instrprof failed to lower an increment");
6142   case Intrinsic::instrprof_value_profile:
6143     llvm_unreachable("instrprof failed to lower a value profiling call");
6144   case Intrinsic::localescape: {
6145     MachineFunction &MF = DAG.getMachineFunction();
6146     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6147 
6148     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6149     // is the same on all targets.
6150     for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
6151       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6152       if (isa<ConstantPointerNull>(Arg))
6153         continue; // Skip null pointers. They represent a hole in index space.
6154       AllocaInst *Slot = cast<AllocaInst>(Arg);
6155       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
6156              "can only escape static allocas");
6157       int FI = FuncInfo.StaticAllocaMap[Slot];
6158       MCSymbol *FrameAllocSym =
6159           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6160               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6161       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6162               TII->get(TargetOpcode::LOCAL_ESCAPE))
6163           .addSym(FrameAllocSym)
6164           .addFrameIndex(FI);
6165     }
6166 
6167     return nullptr;
6168   }
6169 
6170   case Intrinsic::localrecover: {
6171     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6172     MachineFunction &MF = DAG.getMachineFunction();
6173     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
6174 
6175     // Get the symbol that defines the frame offset.
6176     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6177     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6178     unsigned IdxVal =
6179         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6180     MCSymbol *FrameAllocSym =
6181         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6182             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6183 
6184     // Create a MCSymbol for the label to avoid any target lowering
6185     // that would make this PC relative.
6186     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6187     SDValue OffsetVal =
6188         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6189 
6190     // Add the offset to the FP.
6191     Value *FP = I.getArgOperand(1);
6192     SDValue FPVal = getValue(FP);
6193     SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
6194     setValue(&I, Add);
6195 
6196     return nullptr;
6197   }
6198 
6199   case Intrinsic::eh_exceptionpointer:
6200   case Intrinsic::eh_exceptioncode: {
6201     // Get the exception pointer vreg, copy from it, and resize it to fit.
6202     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
6203     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
6204     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
6205     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
6206     SDValue N =
6207         DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
6208     if (Intrinsic == Intrinsic::eh_exceptioncode)
6209       N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
6210     setValue(&I, N);
6211     return nullptr;
6212   }
6213   case Intrinsic::xray_customevent: {
6214     // Here we want to make sure that the intrinsic behaves as if it has a
6215     // specific calling convention, and only for x86_64.
6216     // FIXME: Support other platforms later.
6217     const auto &Triple = DAG.getTarget().getTargetTriple();
6218     if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6219       return nullptr;
6220 
6221     SDLoc DL = getCurSDLoc();
6222     SmallVector<SDValue, 8> Ops;
6223 
6224     // We want to say that we always want the arguments in registers.
6225     SDValue LogEntryVal = getValue(I.getArgOperand(0));
6226     SDValue StrSizeVal = getValue(I.getArgOperand(1));
6227     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6228     SDValue Chain = getRoot();
6229     Ops.push_back(LogEntryVal);
6230     Ops.push_back(StrSizeVal);
6231     Ops.push_back(Chain);
6232 
6233     // We need to enforce the calling convention for the callsite, so that
6234     // argument ordering is enforced correctly, and that register allocation can
6235     // see that some registers may be assumed clobbered and have to preserve
6236     // them across calls to the intrinsic.
6237     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
6238                                            DL, NodeTys, Ops);
6239     SDValue patchableNode = SDValue(MN, 0);
6240     DAG.setRoot(patchableNode);
6241     setValue(&I, patchableNode);
6242     return nullptr;
6243   }
6244   case Intrinsic::xray_typedevent: {
6245     // Here we want to make sure that the intrinsic behaves as if it has a
6246     // specific calling convention, and only for x86_64.
6247     // FIXME: Support other platforms later.
6248     const auto &Triple = DAG.getTarget().getTargetTriple();
6249     if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6250       return nullptr;
6251 
6252     SDLoc DL = getCurSDLoc();
6253     SmallVector<SDValue, 8> Ops;
6254 
6255     // We want to say that we always want the arguments in registers.
6256     // It's unclear to me how manipulating the selection DAG here forces callers
6257     // to provide arguments in registers instead of on the stack.
6258     SDValue LogTypeId = getValue(I.getArgOperand(0));
6259     SDValue LogEntryVal = getValue(I.getArgOperand(1));
6260     SDValue StrSizeVal = getValue(I.getArgOperand(2));
6261     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6262     SDValue Chain = getRoot();
6263     Ops.push_back(LogTypeId);
6264     Ops.push_back(LogEntryVal);
6265     Ops.push_back(StrSizeVal);
6266     Ops.push_back(Chain);
6267 
6268     // We need to enforce the calling convention for the callsite, so that
6269     // argument ordering is enforced correctly, and that register allocation can
6270     // see that some registers may be assumed clobbered and have to preserve
6271     // them across calls to the intrinsic.
6272     MachineSDNode *MN = DAG.getMachineNode(
6273         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, DL, NodeTys, Ops);
6274     SDValue patchableNode = SDValue(MN, 0);
6275     DAG.setRoot(patchableNode);
6276     setValue(&I, patchableNode);
6277     return nullptr;
6278   }
6279   case Intrinsic::experimental_deoptimize:
6280     LowerDeoptimizeCall(&I);
6281     return nullptr;
6282 
6283   case Intrinsic::experimental_vector_reduce_fadd:
6284   case Intrinsic::experimental_vector_reduce_fmul:
6285   case Intrinsic::experimental_vector_reduce_add:
6286   case Intrinsic::experimental_vector_reduce_mul:
6287   case Intrinsic::experimental_vector_reduce_and:
6288   case Intrinsic::experimental_vector_reduce_or:
6289   case Intrinsic::experimental_vector_reduce_xor:
6290   case Intrinsic::experimental_vector_reduce_smax:
6291   case Intrinsic::experimental_vector_reduce_smin:
6292   case Intrinsic::experimental_vector_reduce_umax:
6293   case Intrinsic::experimental_vector_reduce_umin:
6294   case Intrinsic::experimental_vector_reduce_fmax:
6295   case Intrinsic::experimental_vector_reduce_fmin:
6296     visitVectorReduce(I, Intrinsic);
6297     return nullptr;
6298 
6299   case Intrinsic::icall_branch_funnel: {
6300     SmallVector<SDValue, 16> Ops;
6301     Ops.push_back(DAG.getRoot());
6302     Ops.push_back(getValue(I.getArgOperand(0)));
6303 
6304     int64_t Offset;
6305     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6306         I.getArgOperand(1), Offset, DAG.getDataLayout()));
6307     if (!Base)
6308       report_fatal_error(
6309           "llvm.icall.branch.funnel operand must be a GlobalValue");
6310     Ops.push_back(DAG.getTargetGlobalAddress(Base, getCurSDLoc(), MVT::i64, 0));
6311 
6312     struct BranchFunnelTarget {
6313       int64_t Offset;
6314       SDValue Target;
6315     };
6316     SmallVector<BranchFunnelTarget, 8> Targets;
6317 
6318     for (unsigned Op = 1, N = I.getNumArgOperands(); Op != N; Op += 2) {
6319       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6320           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
6321       if (ElemBase != Base)
6322         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
6323                            "to the same GlobalValue");
6324 
6325       SDValue Val = getValue(I.getArgOperand(Op + 1));
6326       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
6327       if (!GA)
6328         report_fatal_error(
6329             "llvm.icall.branch.funnel operand must be a GlobalValue");
6330       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
6331                                      GA->getGlobal(), getCurSDLoc(),
6332                                      Val.getValueType(), GA->getOffset())});
6333     }
6334     llvm::sort(Targets,
6335                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
6336                  return T1.Offset < T2.Offset;
6337                });
6338 
6339     for (auto &T : Targets) {
6340       Ops.push_back(DAG.getTargetConstant(T.Offset, getCurSDLoc(), MVT::i32));
6341       Ops.push_back(T.Target);
6342     }
6343 
6344     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL,
6345                                  getCurSDLoc(), MVT::Other, Ops),
6346               0);
6347     DAG.setRoot(N);
6348     setValue(&I, N);
6349     HasTailCall = true;
6350     return nullptr;
6351   }
6352 
6353   case Intrinsic::wasm_landingpad_index:
6354     // Information this intrinsic contained has been transferred to
6355     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
6356     // delete it now.
6357     return nullptr;
6358   case Intrinsic::objc_autorelease:
6359     return "objc_autorelease";
6360   case Intrinsic::objc_autoreleasePoolPop:
6361     return "objc_autoreleasePoolPop";
6362   case Intrinsic::objc_autoreleasePoolPush:
6363     return "objc_autoreleasePoolPush";
6364   case Intrinsic::objc_autoreleaseReturnValue:
6365     return "objc_autoreleaseReturnValue";
6366   case Intrinsic::objc_copyWeak:
6367     return "objc_copyWeak";
6368   case Intrinsic::objc_destroyWeak:
6369     return "objc_destroyWeak";
6370   case Intrinsic::objc_initWeak:
6371     return "objc_initWeak";
6372   case Intrinsic::objc_loadWeak:
6373     return "objc_loadWeak";
6374   case Intrinsic::objc_loadWeakRetained:
6375     return "objc_loadWeakRetained";
6376   case Intrinsic::objc_moveWeak:
6377     return "objc_moveWeak";
6378   case Intrinsic::objc_release:
6379     return "objc_release";
6380   case Intrinsic::objc_retain:
6381     return "objc_retain";
6382   case Intrinsic::objc_retainAutorelease:
6383     return "objc_retainAutorelease";
6384   case Intrinsic::objc_retainAutoreleaseReturnValue:
6385     return "objc_retainAutoreleaseReturnValue";
6386   case Intrinsic::objc_retainAutoreleasedReturnValue:
6387     return "objc_retainAutoreleasedReturnValue";
6388   case Intrinsic::objc_retainBlock:
6389     return "objc_retainBlock";
6390   case Intrinsic::objc_storeStrong:
6391     return "objc_storeStrong";
6392   case Intrinsic::objc_storeWeak:
6393     return "objc_storeWeak";
6394   case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
6395     return "objc_unsafeClaimAutoreleasedReturnValue";
6396   case Intrinsic::objc_retainedObject:
6397     return "objc_retainedObject";
6398   case Intrinsic::objc_unretainedObject:
6399     return "objc_unretainedObject";
6400   case Intrinsic::objc_unretainedPointer:
6401     return "objc_unretainedPointer";
6402   case Intrinsic::objc_retain_autorelease:
6403     return "objc_retain_autorelease";
6404   case Intrinsic::objc_sync_enter:
6405     return "objc_sync_enter";
6406   case Intrinsic::objc_sync_exit:
6407     return "objc_sync_exit";
6408   }
6409 }
6410 
6411 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
6412     const ConstrainedFPIntrinsic &FPI) {
6413   SDLoc sdl = getCurSDLoc();
6414   unsigned Opcode;
6415   switch (FPI.getIntrinsicID()) {
6416   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6417   case Intrinsic::experimental_constrained_fadd:
6418     Opcode = ISD::STRICT_FADD;
6419     break;
6420   case Intrinsic::experimental_constrained_fsub:
6421     Opcode = ISD::STRICT_FSUB;
6422     break;
6423   case Intrinsic::experimental_constrained_fmul:
6424     Opcode = ISD::STRICT_FMUL;
6425     break;
6426   case Intrinsic::experimental_constrained_fdiv:
6427     Opcode = ISD::STRICT_FDIV;
6428     break;
6429   case Intrinsic::experimental_constrained_frem:
6430     Opcode = ISD::STRICT_FREM;
6431     break;
6432   case Intrinsic::experimental_constrained_fma:
6433     Opcode = ISD::STRICT_FMA;
6434     break;
6435   case Intrinsic::experimental_constrained_sqrt:
6436     Opcode = ISD::STRICT_FSQRT;
6437     break;
6438   case Intrinsic::experimental_constrained_pow:
6439     Opcode = ISD::STRICT_FPOW;
6440     break;
6441   case Intrinsic::experimental_constrained_powi:
6442     Opcode = ISD::STRICT_FPOWI;
6443     break;
6444   case Intrinsic::experimental_constrained_sin:
6445     Opcode = ISD::STRICT_FSIN;
6446     break;
6447   case Intrinsic::experimental_constrained_cos:
6448     Opcode = ISD::STRICT_FCOS;
6449     break;
6450   case Intrinsic::experimental_constrained_exp:
6451     Opcode = ISD::STRICT_FEXP;
6452     break;
6453   case Intrinsic::experimental_constrained_exp2:
6454     Opcode = ISD::STRICT_FEXP2;
6455     break;
6456   case Intrinsic::experimental_constrained_log:
6457     Opcode = ISD::STRICT_FLOG;
6458     break;
6459   case Intrinsic::experimental_constrained_log10:
6460     Opcode = ISD::STRICT_FLOG10;
6461     break;
6462   case Intrinsic::experimental_constrained_log2:
6463     Opcode = ISD::STRICT_FLOG2;
6464     break;
6465   case Intrinsic::experimental_constrained_rint:
6466     Opcode = ISD::STRICT_FRINT;
6467     break;
6468   case Intrinsic::experimental_constrained_nearbyint:
6469     Opcode = ISD::STRICT_FNEARBYINT;
6470     break;
6471   case Intrinsic::experimental_constrained_maxnum:
6472     Opcode = ISD::STRICT_FMAXNUM;
6473     break;
6474   case Intrinsic::experimental_constrained_minnum:
6475     Opcode = ISD::STRICT_FMINNUM;
6476     break;
6477   case Intrinsic::experimental_constrained_ceil:
6478     Opcode = ISD::STRICT_FCEIL;
6479     break;
6480   case Intrinsic::experimental_constrained_floor:
6481     Opcode = ISD::STRICT_FFLOOR;
6482     break;
6483   case Intrinsic::experimental_constrained_round:
6484     Opcode = ISD::STRICT_FROUND;
6485     break;
6486   case Intrinsic::experimental_constrained_trunc:
6487     Opcode = ISD::STRICT_FTRUNC;
6488     break;
6489   }
6490   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6491   SDValue Chain = getRoot();
6492   SmallVector<EVT, 4> ValueVTs;
6493   ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
6494   ValueVTs.push_back(MVT::Other); // Out chain
6495 
6496   SDVTList VTs = DAG.getVTList(ValueVTs);
6497   SDValue Result;
6498   if (FPI.isUnaryOp())
6499     Result = DAG.getNode(Opcode, sdl, VTs,
6500                          { Chain, getValue(FPI.getArgOperand(0)) });
6501   else if (FPI.isTernaryOp())
6502     Result = DAG.getNode(Opcode, sdl, VTs,
6503                          { Chain, getValue(FPI.getArgOperand(0)),
6504                                   getValue(FPI.getArgOperand(1)),
6505                                   getValue(FPI.getArgOperand(2)) });
6506   else
6507     Result = DAG.getNode(Opcode, sdl, VTs,
6508                          { Chain, getValue(FPI.getArgOperand(0)),
6509                            getValue(FPI.getArgOperand(1))  });
6510 
6511   assert(Result.getNode()->getNumValues() == 2);
6512   SDValue OutChain = Result.getValue(1);
6513   DAG.setRoot(OutChain);
6514   SDValue FPResult = Result.getValue(0);
6515   setValue(&FPI, FPResult);
6516 }
6517 
6518 std::pair<SDValue, SDValue>
6519 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
6520                                     const BasicBlock *EHPadBB) {
6521   MachineFunction &MF = DAG.getMachineFunction();
6522   MachineModuleInfo &MMI = MF.getMMI();
6523   MCSymbol *BeginLabel = nullptr;
6524 
6525   if (EHPadBB) {
6526     // Insert a label before the invoke call to mark the try range.  This can be
6527     // used to detect deletion of the invoke via the MachineModuleInfo.
6528     BeginLabel = MMI.getContext().createTempSymbol();
6529 
6530     // For SjLj, keep track of which landing pads go with which invokes
6531     // so as to maintain the ordering of pads in the LSDA.
6532     unsigned CallSiteIndex = MMI.getCurrentCallSite();
6533     if (CallSiteIndex) {
6534       MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
6535       LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
6536 
6537       // Now that the call site is handled, stop tracking it.
6538       MMI.setCurrentCallSite(0);
6539     }
6540 
6541     // Both PendingLoads and PendingExports must be flushed here;
6542     // this call might not return.
6543     (void)getRoot();
6544     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
6545 
6546     CLI.setChain(getRoot());
6547   }
6548   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6549   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6550 
6551   assert((CLI.IsTailCall || Result.second.getNode()) &&
6552          "Non-null chain expected with non-tail call!");
6553   assert((Result.second.getNode() || !Result.first.getNode()) &&
6554          "Null value expected with tail call!");
6555 
6556   if (!Result.second.getNode()) {
6557     // As a special case, a null chain means that a tail call has been emitted
6558     // and the DAG root is already updated.
6559     HasTailCall = true;
6560 
6561     // Since there's no actual continuation from this block, nothing can be
6562     // relying on us setting vregs for them.
6563     PendingExports.clear();
6564   } else {
6565     DAG.setRoot(Result.second);
6566   }
6567 
6568   if (EHPadBB) {
6569     // Insert a label at the end of the invoke call to mark the try range.  This
6570     // can be used to detect deletion of the invoke via the MachineModuleInfo.
6571     MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
6572     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
6573 
6574     // Inform MachineModuleInfo of range.
6575     auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
6576     // There is a platform (e.g. wasm) that uses funclet style IR but does not
6577     // actually use outlined funclets and their LSDA info style.
6578     if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
6579       assert(CLI.CS);
6580       WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
6581       EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS.getInstruction()),
6582                                 BeginLabel, EndLabel);
6583     } else if (!isScopedEHPersonality(Pers)) {
6584       MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
6585     }
6586   }
6587 
6588   return Result;
6589 }
6590 
6591 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
6592                                       bool isTailCall,
6593                                       const BasicBlock *EHPadBB) {
6594   auto &DL = DAG.getDataLayout();
6595   FunctionType *FTy = CS.getFunctionType();
6596   Type *RetTy = CS.getType();
6597 
6598   TargetLowering::ArgListTy Args;
6599   Args.reserve(CS.arg_size());
6600 
6601   const Value *SwiftErrorVal = nullptr;
6602   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6603 
6604   // We can't tail call inside a function with a swifterror argument. Lowering
6605   // does not support this yet. It would have to move into the swifterror
6606   // register before the call.
6607   auto *Caller = CS.getInstruction()->getParent()->getParent();
6608   if (TLI.supportSwiftError() &&
6609       Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
6610     isTailCall = false;
6611 
6612   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
6613        i != e; ++i) {
6614     TargetLowering::ArgListEntry Entry;
6615     const Value *V = *i;
6616 
6617     // Skip empty types
6618     if (V->getType()->isEmptyTy())
6619       continue;
6620 
6621     SDValue ArgNode = getValue(V);
6622     Entry.Node = ArgNode; Entry.Ty = V->getType();
6623 
6624     Entry.setAttributes(&CS, i - CS.arg_begin());
6625 
6626     // Use swifterror virtual register as input to the call.
6627     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
6628       SwiftErrorVal = V;
6629       // We find the virtual register for the actual swifterror argument.
6630       // Instead of using the Value, we use the virtual register instead.
6631       Entry.Node = DAG.getRegister(FuncInfo
6632                                        .getOrCreateSwiftErrorVRegUseAt(
6633                                            CS.getInstruction(), FuncInfo.MBB, V)
6634                                        .first,
6635                                    EVT(TLI.getPointerTy(DL)));
6636     }
6637 
6638     Args.push_back(Entry);
6639 
6640     // If we have an explicit sret argument that is an Instruction, (i.e., it
6641     // might point to function-local memory), we can't meaningfully tail-call.
6642     if (Entry.IsSRet && isa<Instruction>(V))
6643       isTailCall = false;
6644   }
6645 
6646   // Check if target-independent constraints permit a tail call here.
6647   // Target-dependent constraints are checked within TLI->LowerCallTo.
6648   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
6649     isTailCall = false;
6650 
6651   // Disable tail calls if there is an swifterror argument. Targets have not
6652   // been updated to support tail calls.
6653   if (TLI.supportSwiftError() && SwiftErrorVal)
6654     isTailCall = false;
6655 
6656   TargetLowering::CallLoweringInfo CLI(DAG);
6657   CLI.setDebugLoc(getCurSDLoc())
6658       .setChain(getRoot())
6659       .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
6660       .setTailCall(isTailCall)
6661       .setConvergent(CS.isConvergent());
6662   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
6663 
6664   if (Result.first.getNode()) {
6665     const Instruction *Inst = CS.getInstruction();
6666     Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
6667     setValue(Inst, Result.first);
6668   }
6669 
6670   // The last element of CLI.InVals has the SDValue for swifterror return.
6671   // Here we copy it to a virtual register and update SwiftErrorMap for
6672   // book-keeping.
6673   if (SwiftErrorVal && TLI.supportSwiftError()) {
6674     // Get the last element of InVals.
6675     SDValue Src = CLI.InVals.back();
6676     unsigned VReg; bool CreatedVReg;
6677     std::tie(VReg, CreatedVReg) =
6678         FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction());
6679     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
6680     // We update the virtual register for the actual swifterror argument.
6681     if (CreatedVReg)
6682       FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
6683     DAG.setRoot(CopyNode);
6684   }
6685 }
6686 
6687 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
6688                              SelectionDAGBuilder &Builder) {
6689   // Check to see if this load can be trivially constant folded, e.g. if the
6690   // input is from a string literal.
6691   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
6692     // Cast pointer to the type we really want to load.
6693     Type *LoadTy =
6694         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
6695     if (LoadVT.isVector())
6696       LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
6697 
6698     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
6699                                          PointerType::getUnqual(LoadTy));
6700 
6701     if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
6702             const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
6703       return Builder.getValue(LoadCst);
6704   }
6705 
6706   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
6707   // still constant memory, the input chain can be the entry node.
6708   SDValue Root;
6709   bool ConstantMemory = false;
6710 
6711   // Do not serialize (non-volatile) loads of constant memory with anything.
6712   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
6713     Root = Builder.DAG.getEntryNode();
6714     ConstantMemory = true;
6715   } else {
6716     // Do not serialize non-volatile loads against each other.
6717     Root = Builder.DAG.getRoot();
6718   }
6719 
6720   SDValue Ptr = Builder.getValue(PtrVal);
6721   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6722                                         Ptr, MachinePointerInfo(PtrVal),
6723                                         /* Alignment = */ 1);
6724 
6725   if (!ConstantMemory)
6726     Builder.PendingLoads.push_back(LoadVal.getValue(1));
6727   return LoadVal;
6728 }
6729 
6730 /// Record the value for an instruction that produces an integer result,
6731 /// converting the type where necessary.
6732 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6733                                                   SDValue Value,
6734                                                   bool IsSigned) {
6735   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6736                                                     I.getType(), true);
6737   if (IsSigned)
6738     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6739   else
6740     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6741   setValue(&I, Value);
6742 }
6743 
6744 /// See if we can lower a memcmp call into an optimized form. If so, return
6745 /// true and lower it. Otherwise return false, and it will be lowered like a
6746 /// normal call.
6747 /// The caller already checked that \p I calls the appropriate LibFunc with a
6748 /// correct prototype.
6749 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6750   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6751   const Value *Size = I.getArgOperand(2);
6752   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6753   if (CSize && CSize->getZExtValue() == 0) {
6754     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6755                                                           I.getType(), true);
6756     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6757     return true;
6758   }
6759 
6760   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6761   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6762       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6763       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6764   if (Res.first.getNode()) {
6765     processIntegerCallValue(I, Res.first, true);
6766     PendingLoads.push_back(Res.second);
6767     return true;
6768   }
6769 
6770   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
6771   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
6772   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
6773     return false;
6774 
6775   // If the target has a fast compare for the given size, it will return a
6776   // preferred load type for that size. Require that the load VT is legal and
6777   // that the target supports unaligned loads of that type. Otherwise, return
6778   // INVALID.
6779   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6780     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6781     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6782     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6783       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6784       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6785       // TODO: Check alignment of src and dest ptrs.
6786       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6787       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6788       if (!TLI.isTypeLegal(LVT) ||
6789           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6790           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6791         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6792     }
6793 
6794     return LVT;
6795   };
6796 
6797   // This turns into unaligned loads. We only do this if the target natively
6798   // supports the MVT we'll be loading or if it is small enough (<= 4) that
6799   // we'll only produce a small number of byte loads.
6800   MVT LoadVT;
6801   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6802   switch (NumBitsToCompare) {
6803   default:
6804     return false;
6805   case 16:
6806     LoadVT = MVT::i16;
6807     break;
6808   case 32:
6809     LoadVT = MVT::i32;
6810     break;
6811   case 64:
6812   case 128:
6813   case 256:
6814     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6815     break;
6816   }
6817 
6818   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6819     return false;
6820 
6821   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6822   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6823 
6824   // Bitcast to a wide integer type if the loads are vectors.
6825   if (LoadVT.isVector()) {
6826     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6827     LoadL = DAG.getBitcast(CmpVT, LoadL);
6828     LoadR = DAG.getBitcast(CmpVT, LoadR);
6829   }
6830 
6831   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6832   processIntegerCallValue(I, Cmp, false);
6833   return true;
6834 }
6835 
6836 /// See if we can lower a memchr call into an optimized form. If so, return
6837 /// true and lower it. Otherwise return false, and it will be lowered like a
6838 /// normal call.
6839 /// The caller already checked that \p I calls the appropriate LibFunc with a
6840 /// correct prototype.
6841 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6842   const Value *Src = I.getArgOperand(0);
6843   const Value *Char = I.getArgOperand(1);
6844   const Value *Length = I.getArgOperand(2);
6845 
6846   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6847   std::pair<SDValue, SDValue> Res =
6848     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6849                                 getValue(Src), getValue(Char), getValue(Length),
6850                                 MachinePointerInfo(Src));
6851   if (Res.first.getNode()) {
6852     setValue(&I, Res.first);
6853     PendingLoads.push_back(Res.second);
6854     return true;
6855   }
6856 
6857   return false;
6858 }
6859 
6860 /// See if we can lower a mempcpy call into an optimized form. If so, return
6861 /// true and lower it. Otherwise return false, and it will be lowered like a
6862 /// normal call.
6863 /// The caller already checked that \p I calls the appropriate LibFunc with a
6864 /// correct prototype.
6865 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6866   SDValue Dst = getValue(I.getArgOperand(0));
6867   SDValue Src = getValue(I.getArgOperand(1));
6868   SDValue Size = getValue(I.getArgOperand(2));
6869 
6870   unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6871   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6872   unsigned Align = std::min(DstAlign, SrcAlign);
6873   if (Align == 0) // Alignment of one or both could not be inferred.
6874     Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6875 
6876   bool isVol = false;
6877   SDLoc sdl = getCurSDLoc();
6878 
6879   // In the mempcpy context we need to pass in a false value for isTailCall
6880   // because the return pointer needs to be adjusted by the size of
6881   // the copied memory.
6882   SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6883                              false, /*isTailCall=*/false,
6884                              MachinePointerInfo(I.getArgOperand(0)),
6885                              MachinePointerInfo(I.getArgOperand(1)));
6886   assert(MC.getNode() != nullptr &&
6887          "** memcpy should not be lowered as TailCall in mempcpy context **");
6888   DAG.setRoot(MC);
6889 
6890   // Check if Size needs to be truncated or extended.
6891   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6892 
6893   // Adjust return pointer to point just past the last dst byte.
6894   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6895                                     Dst, Size);
6896   setValue(&I, DstPlusSize);
6897   return true;
6898 }
6899 
6900 /// See if we can lower a strcpy call into an optimized form.  If so, return
6901 /// true and lower it, otherwise return false and it will be lowered like a
6902 /// normal call.
6903 /// The caller already checked that \p I calls the appropriate LibFunc with a
6904 /// correct prototype.
6905 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6906   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6907 
6908   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6909   std::pair<SDValue, SDValue> Res =
6910     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6911                                 getValue(Arg0), getValue(Arg1),
6912                                 MachinePointerInfo(Arg0),
6913                                 MachinePointerInfo(Arg1), isStpcpy);
6914   if (Res.first.getNode()) {
6915     setValue(&I, Res.first);
6916     DAG.setRoot(Res.second);
6917     return true;
6918   }
6919 
6920   return false;
6921 }
6922 
6923 /// See if we can lower a strcmp call into an optimized form.  If so, return
6924 /// true and lower it, otherwise return false and it will be lowered like a
6925 /// normal call.
6926 /// The caller already checked that \p I calls the appropriate LibFunc with a
6927 /// correct prototype.
6928 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6929   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6930 
6931   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6932   std::pair<SDValue, SDValue> Res =
6933     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6934                                 getValue(Arg0), getValue(Arg1),
6935                                 MachinePointerInfo(Arg0),
6936                                 MachinePointerInfo(Arg1));
6937   if (Res.first.getNode()) {
6938     processIntegerCallValue(I, Res.first, true);
6939     PendingLoads.push_back(Res.second);
6940     return true;
6941   }
6942 
6943   return false;
6944 }
6945 
6946 /// See if we can lower a strlen call into an optimized form.  If so, return
6947 /// true and lower it, otherwise return false and it will be lowered like a
6948 /// normal call.
6949 /// The caller already checked that \p I calls the appropriate LibFunc with a
6950 /// correct prototype.
6951 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6952   const Value *Arg0 = I.getArgOperand(0);
6953 
6954   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6955   std::pair<SDValue, SDValue> Res =
6956     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6957                                 getValue(Arg0), MachinePointerInfo(Arg0));
6958   if (Res.first.getNode()) {
6959     processIntegerCallValue(I, Res.first, false);
6960     PendingLoads.push_back(Res.second);
6961     return true;
6962   }
6963 
6964   return false;
6965 }
6966 
6967 /// See if we can lower a strnlen call into an optimized form.  If so, return
6968 /// true and lower it, otherwise return false and it will be lowered like a
6969 /// normal call.
6970 /// The caller already checked that \p I calls the appropriate LibFunc with a
6971 /// correct prototype.
6972 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6973   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6974 
6975   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6976   std::pair<SDValue, SDValue> Res =
6977     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6978                                  getValue(Arg0), getValue(Arg1),
6979                                  MachinePointerInfo(Arg0));
6980   if (Res.first.getNode()) {
6981     processIntegerCallValue(I, Res.first, false);
6982     PendingLoads.push_back(Res.second);
6983     return true;
6984   }
6985 
6986   return false;
6987 }
6988 
6989 /// See if we can lower a unary floating-point operation into an SDNode with
6990 /// the specified Opcode.  If so, return true and lower it, otherwise return
6991 /// false and it will be lowered like a normal call.
6992 /// The caller already checked that \p I calls the appropriate LibFunc with a
6993 /// correct prototype.
6994 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6995                                               unsigned Opcode) {
6996   // We already checked this call's prototype; verify it doesn't modify errno.
6997   if (!I.onlyReadsMemory())
6998     return false;
6999 
7000   SDValue Tmp = getValue(I.getArgOperand(0));
7001   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
7002   return true;
7003 }
7004 
7005 /// See if we can lower a binary floating-point operation into an SDNode with
7006 /// the specified Opcode. If so, return true and lower it. Otherwise return
7007 /// false, and it will be lowered like a normal call.
7008 /// The caller already checked that \p I calls the appropriate LibFunc with a
7009 /// correct prototype.
7010 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
7011                                                unsigned Opcode) {
7012   // We already checked this call's prototype; verify it doesn't modify errno.
7013   if (!I.onlyReadsMemory())
7014     return false;
7015 
7016   SDValue Tmp0 = getValue(I.getArgOperand(0));
7017   SDValue Tmp1 = getValue(I.getArgOperand(1));
7018   EVT VT = Tmp0.getValueType();
7019   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
7020   return true;
7021 }
7022 
7023 void SelectionDAGBuilder::visitCall(const CallInst &I) {
7024   // Handle inline assembly differently.
7025   if (isa<InlineAsm>(I.getCalledValue())) {
7026     visitInlineAsm(&I);
7027     return;
7028   }
7029 
7030   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
7031   computeUsesVAFloatArgument(I, MMI);
7032 
7033   const char *RenameFn = nullptr;
7034   if (Function *F = I.getCalledFunction()) {
7035     if (F->isDeclaration()) {
7036       // Is this an LLVM intrinsic or a target-specific intrinsic?
7037       unsigned IID = F->getIntrinsicID();
7038       if (!IID)
7039         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
7040           IID = II->getIntrinsicID(F);
7041 
7042       if (IID) {
7043         RenameFn = visitIntrinsicCall(I, IID);
7044         if (!RenameFn)
7045           return;
7046       }
7047     }
7048 
7049     // Check for well-known libc/libm calls.  If the function is internal, it
7050     // can't be a library call.  Don't do the check if marked as nobuiltin for
7051     // some reason or the call site requires strict floating point semantics.
7052     LibFunc Func;
7053     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
7054         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
7055         LibInfo->hasOptimizedCodeGen(Func)) {
7056       switch (Func) {
7057       default: break;
7058       case LibFunc_copysign:
7059       case LibFunc_copysignf:
7060       case LibFunc_copysignl:
7061         // We already checked this call's prototype; verify it doesn't modify
7062         // errno.
7063         if (I.onlyReadsMemory()) {
7064           SDValue LHS = getValue(I.getArgOperand(0));
7065           SDValue RHS = getValue(I.getArgOperand(1));
7066           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
7067                                    LHS.getValueType(), LHS, RHS));
7068           return;
7069         }
7070         break;
7071       case LibFunc_fabs:
7072       case LibFunc_fabsf:
7073       case LibFunc_fabsl:
7074         if (visitUnaryFloatCall(I, ISD::FABS))
7075           return;
7076         break;
7077       case LibFunc_fmin:
7078       case LibFunc_fminf:
7079       case LibFunc_fminl:
7080         if (visitBinaryFloatCall(I, ISD::FMINNUM))
7081           return;
7082         break;
7083       case LibFunc_fmax:
7084       case LibFunc_fmaxf:
7085       case LibFunc_fmaxl:
7086         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
7087           return;
7088         break;
7089       case LibFunc_sin:
7090       case LibFunc_sinf:
7091       case LibFunc_sinl:
7092         if (visitUnaryFloatCall(I, ISD::FSIN))
7093           return;
7094         break;
7095       case LibFunc_cos:
7096       case LibFunc_cosf:
7097       case LibFunc_cosl:
7098         if (visitUnaryFloatCall(I, ISD::FCOS))
7099           return;
7100         break;
7101       case LibFunc_sqrt:
7102       case LibFunc_sqrtf:
7103       case LibFunc_sqrtl:
7104       case LibFunc_sqrt_finite:
7105       case LibFunc_sqrtf_finite:
7106       case LibFunc_sqrtl_finite:
7107         if (visitUnaryFloatCall(I, ISD::FSQRT))
7108           return;
7109         break;
7110       case LibFunc_floor:
7111       case LibFunc_floorf:
7112       case LibFunc_floorl:
7113         if (visitUnaryFloatCall(I, ISD::FFLOOR))
7114           return;
7115         break;
7116       case LibFunc_nearbyint:
7117       case LibFunc_nearbyintf:
7118       case LibFunc_nearbyintl:
7119         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
7120           return;
7121         break;
7122       case LibFunc_ceil:
7123       case LibFunc_ceilf:
7124       case LibFunc_ceill:
7125         if (visitUnaryFloatCall(I, ISD::FCEIL))
7126           return;
7127         break;
7128       case LibFunc_rint:
7129       case LibFunc_rintf:
7130       case LibFunc_rintl:
7131         if (visitUnaryFloatCall(I, ISD::FRINT))
7132           return;
7133         break;
7134       case LibFunc_round:
7135       case LibFunc_roundf:
7136       case LibFunc_roundl:
7137         if (visitUnaryFloatCall(I, ISD::FROUND))
7138           return;
7139         break;
7140       case LibFunc_trunc:
7141       case LibFunc_truncf:
7142       case LibFunc_truncl:
7143         if (visitUnaryFloatCall(I, ISD::FTRUNC))
7144           return;
7145         break;
7146       case LibFunc_log2:
7147       case LibFunc_log2f:
7148       case LibFunc_log2l:
7149         if (visitUnaryFloatCall(I, ISD::FLOG2))
7150           return;
7151         break;
7152       case LibFunc_exp2:
7153       case LibFunc_exp2f:
7154       case LibFunc_exp2l:
7155         if (visitUnaryFloatCall(I, ISD::FEXP2))
7156           return;
7157         break;
7158       case LibFunc_memcmp:
7159         if (visitMemCmpCall(I))
7160           return;
7161         break;
7162       case LibFunc_mempcpy:
7163         if (visitMemPCpyCall(I))
7164           return;
7165         break;
7166       case LibFunc_memchr:
7167         if (visitMemChrCall(I))
7168           return;
7169         break;
7170       case LibFunc_strcpy:
7171         if (visitStrCpyCall(I, false))
7172           return;
7173         break;
7174       case LibFunc_stpcpy:
7175         if (visitStrCpyCall(I, true))
7176           return;
7177         break;
7178       case LibFunc_strcmp:
7179         if (visitStrCmpCall(I))
7180           return;
7181         break;
7182       case LibFunc_strlen:
7183         if (visitStrLenCall(I))
7184           return;
7185         break;
7186       case LibFunc_strnlen:
7187         if (visitStrNLenCall(I))
7188           return;
7189         break;
7190       }
7191     }
7192   }
7193 
7194   SDValue Callee;
7195   if (!RenameFn)
7196     Callee = getValue(I.getCalledValue());
7197   else
7198     Callee = DAG.getExternalSymbol(
7199         RenameFn,
7200         DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
7201 
7202   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
7203   // have to do anything here to lower funclet bundles.
7204   assert(!I.hasOperandBundlesOtherThan(
7205              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
7206          "Cannot lower calls with arbitrary operand bundles!");
7207 
7208   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
7209     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
7210   else
7211     // Check if we can potentially perform a tail call. More detailed checking
7212     // is be done within LowerCallTo, after more information about the call is
7213     // known.
7214     LowerCallTo(&I, Callee, I.isTailCall());
7215 }
7216 
7217 namespace {
7218 
7219 /// AsmOperandInfo - This contains information for each constraint that we are
7220 /// lowering.
7221 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
7222 public:
7223   /// CallOperand - If this is the result output operand or a clobber
7224   /// this is null, otherwise it is the incoming operand to the CallInst.
7225   /// This gets modified as the asm is processed.
7226   SDValue CallOperand;
7227 
7228   /// AssignedRegs - If this is a register or register class operand, this
7229   /// contains the set of register corresponding to the operand.
7230   RegsForValue AssignedRegs;
7231 
7232   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
7233     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
7234   }
7235 
7236   /// Whether or not this operand accesses memory
7237   bool hasMemory(const TargetLowering &TLI) const {
7238     // Indirect operand accesses access memory.
7239     if (isIndirect)
7240       return true;
7241 
7242     for (const auto &Code : Codes)
7243       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
7244         return true;
7245 
7246     return false;
7247   }
7248 
7249   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
7250   /// corresponds to.  If there is no Value* for this operand, it returns
7251   /// MVT::Other.
7252   EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
7253                            const DataLayout &DL) const {
7254     if (!CallOperandVal) return MVT::Other;
7255 
7256     if (isa<BasicBlock>(CallOperandVal))
7257       return TLI.getPointerTy(DL);
7258 
7259     llvm::Type *OpTy = CallOperandVal->getType();
7260 
7261     // FIXME: code duplicated from TargetLowering::ParseConstraints().
7262     // If this is an indirect operand, the operand is a pointer to the
7263     // accessed type.
7264     if (isIndirect) {
7265       PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
7266       if (!PtrTy)
7267         report_fatal_error("Indirect operand for inline asm not a pointer!");
7268       OpTy = PtrTy->getElementType();
7269     }
7270 
7271     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
7272     if (StructType *STy = dyn_cast<StructType>(OpTy))
7273       if (STy->getNumElements() == 1)
7274         OpTy = STy->getElementType(0);
7275 
7276     // If OpTy is not a single value, it may be a struct/union that we
7277     // can tile with integers.
7278     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
7279       unsigned BitSize = DL.getTypeSizeInBits(OpTy);
7280       switch (BitSize) {
7281       default: break;
7282       case 1:
7283       case 8:
7284       case 16:
7285       case 32:
7286       case 64:
7287       case 128:
7288         OpTy = IntegerType::get(Context, BitSize);
7289         break;
7290       }
7291     }
7292 
7293     return TLI.getValueType(DL, OpTy, true);
7294   }
7295 };
7296 
7297 using SDISelAsmOperandInfoVector = SmallVector<SDISelAsmOperandInfo, 16>;
7298 
7299 } // end anonymous namespace
7300 
7301 /// Make sure that the output operand \p OpInfo and its corresponding input
7302 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
7303 /// out).
7304 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
7305                                SDISelAsmOperandInfo &MatchingOpInfo,
7306                                SelectionDAG &DAG) {
7307   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
7308     return;
7309 
7310   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
7311   const auto &TLI = DAG.getTargetLoweringInfo();
7312 
7313   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
7314       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
7315                                        OpInfo.ConstraintVT);
7316   std::pair<unsigned, const TargetRegisterClass *> InputRC =
7317       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
7318                                        MatchingOpInfo.ConstraintVT);
7319   if ((OpInfo.ConstraintVT.isInteger() !=
7320        MatchingOpInfo.ConstraintVT.isInteger()) ||
7321       (MatchRC.second != InputRC.second)) {
7322     // FIXME: error out in a more elegant fashion
7323     report_fatal_error("Unsupported asm: input constraint"
7324                        " with a matching output constraint of"
7325                        " incompatible type!");
7326   }
7327   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
7328 }
7329 
7330 /// Get a direct memory input to behave well as an indirect operand.
7331 /// This may introduce stores, hence the need for a \p Chain.
7332 /// \return The (possibly updated) chain.
7333 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
7334                                         SDISelAsmOperandInfo &OpInfo,
7335                                         SelectionDAG &DAG) {
7336   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7337 
7338   // If we don't have an indirect input, put it in the constpool if we can,
7339   // otherwise spill it to a stack slot.
7340   // TODO: This isn't quite right. We need to handle these according to
7341   // the addressing mode that the constraint wants. Also, this may take
7342   // an additional register for the computation and we don't want that
7343   // either.
7344 
7345   // If the operand is a float, integer, or vector constant, spill to a
7346   // constant pool entry to get its address.
7347   const Value *OpVal = OpInfo.CallOperandVal;
7348   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
7349       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
7350     OpInfo.CallOperand = DAG.getConstantPool(
7351         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
7352     return Chain;
7353   }
7354 
7355   // Otherwise, create a stack slot and emit a store to it before the asm.
7356   Type *Ty = OpVal->getType();
7357   auto &DL = DAG.getDataLayout();
7358   uint64_t TySize = DL.getTypeAllocSize(Ty);
7359   unsigned Align = DL.getPrefTypeAlignment(Ty);
7360   MachineFunction &MF = DAG.getMachineFunction();
7361   int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7362   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
7363   Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
7364                        MachinePointerInfo::getFixedStack(MF, SSFI));
7365   OpInfo.CallOperand = StackSlot;
7366 
7367   return Chain;
7368 }
7369 
7370 /// GetRegistersForValue - Assign registers (virtual or physical) for the
7371 /// specified operand.  We prefer to assign virtual registers, to allow the
7372 /// register allocator to handle the assignment process.  However, if the asm
7373 /// uses features that we can't model on machineinstrs, we have SDISel do the
7374 /// allocation.  This produces generally horrible, but correct, code.
7375 ///
7376 ///   OpInfo describes the operand
7377 ///   RefOpInfo describes the matching operand if any, the operand otherwise
7378 static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
7379                                  const SDLoc &DL, SDISelAsmOperandInfo &OpInfo,
7380                                  SDISelAsmOperandInfo &RefOpInfo) {
7381   LLVMContext &Context = *DAG.getContext();
7382 
7383   MachineFunction &MF = DAG.getMachineFunction();
7384   SmallVector<unsigned, 4> Regs;
7385   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7386 
7387   // If this is a constraint for a single physreg, or a constraint for a
7388   // register class, find it.
7389   std::pair<unsigned, const TargetRegisterClass *> PhysReg =
7390       TLI.getRegForInlineAsmConstraint(&TRI, RefOpInfo.ConstraintCode,
7391                                        RefOpInfo.ConstraintVT);
7392 
7393   unsigned NumRegs = 1;
7394   if (OpInfo.ConstraintVT != MVT::Other) {
7395     // If this is an FP operand in an integer register (or visa versa), or more
7396     // generally if the operand value disagrees with the register class we plan
7397     // to stick it in, fix the operand type.
7398     //
7399     // If this is an input value, the bitcast to the new type is done now.
7400     // Bitcast for output value is done at the end of visitInlineAsm().
7401     if ((OpInfo.Type == InlineAsm::isOutput ||
7402          OpInfo.Type == InlineAsm::isInput) &&
7403         PhysReg.second &&
7404         !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
7405       // Try to convert to the first EVT that the reg class contains.  If the
7406       // types are identical size, use a bitcast to convert (e.g. two differing
7407       // vector types).  Note: output bitcast is done at the end of
7408       // visitInlineAsm().
7409       MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
7410       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
7411         // Exclude indirect inputs while they are unsupported because the code
7412         // to perform the load is missing and thus OpInfo.CallOperand still
7413         // refers to the input address rather than the pointed-to value.
7414         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
7415           OpInfo.CallOperand =
7416               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
7417         OpInfo.ConstraintVT = RegVT;
7418         // If the operand is an FP value and we want it in integer registers,
7419         // use the corresponding integer type. This turns an f64 value into
7420         // i64, which can be passed with two i32 values on a 32-bit machine.
7421       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
7422         RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
7423         if (OpInfo.Type == InlineAsm::isInput)
7424           OpInfo.CallOperand =
7425               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
7426         OpInfo.ConstraintVT = RegVT;
7427       }
7428     }
7429 
7430     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
7431   }
7432 
7433   // No need to allocate a matching input constraint since the constraint it's
7434   // matching to has already been allocated.
7435   if (OpInfo.isMatchingInputConstraint())
7436     return;
7437 
7438   MVT RegVT;
7439   EVT ValueVT = OpInfo.ConstraintVT;
7440 
7441   // If this is a constraint for a specific physical register, like {r17},
7442   // assign it now.
7443   if (unsigned AssignedReg = PhysReg.first) {
7444     const TargetRegisterClass *RC = PhysReg.second;
7445     if (OpInfo.ConstraintVT == MVT::Other)
7446       ValueVT = *TRI.legalclasstypes_begin(*RC);
7447 
7448     // Get the actual register value type.  This is important, because the user
7449     // may have asked for (e.g.) the AX register in i32 type.  We need to
7450     // remember that AX is actually i16 to get the right extension.
7451     RegVT = *TRI.legalclasstypes_begin(*RC);
7452 
7453     // This is an explicit reference to a physical register.
7454     Regs.push_back(AssignedReg);
7455 
7456     // If this is an expanded reference, add the rest of the regs to Regs.
7457     if (NumRegs != 1) {
7458       TargetRegisterClass::iterator I = RC->begin();
7459       for (; *I != AssignedReg; ++I)
7460         assert(I != RC->end() && "Didn't find reg!");
7461 
7462       // Already added the first reg.
7463       --NumRegs; ++I;
7464       for (; NumRegs; --NumRegs, ++I) {
7465         assert(I != RC->end() && "Ran out of registers to allocate!");
7466         Regs.push_back(*I);
7467       }
7468     }
7469 
7470     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
7471     return;
7472   }
7473 
7474   // Otherwise, if this was a reference to an LLVM register class, create vregs
7475   // for this reference.
7476   if (const TargetRegisterClass *RC = PhysReg.second) {
7477     RegVT = *TRI.legalclasstypes_begin(*RC);
7478     if (OpInfo.ConstraintVT == MVT::Other)
7479       ValueVT = RegVT;
7480 
7481     // Create the appropriate number of virtual registers.
7482     MachineRegisterInfo &RegInfo = MF.getRegInfo();
7483     for (; NumRegs; --NumRegs)
7484       Regs.push_back(RegInfo.createVirtualRegister(RC));
7485 
7486     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
7487     return;
7488   }
7489 
7490   // Otherwise, we couldn't allocate enough registers for this.
7491 }
7492 
7493 static unsigned
7494 findMatchingInlineAsmOperand(unsigned OperandNo,
7495                              const std::vector<SDValue> &AsmNodeOperands) {
7496   // Scan until we find the definition we already emitted of this operand.
7497   unsigned CurOp = InlineAsm::Op_FirstOperand;
7498   for (; OperandNo; --OperandNo) {
7499     // Advance to the next operand.
7500     unsigned OpFlag =
7501         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7502     assert((InlineAsm::isRegDefKind(OpFlag) ||
7503             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
7504             InlineAsm::isMemKind(OpFlag)) &&
7505            "Skipped past definitions?");
7506     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
7507   }
7508   return CurOp;
7509 }
7510 
7511 /// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
7512 /// \return true if it has succeeded, false otherwise
7513 static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
7514                               MVT RegVT, SelectionDAG &DAG) {
7515   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7516   MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
7517   for (unsigned i = 0, e = NumRegs; i != e; ++i) {
7518     if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
7519       Regs.push_back(RegInfo.createVirtualRegister(RC));
7520     else
7521       return false;
7522   }
7523   return true;
7524 }
7525 
7526 namespace {
7527 
7528 class ExtraFlags {
7529   unsigned Flags = 0;
7530 
7531 public:
7532   explicit ExtraFlags(ImmutableCallSite CS) {
7533     const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7534     if (IA->hasSideEffects())
7535       Flags |= InlineAsm::Extra_HasSideEffects;
7536     if (IA->isAlignStack())
7537       Flags |= InlineAsm::Extra_IsAlignStack;
7538     if (CS.isConvergent())
7539       Flags |= InlineAsm::Extra_IsConvergent;
7540     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
7541   }
7542 
7543   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
7544     // Ideally, we would only check against memory constraints.  However, the
7545     // meaning of an Other constraint can be target-specific and we can't easily
7546     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
7547     // for Other constraints as well.
7548     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
7549         OpInfo.ConstraintType == TargetLowering::C_Other) {
7550       if (OpInfo.Type == InlineAsm::isInput)
7551         Flags |= InlineAsm::Extra_MayLoad;
7552       else if (OpInfo.Type == InlineAsm::isOutput)
7553         Flags |= InlineAsm::Extra_MayStore;
7554       else if (OpInfo.Type == InlineAsm::isClobber)
7555         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
7556     }
7557   }
7558 
7559   unsigned get() const { return Flags; }
7560 };
7561 
7562 } // end anonymous namespace
7563 
7564 /// visitInlineAsm - Handle a call to an InlineAsm object.
7565 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
7566   const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7567 
7568   /// ConstraintOperands - Information about all of the constraints.
7569   SDISelAsmOperandInfoVector ConstraintOperands;
7570 
7571   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7572   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
7573       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
7574 
7575   bool hasMemory = false;
7576 
7577   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7578   ExtraFlags ExtraInfo(CS);
7579 
7580   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
7581   unsigned ResNo = 0;   // ResNo - The result number of the next output.
7582   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
7583     ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
7584     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
7585 
7586     MVT OpVT = MVT::Other;
7587 
7588     // Compute the value type for each operand.
7589     if (OpInfo.Type == InlineAsm::isInput ||
7590         (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
7591       OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
7592 
7593       // Process the call argument. BasicBlocks are labels, currently appearing
7594       // only in asm's.
7595       if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
7596         OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
7597       } else {
7598         OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
7599       }
7600 
7601       OpVT =
7602           OpInfo
7603               .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
7604               .getSimpleVT();
7605     }
7606 
7607     if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
7608       // The return value of the call is this value.  As such, there is no
7609       // corresponding argument.
7610       assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7611       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
7612         OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
7613                                       STy->getElementType(ResNo));
7614       } else {
7615         assert(ResNo == 0 && "Asm only has one result!");
7616         OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
7617       }
7618       ++ResNo;
7619     }
7620 
7621     OpInfo.ConstraintVT = OpVT;
7622 
7623     if (!hasMemory)
7624       hasMemory = OpInfo.hasMemory(TLI);
7625 
7626     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
7627     // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
7628     auto TargetConstraint = TargetConstraints[i];
7629 
7630     // Compute the constraint code and ConstraintType to use.
7631     TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
7632 
7633     ExtraInfo.update(TargetConstraint);
7634   }
7635 
7636   SDValue Chain, Flag;
7637 
7638   // We won't need to flush pending loads if this asm doesn't touch
7639   // memory and is nonvolatile.
7640   if (hasMemory || IA->hasSideEffects())
7641     Chain = getRoot();
7642   else
7643     Chain = DAG.getRoot();
7644 
7645   // Second pass over the constraints: compute which constraint option to use
7646   // and assign registers to constraints that want a specific physreg.
7647   for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7648     SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7649 
7650     // If this is an output operand with a matching input operand, look up the
7651     // matching input. If their types mismatch, e.g. one is an integer, the
7652     // other is floating point, or their sizes are different, flag it as an
7653     // error.
7654     if (OpInfo.hasMatchingInput()) {
7655       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
7656       patchMatchingInput(OpInfo, Input, DAG);
7657     }
7658 
7659     // Compute the constraint code and ConstraintType to use.
7660     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
7661 
7662     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7663         OpInfo.Type == InlineAsm::isClobber)
7664       continue;
7665 
7666     // If this is a memory input, and if the operand is not indirect, do what we
7667     // need to provide an address for the memory input.
7668     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7669         !OpInfo.isIndirect) {
7670       assert((OpInfo.isMultipleAlternative ||
7671               (OpInfo.Type == InlineAsm::isInput)) &&
7672              "Can only indirectify direct input operands!");
7673 
7674       // Memory operands really want the address of the value.
7675       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
7676 
7677       // There is no longer a Value* corresponding to this operand.
7678       OpInfo.CallOperandVal = nullptr;
7679 
7680       // It is now an indirect operand.
7681       OpInfo.isIndirect = true;
7682     }
7683 
7684     // If this constraint is for a specific register, allocate it before
7685     // anything else.
7686     SDISelAsmOperandInfo &RefOpInfo =
7687         OpInfo.isMatchingInputConstraint()
7688             ? ConstraintOperands[OpInfo.getMatchedOperand()]
7689             : ConstraintOperands[i];
7690     if (RefOpInfo.ConstraintType == TargetLowering::C_Register)
7691       GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo, RefOpInfo);
7692   }
7693 
7694   // Third pass - Loop over all of the operands, assigning virtual or physregs
7695   // to register class operands.
7696   for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7697     SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7698     SDISelAsmOperandInfo &RefOpInfo =
7699         OpInfo.isMatchingInputConstraint()
7700             ? ConstraintOperands[OpInfo.getMatchedOperand()]
7701             : ConstraintOperands[i];
7702 
7703     // C_Register operands have already been allocated, Other/Memory don't need
7704     // to be.
7705     if (RefOpInfo.ConstraintType == TargetLowering::C_RegisterClass)
7706       GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo, RefOpInfo);
7707   }
7708 
7709   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
7710   std::vector<SDValue> AsmNodeOperands;
7711   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
7712   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
7713       IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
7714 
7715   // If we have a !srcloc metadata node associated with it, we want to attach
7716   // this to the ultimately generated inline asm machineinstr.  To do this, we
7717   // pass in the third operand as this (potentially null) inline asm MDNode.
7718   const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
7719   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
7720 
7721   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7722   // bits as operand 3.
7723   AsmNodeOperands.push_back(DAG.getTargetConstant(
7724       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7725 
7726   // Loop over all of the inputs, copying the operand values into the
7727   // appropriate registers and processing the output regs.
7728   RegsForValue RetValRegs;
7729 
7730   // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
7731   std::vector<std::pair<RegsForValue, Value *>> IndirectStoresToEmit;
7732 
7733   for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7734     SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7735 
7736     switch (OpInfo.Type) {
7737     case InlineAsm::isOutput:
7738       if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7739           OpInfo.ConstraintType != TargetLowering::C_Register) {
7740         // Memory output, or 'other' output (e.g. 'X' constraint).
7741         assert(OpInfo.isIndirect && "Memory output must be indirect operand");
7742 
7743         unsigned ConstraintID =
7744             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7745         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7746                "Failed to convert memory constraint code to constraint id.");
7747 
7748         // Add information to the INLINEASM node to know about this output.
7749         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7750         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7751         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7752                                                         MVT::i32));
7753         AsmNodeOperands.push_back(OpInfo.CallOperand);
7754         break;
7755       }
7756 
7757       // Otherwise, this is a register or register class output.
7758 
7759       // Copy the output from the appropriate register.  Find a register that
7760       // we can use.
7761       if (OpInfo.AssignedRegs.Regs.empty()) {
7762         emitInlineAsmError(
7763             CS, "couldn't allocate output register for constraint '" +
7764                     Twine(OpInfo.ConstraintCode) + "'");
7765         return;
7766       }
7767 
7768       // If this is an indirect operand, store through the pointer after the
7769       // asm.
7770       if (OpInfo.isIndirect) {
7771         IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7772                                                       OpInfo.CallOperandVal));
7773       } else {
7774         // This is the result value of the call.
7775         assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7776         // Concatenate this output onto the outputs list.
7777         RetValRegs.append(OpInfo.AssignedRegs);
7778       }
7779 
7780       // Add information to the INLINEASM node to know that this register is
7781       // set.
7782       OpInfo.AssignedRegs
7783           .AddInlineAsmOperands(OpInfo.isEarlyClobber
7784                                     ? InlineAsm::Kind_RegDefEarlyClobber
7785                                     : InlineAsm::Kind_RegDef,
7786                                 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7787       break;
7788 
7789     case InlineAsm::isInput: {
7790       SDValue InOperandVal = OpInfo.CallOperand;
7791 
7792       if (OpInfo.isMatchingInputConstraint()) {
7793         // If this is required to match an output register we have already set,
7794         // just use its register.
7795         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7796                                                   AsmNodeOperands);
7797         unsigned OpFlag =
7798           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7799         if (InlineAsm::isRegDefKind(OpFlag) ||
7800             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7801           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7802           if (OpInfo.isIndirect) {
7803             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7804             emitInlineAsmError(CS, "inline asm not supported yet:"
7805                                    " don't know how to handle tied "
7806                                    "indirect register inputs");
7807             return;
7808           }
7809 
7810           MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7811           SmallVector<unsigned, 4> Regs;
7812 
7813           if (!createVirtualRegs(Regs,
7814                                  InlineAsm::getNumOperandRegisters(OpFlag),
7815                                  RegVT, DAG)) {
7816             emitInlineAsmError(CS, "inline asm error: This value type register "
7817                                    "class is not natively supported!");
7818             return;
7819           }
7820 
7821           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7822 
7823           SDLoc dl = getCurSDLoc();
7824           // Use the produced MatchedRegs object to
7825           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
7826                                     CS.getInstruction());
7827           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7828                                            true, OpInfo.getMatchedOperand(), dl,
7829                                            DAG, AsmNodeOperands);
7830           break;
7831         }
7832 
7833         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
7834         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
7835                "Unexpected number of operands");
7836         // Add information to the INLINEASM node to know about this input.
7837         // See InlineAsm.h isUseOperandTiedToDef.
7838         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7839         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7840                                                     OpInfo.getMatchedOperand());
7841         AsmNodeOperands.push_back(DAG.getTargetConstant(
7842             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7843         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7844         break;
7845       }
7846 
7847       // Treat indirect 'X' constraint as memory.
7848       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7849           OpInfo.isIndirect)
7850         OpInfo.ConstraintType = TargetLowering::C_Memory;
7851 
7852       if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7853         std::vector<SDValue> Ops;
7854         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7855                                           Ops, DAG);
7856         if (Ops.empty()) {
7857           emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7858                                      Twine(OpInfo.ConstraintCode) + "'");
7859           return;
7860         }
7861 
7862         // Add information to the INLINEASM node to know about this input.
7863         unsigned ResOpType =
7864           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7865         AsmNodeOperands.push_back(DAG.getTargetConstant(
7866             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7867         AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7868         break;
7869       }
7870 
7871       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7872         assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
7873         assert(InOperandVal.getValueType() ==
7874                    TLI.getPointerTy(DAG.getDataLayout()) &&
7875                "Memory operands expect pointer values");
7876 
7877         unsigned ConstraintID =
7878             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7879         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7880                "Failed to convert memory constraint code to constraint id.");
7881 
7882         // Add information to the INLINEASM node to know about this input.
7883         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7884         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7885         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7886                                                         getCurSDLoc(),
7887                                                         MVT::i32));
7888         AsmNodeOperands.push_back(InOperandVal);
7889         break;
7890       }
7891 
7892       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
7893               OpInfo.ConstraintType == TargetLowering::C_Register) &&
7894              "Unknown constraint type!");
7895 
7896       // TODO: Support this.
7897       if (OpInfo.isIndirect) {
7898         emitInlineAsmError(
7899             CS, "Don't know how to handle indirect register inputs yet "
7900                 "for constraint '" +
7901                     Twine(OpInfo.ConstraintCode) + "'");
7902         return;
7903       }
7904 
7905       // Copy the input into the appropriate registers.
7906       if (OpInfo.AssignedRegs.Regs.empty()) {
7907         emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7908                                    Twine(OpInfo.ConstraintCode) + "'");
7909         return;
7910       }
7911 
7912       SDLoc dl = getCurSDLoc();
7913 
7914       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7915                                         Chain, &Flag, CS.getInstruction());
7916 
7917       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7918                                                dl, DAG, AsmNodeOperands);
7919       break;
7920     }
7921     case InlineAsm::isClobber:
7922       // Add the clobbered value to the operand list, so that the register
7923       // allocator is aware that the physreg got clobbered.
7924       if (!OpInfo.AssignedRegs.Regs.empty())
7925         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7926                                                  false, 0, getCurSDLoc(), DAG,
7927                                                  AsmNodeOperands);
7928       break;
7929     }
7930   }
7931 
7932   // Finish up input operands.  Set the input chain and add the flag last.
7933   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7934   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7935 
7936   Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7937                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7938   Flag = Chain.getValue(1);
7939 
7940   // If this asm returns a register value, copy the result from that register
7941   // and set it as the value of the call.
7942   if (!RetValRegs.Regs.empty()) {
7943     SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7944                                              Chain, &Flag, CS.getInstruction());
7945 
7946     llvm::Type *CSResultType = CS.getType();
7947     unsigned numRet;
7948     ArrayRef<Type *> ResultTypes;
7949     SmallVector<SDValue, 1> ResultValues(1);
7950     if (CSResultType->isSingleValueType()) {
7951       numRet = 1;
7952       ResultValues[0] = Val;
7953       ResultTypes = makeArrayRef(CSResultType);
7954     } else {
7955       numRet = CSResultType->getNumContainedTypes();
7956       assert(Val->getNumOperands() == numRet &&
7957              "Mismatch in number of output operands in asm result");
7958       ResultTypes = CSResultType->subtypes();
7959       ArrayRef<SDUse> ValueUses = Val->ops();
7960       ResultValues.resize(numRet);
7961       std::transform(ValueUses.begin(), ValueUses.end(), ResultValues.begin(),
7962                      [](const SDUse &u) -> SDValue { return u.get(); });
7963     }
7964     SmallVector<EVT, 1> ResultVTs(numRet);
7965     for (unsigned i = 0; i < numRet; i++) {
7966       EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), ResultTypes[i]);
7967       SDValue Val = ResultValues[i];
7968       assert(ResultTypes[i]->isSized() && "Unexpected unsized type");
7969       // If the type of the inline asm call site return value is different but
7970       // has same size as the type of the asm output bitcast it.  One example
7971       // of this is for vectors with different width / number of elements.
7972       // This can happen for register classes that can contain multiple
7973       // different value types.  The preg or vreg allocated may not have the
7974       // same VT as was expected.
7975       //
7976       // This can also happen for a return value that disagrees with the
7977       // register class it is put in, eg. a double in a general-purpose
7978       // register on a 32-bit machine.
7979       if (ResultVT != Val.getValueType() &&
7980           ResultVT.getSizeInBits() == Val.getValueSizeInBits())
7981         Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, Val);
7982       else if (ResultVT != Val.getValueType() && ResultVT.isInteger() &&
7983                Val.getValueType().isInteger()) {
7984         // If a result value was tied to an input value, the computed result
7985         // may have a wider width than the expected result.  Extract the
7986         // relevant portion.
7987         Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, Val);
7988       }
7989 
7990       assert(ResultVT == Val.getValueType() && "Asm result value mismatch!");
7991       ResultVTs[i] = ResultVT;
7992       ResultValues[i] = Val;
7993     }
7994 
7995     Val = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
7996                       DAG.getVTList(ResultVTs), ResultValues);
7997     setValue(CS.getInstruction(), Val);
7998     // Don't need to use this as a chain in this case.
7999     if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
8000       return;
8001   }
8002 
8003   std::vector<std::pair<SDValue, const Value *>> StoresToEmit;
8004 
8005   // Process indirect outputs, first output all of the flagged copies out of
8006   // physregs.
8007   for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
8008     RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
8009     const Value *Ptr = IndirectStoresToEmit[i].second;
8010     SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
8011                                              Chain, &Flag, IA);
8012     StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
8013   }
8014 
8015   // Emit the non-flagged stores from the physregs.
8016   SmallVector<SDValue, 8> OutChains;
8017   for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
8018     SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
8019                                getValue(StoresToEmit[i].second),
8020                                MachinePointerInfo(StoresToEmit[i].second));
8021     OutChains.push_back(Val);
8022   }
8023 
8024   if (!OutChains.empty())
8025     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
8026 
8027   DAG.setRoot(Chain);
8028 }
8029 
8030 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
8031                                              const Twine &Message) {
8032   LLVMContext &Ctx = *DAG.getContext();
8033   Ctx.emitError(CS.getInstruction(), Message);
8034 
8035   // Make sure we leave the DAG in a valid state
8036   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8037   SmallVector<EVT, 1> ValueVTs;
8038   ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
8039 
8040   if (ValueVTs.empty())
8041     return;
8042 
8043   SmallVector<SDValue, 1> Ops;
8044   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
8045     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
8046 
8047   setValue(CS.getInstruction(), DAG.getMergeValues(Ops, getCurSDLoc()));
8048 }
8049 
8050 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
8051   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
8052                           MVT::Other, getRoot(),
8053                           getValue(I.getArgOperand(0)),
8054                           DAG.getSrcValue(I.getArgOperand(0))));
8055 }
8056 
8057 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
8058   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8059   const DataLayout &DL = DAG.getDataLayout();
8060   SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
8061                            getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
8062                            DAG.getSrcValue(I.getOperand(0)),
8063                            DL.getABITypeAlignment(I.getType()));
8064   setValue(&I, V);
8065   DAG.setRoot(V.getValue(1));
8066 }
8067 
8068 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
8069   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
8070                           MVT::Other, getRoot(),
8071                           getValue(I.getArgOperand(0)),
8072                           DAG.getSrcValue(I.getArgOperand(0))));
8073 }
8074 
8075 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
8076   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
8077                           MVT::Other, getRoot(),
8078                           getValue(I.getArgOperand(0)),
8079                           getValue(I.getArgOperand(1)),
8080                           DAG.getSrcValue(I.getArgOperand(0)),
8081                           DAG.getSrcValue(I.getArgOperand(1))));
8082 }
8083 
8084 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
8085                                                     const Instruction &I,
8086                                                     SDValue Op) {
8087   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
8088   if (!Range)
8089     return Op;
8090 
8091   ConstantRange CR = getConstantRangeFromMetadata(*Range);
8092   if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
8093     return Op;
8094 
8095   APInt Lo = CR.getUnsignedMin();
8096   if (!Lo.isMinValue())
8097     return Op;
8098 
8099   APInt Hi = CR.getUnsignedMax();
8100   unsigned Bits = std::max(Hi.getActiveBits(),
8101                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
8102 
8103   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
8104 
8105   SDLoc SL = getCurSDLoc();
8106 
8107   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
8108                              DAG.getValueType(SmallVT));
8109   unsigned NumVals = Op.getNode()->getNumValues();
8110   if (NumVals == 1)
8111     return ZExt;
8112 
8113   SmallVector<SDValue, 4> Ops;
8114 
8115   Ops.push_back(ZExt);
8116   for (unsigned I = 1; I != NumVals; ++I)
8117     Ops.push_back(Op.getValue(I));
8118 
8119   return DAG.getMergeValues(Ops, SL);
8120 }
8121 
8122 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
8123 /// the call being lowered.
8124 ///
8125 /// This is a helper for lowering intrinsics that follow a target calling
8126 /// convention or require stack pointer adjustment. Only a subset of the
8127 /// intrinsic's operands need to participate in the calling convention.
8128 void SelectionDAGBuilder::populateCallLoweringInfo(
8129     TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
8130     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
8131     bool IsPatchPoint) {
8132   TargetLowering::ArgListTy Args;
8133   Args.reserve(NumArgs);
8134 
8135   // Populate the argument list.
8136   // Attributes for args start at offset 1, after the return attribute.
8137   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
8138        ArgI != ArgE; ++ArgI) {
8139     const Value *V = CS->getOperand(ArgI);
8140 
8141     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
8142 
8143     TargetLowering::ArgListEntry Entry;
8144     Entry.Node = getValue(V);
8145     Entry.Ty = V->getType();
8146     Entry.setAttributes(&CS, ArgI);
8147     Args.push_back(Entry);
8148   }
8149 
8150   CLI.setDebugLoc(getCurSDLoc())
8151       .setChain(getRoot())
8152       .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
8153       .setDiscardResult(CS->use_empty())
8154       .setIsPatchPoint(IsPatchPoint);
8155 }
8156 
8157 /// Add a stack map intrinsic call's live variable operands to a stackmap
8158 /// or patchpoint target node's operand list.
8159 ///
8160 /// Constants are converted to TargetConstants purely as an optimization to
8161 /// avoid constant materialization and register allocation.
8162 ///
8163 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
8164 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
8165 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
8166 /// address materialization and register allocation, but may also be required
8167 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
8168 /// alloca in the entry block, then the runtime may assume that the alloca's
8169 /// StackMap location can be read immediately after compilation and that the
8170 /// location is valid at any point during execution (this is similar to the
8171 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
8172 /// only available in a register, then the runtime would need to trap when
8173 /// execution reaches the StackMap in order to read the alloca's location.
8174 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
8175                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
8176                                 SelectionDAGBuilder &Builder) {
8177   for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
8178     SDValue OpVal = Builder.getValue(CS.getArgument(i));
8179     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
8180       Ops.push_back(
8181         Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
8182       Ops.push_back(
8183         Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
8184     } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
8185       const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
8186       Ops.push_back(Builder.DAG.getTargetFrameIndex(
8187           FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
8188     } else
8189       Ops.push_back(OpVal);
8190   }
8191 }
8192 
8193 /// Lower llvm.experimental.stackmap directly to its target opcode.
8194 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
8195   // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
8196   //                                  [live variables...])
8197 
8198   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
8199 
8200   SDValue Chain, InFlag, Callee, NullPtr;
8201   SmallVector<SDValue, 32> Ops;
8202 
8203   SDLoc DL = getCurSDLoc();
8204   Callee = getValue(CI.getCalledValue());
8205   NullPtr = DAG.getIntPtrConstant(0, DL, true);
8206 
8207   // The stackmap intrinsic only records the live variables (the arguemnts
8208   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
8209   // intrinsic, this won't be lowered to a function call. This means we don't
8210   // have to worry about calling conventions and target specific lowering code.
8211   // Instead we perform the call lowering right here.
8212   //
8213   // chain, flag = CALLSEQ_START(chain, 0, 0)
8214   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
8215   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
8216   //
8217   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
8218   InFlag = Chain.getValue(1);
8219 
8220   // Add the <id> and <numBytes> constants.
8221   SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
8222   Ops.push_back(DAG.getTargetConstant(
8223                   cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
8224   SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
8225   Ops.push_back(DAG.getTargetConstant(
8226                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
8227                   MVT::i32));
8228 
8229   // Push live variables for the stack map.
8230   addStackMapLiveVars(&CI, 2, DL, Ops, *this);
8231 
8232   // We are not pushing any register mask info here on the operands list,
8233   // because the stackmap doesn't clobber anything.
8234 
8235   // Push the chain and the glue flag.
8236   Ops.push_back(Chain);
8237   Ops.push_back(InFlag);
8238 
8239   // Create the STACKMAP node.
8240   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8241   SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
8242   Chain = SDValue(SM, 0);
8243   InFlag = Chain.getValue(1);
8244 
8245   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
8246 
8247   // Stackmaps don't generate values, so nothing goes into the NodeMap.
8248 
8249   // Set the root to the target-lowered call chain.
8250   DAG.setRoot(Chain);
8251 
8252   // Inform the Frame Information that we have a stackmap in this function.
8253   FuncInfo.MF->getFrameInfo().setHasStackMap();
8254 }
8255 
8256 /// Lower llvm.experimental.patchpoint directly to its target opcode.
8257 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
8258                                           const BasicBlock *EHPadBB) {
8259   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
8260   //                                                 i32 <numBytes>,
8261   //                                                 i8* <target>,
8262   //                                                 i32 <numArgs>,
8263   //                                                 [Args...],
8264   //                                                 [live variables...])
8265 
8266   CallingConv::ID CC = CS.getCallingConv();
8267   bool IsAnyRegCC = CC == CallingConv::AnyReg;
8268   bool HasDef = !CS->getType()->isVoidTy();
8269   SDLoc dl = getCurSDLoc();
8270   SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
8271 
8272   // Handle immediate and symbolic callees.
8273   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
8274     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
8275                                    /*isTarget=*/true);
8276   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
8277     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
8278                                          SDLoc(SymbolicCallee),
8279                                          SymbolicCallee->getValueType(0));
8280 
8281   // Get the real number of arguments participating in the call <numArgs>
8282   SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
8283   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
8284 
8285   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
8286   // Intrinsics include all meta-operands up to but not including CC.
8287   unsigned NumMetaOpers = PatchPointOpers::CCPos;
8288   assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
8289          "Not enough arguments provided to the patchpoint intrinsic");
8290 
8291   // For AnyRegCC the arguments are lowered later on manually.
8292   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
8293   Type *ReturnTy =
8294     IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
8295 
8296   TargetLowering::CallLoweringInfo CLI(DAG);
8297   populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
8298                            true);
8299   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8300 
8301   SDNode *CallEnd = Result.second.getNode();
8302   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
8303     CallEnd = CallEnd->getOperand(0).getNode();
8304 
8305   /// Get a call instruction from the call sequence chain.
8306   /// Tail calls are not allowed.
8307   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
8308          "Expected a callseq node.");
8309   SDNode *Call = CallEnd->getOperand(0).getNode();
8310   bool HasGlue = Call->getGluedNode();
8311 
8312   // Replace the target specific call node with the patchable intrinsic.
8313   SmallVector<SDValue, 8> Ops;
8314 
8315   // Add the <id> and <numBytes> constants.
8316   SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
8317   Ops.push_back(DAG.getTargetConstant(
8318                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
8319   SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
8320   Ops.push_back(DAG.getTargetConstant(
8321                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
8322                   MVT::i32));
8323 
8324   // Add the callee.
8325   Ops.push_back(Callee);
8326 
8327   // Adjust <numArgs> to account for any arguments that have been passed on the
8328   // stack instead.
8329   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
8330   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
8331   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
8332   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
8333 
8334   // Add the calling convention
8335   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
8336 
8337   // Add the arguments we omitted previously. The register allocator should
8338   // place these in any free register.
8339   if (IsAnyRegCC)
8340     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
8341       Ops.push_back(getValue(CS.getArgument(i)));
8342 
8343   // Push the arguments from the call instruction up to the register mask.
8344   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
8345   Ops.append(Call->op_begin() + 2, e);
8346 
8347   // Push live variables for the stack map.
8348   addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
8349 
8350   // Push the register mask info.
8351   if (HasGlue)
8352     Ops.push_back(*(Call->op_end()-2));
8353   else
8354     Ops.push_back(*(Call->op_end()-1));
8355 
8356   // Push the chain (this is originally the first operand of the call, but
8357   // becomes now the last or second to last operand).
8358   Ops.push_back(*(Call->op_begin()));
8359 
8360   // Push the glue flag (last operand).
8361   if (HasGlue)
8362     Ops.push_back(*(Call->op_end()-1));
8363 
8364   SDVTList NodeTys;
8365   if (IsAnyRegCC && HasDef) {
8366     // Create the return types based on the intrinsic definition
8367     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8368     SmallVector<EVT, 3> ValueVTs;
8369     ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
8370     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
8371 
8372     // There is always a chain and a glue type at the end
8373     ValueVTs.push_back(MVT::Other);
8374     ValueVTs.push_back(MVT::Glue);
8375     NodeTys = DAG.getVTList(ValueVTs);
8376   } else
8377     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8378 
8379   // Replace the target specific call node with a PATCHPOINT node.
8380   MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
8381                                          dl, NodeTys, Ops);
8382 
8383   // Update the NodeMap.
8384   if (HasDef) {
8385     if (IsAnyRegCC)
8386       setValue(CS.getInstruction(), SDValue(MN, 0));
8387     else
8388       setValue(CS.getInstruction(), Result.first);
8389   }
8390 
8391   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
8392   // call sequence. Furthermore the location of the chain and glue can change
8393   // when the AnyReg calling convention is used and the intrinsic returns a
8394   // value.
8395   if (IsAnyRegCC && HasDef) {
8396     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
8397     SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
8398     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
8399   } else
8400     DAG.ReplaceAllUsesWith(Call, MN);
8401   DAG.DeleteNode(Call);
8402 
8403   // Inform the Frame Information that we have a patchpoint in this function.
8404   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
8405 }
8406 
8407 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
8408                                             unsigned Intrinsic) {
8409   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8410   SDValue Op1 = getValue(I.getArgOperand(0));
8411   SDValue Op2;
8412   if (I.getNumArgOperands() > 1)
8413     Op2 = getValue(I.getArgOperand(1));
8414   SDLoc dl = getCurSDLoc();
8415   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8416   SDValue Res;
8417   FastMathFlags FMF;
8418   if (isa<FPMathOperator>(I))
8419     FMF = I.getFastMathFlags();
8420 
8421   switch (Intrinsic) {
8422   case Intrinsic::experimental_vector_reduce_fadd:
8423     if (FMF.isFast())
8424       Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2);
8425     else
8426       Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
8427     break;
8428   case Intrinsic::experimental_vector_reduce_fmul:
8429     if (FMF.isFast())
8430       Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2);
8431     else
8432       Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
8433     break;
8434   case Intrinsic::experimental_vector_reduce_add:
8435     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
8436     break;
8437   case Intrinsic::experimental_vector_reduce_mul:
8438     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
8439     break;
8440   case Intrinsic::experimental_vector_reduce_and:
8441     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
8442     break;
8443   case Intrinsic::experimental_vector_reduce_or:
8444     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
8445     break;
8446   case Intrinsic::experimental_vector_reduce_xor:
8447     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
8448     break;
8449   case Intrinsic::experimental_vector_reduce_smax:
8450     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
8451     break;
8452   case Intrinsic::experimental_vector_reduce_smin:
8453     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
8454     break;
8455   case Intrinsic::experimental_vector_reduce_umax:
8456     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
8457     break;
8458   case Intrinsic::experimental_vector_reduce_umin:
8459     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
8460     break;
8461   case Intrinsic::experimental_vector_reduce_fmax:
8462     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1);
8463     break;
8464   case Intrinsic::experimental_vector_reduce_fmin:
8465     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1);
8466     break;
8467   default:
8468     llvm_unreachable("Unhandled vector reduce intrinsic");
8469   }
8470   setValue(&I, Res);
8471 }
8472 
8473 /// Returns an AttributeList representing the attributes applied to the return
8474 /// value of the given call.
8475 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
8476   SmallVector<Attribute::AttrKind, 2> Attrs;
8477   if (CLI.RetSExt)
8478     Attrs.push_back(Attribute::SExt);
8479   if (CLI.RetZExt)
8480     Attrs.push_back(Attribute::ZExt);
8481   if (CLI.IsInReg)
8482     Attrs.push_back(Attribute::InReg);
8483 
8484   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
8485                             Attrs);
8486 }
8487 
8488 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
8489 /// implementation, which just calls LowerCall.
8490 /// FIXME: When all targets are
8491 /// migrated to using LowerCall, this hook should be integrated into SDISel.
8492 std::pair<SDValue, SDValue>
8493 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
8494   // Handle the incoming return values from the call.
8495   CLI.Ins.clear();
8496   Type *OrigRetTy = CLI.RetTy;
8497   SmallVector<EVT, 4> RetTys;
8498   SmallVector<uint64_t, 4> Offsets;
8499   auto &DL = CLI.DAG.getDataLayout();
8500   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
8501 
8502   if (CLI.IsPostTypeLegalization) {
8503     // If we are lowering a libcall after legalization, split the return type.
8504     SmallVector<EVT, 4> OldRetTys = std::move(RetTys);
8505     SmallVector<uint64_t, 4> OldOffsets = std::move(Offsets);
8506     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
8507       EVT RetVT = OldRetTys[i];
8508       uint64_t Offset = OldOffsets[i];
8509       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
8510       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
8511       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
8512       RetTys.append(NumRegs, RegisterVT);
8513       for (unsigned j = 0; j != NumRegs; ++j)
8514         Offsets.push_back(Offset + j * RegisterVTByteSZ);
8515     }
8516   }
8517 
8518   SmallVector<ISD::OutputArg, 4> Outs;
8519   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
8520 
8521   bool CanLowerReturn =
8522       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
8523                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
8524 
8525   SDValue DemoteStackSlot;
8526   int DemoteStackIdx = -100;
8527   if (!CanLowerReturn) {
8528     // FIXME: equivalent assert?
8529     // assert(!CS.hasInAllocaArgument() &&
8530     //        "sret demotion is incompatible with inalloca");
8531     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
8532     unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
8533     MachineFunction &MF = CLI.DAG.getMachineFunction();
8534     DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
8535     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
8536                                               DL.getAllocaAddrSpace());
8537 
8538     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
8539     ArgListEntry Entry;
8540     Entry.Node = DemoteStackSlot;
8541     Entry.Ty = StackSlotPtrType;
8542     Entry.IsSExt = false;
8543     Entry.IsZExt = false;
8544     Entry.IsInReg = false;
8545     Entry.IsSRet = true;
8546     Entry.IsNest = false;
8547     Entry.IsByVal = false;
8548     Entry.IsReturned = false;
8549     Entry.IsSwiftSelf = false;
8550     Entry.IsSwiftError = false;
8551     Entry.Alignment = Align;
8552     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
8553     CLI.NumFixedArgs += 1;
8554     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
8555 
8556     // sret demotion isn't compatible with tail-calls, since the sret argument
8557     // points into the callers stack frame.
8558     CLI.IsTailCall = false;
8559   } else {
8560     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8561       EVT VT = RetTys[I];
8562       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8563                                                      CLI.CallConv, VT);
8564       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8565                                                        CLI.CallConv, VT);
8566       for (unsigned i = 0; i != NumRegs; ++i) {
8567         ISD::InputArg MyFlags;
8568         MyFlags.VT = RegisterVT;
8569         MyFlags.ArgVT = VT;
8570         MyFlags.Used = CLI.IsReturnValueUsed;
8571         if (CLI.RetSExt)
8572           MyFlags.Flags.setSExt();
8573         if (CLI.RetZExt)
8574           MyFlags.Flags.setZExt();
8575         if (CLI.IsInReg)
8576           MyFlags.Flags.setInReg();
8577         CLI.Ins.push_back(MyFlags);
8578       }
8579     }
8580   }
8581 
8582   // We push in swifterror return as the last element of CLI.Ins.
8583   ArgListTy &Args = CLI.getArgs();
8584   if (supportSwiftError()) {
8585     for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8586       if (Args[i].IsSwiftError) {
8587         ISD::InputArg MyFlags;
8588         MyFlags.VT = getPointerTy(DL);
8589         MyFlags.ArgVT = EVT(getPointerTy(DL));
8590         MyFlags.Flags.setSwiftError();
8591         CLI.Ins.push_back(MyFlags);
8592       }
8593     }
8594   }
8595 
8596   // Handle all of the outgoing arguments.
8597   CLI.Outs.clear();
8598   CLI.OutVals.clear();
8599   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8600     SmallVector<EVT, 4> ValueVTs;
8601     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
8602     // FIXME: Split arguments if CLI.IsPostTypeLegalization
8603     Type *FinalType = Args[i].Ty;
8604     if (Args[i].IsByVal)
8605       FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
8606     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
8607         FinalType, CLI.CallConv, CLI.IsVarArg);
8608     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
8609          ++Value) {
8610       EVT VT = ValueVTs[Value];
8611       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
8612       SDValue Op = SDValue(Args[i].Node.getNode(),
8613                            Args[i].Node.getResNo() + Value);
8614       ISD::ArgFlagsTy Flags;
8615 
8616       // Certain targets (such as MIPS), may have a different ABI alignment
8617       // for a type depending on the context. Give the target a chance to
8618       // specify the alignment it wants.
8619       unsigned OriginalAlignment = getABIAlignmentForCallingConv(ArgTy, DL);
8620 
8621       if (Args[i].IsZExt)
8622         Flags.setZExt();
8623       if (Args[i].IsSExt)
8624         Flags.setSExt();
8625       if (Args[i].IsInReg) {
8626         // If we are using vectorcall calling convention, a structure that is
8627         // passed InReg - is surely an HVA
8628         if (CLI.CallConv == CallingConv::X86_VectorCall &&
8629             isa<StructType>(FinalType)) {
8630           // The first value of a structure is marked
8631           if (0 == Value)
8632             Flags.setHvaStart();
8633           Flags.setHva();
8634         }
8635         // Set InReg Flag
8636         Flags.setInReg();
8637       }
8638       if (Args[i].IsSRet)
8639         Flags.setSRet();
8640       if (Args[i].IsSwiftSelf)
8641         Flags.setSwiftSelf();
8642       if (Args[i].IsSwiftError)
8643         Flags.setSwiftError();
8644       if (Args[i].IsByVal)
8645         Flags.setByVal();
8646       if (Args[i].IsInAlloca) {
8647         Flags.setInAlloca();
8648         // Set the byval flag for CCAssignFn callbacks that don't know about
8649         // inalloca.  This way we can know how many bytes we should've allocated
8650         // and how many bytes a callee cleanup function will pop.  If we port
8651         // inalloca to more targets, we'll have to add custom inalloca handling
8652         // in the various CC lowering callbacks.
8653         Flags.setByVal();
8654       }
8655       if (Args[i].IsByVal || Args[i].IsInAlloca) {
8656         PointerType *Ty = cast<PointerType>(Args[i].Ty);
8657         Type *ElementTy = Ty->getElementType();
8658         Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8659         // For ByVal, alignment should come from FE.  BE will guess if this
8660         // info is not there but there are cases it cannot get right.
8661         unsigned FrameAlign;
8662         if (Args[i].Alignment)
8663           FrameAlign = Args[i].Alignment;
8664         else
8665           FrameAlign = getByValTypeAlignment(ElementTy, DL);
8666         Flags.setByValAlign(FrameAlign);
8667       }
8668       if (Args[i].IsNest)
8669         Flags.setNest();
8670       if (NeedsRegBlock)
8671         Flags.setInConsecutiveRegs();
8672       Flags.setOrigAlign(OriginalAlignment);
8673 
8674       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8675                                                  CLI.CallConv, VT);
8676       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8677                                                         CLI.CallConv, VT);
8678       SmallVector<SDValue, 4> Parts(NumParts);
8679       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
8680 
8681       if (Args[i].IsSExt)
8682         ExtendKind = ISD::SIGN_EXTEND;
8683       else if (Args[i].IsZExt)
8684         ExtendKind = ISD::ZERO_EXTEND;
8685 
8686       // Conservatively only handle 'returned' on non-vectors that can be lowered,
8687       // for now.
8688       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
8689           CanLowerReturn) {
8690         assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
8691                "unexpected use of 'returned'");
8692         // Before passing 'returned' to the target lowering code, ensure that
8693         // either the register MVT and the actual EVT are the same size or that
8694         // the return value and argument are extended in the same way; in these
8695         // cases it's safe to pass the argument register value unchanged as the
8696         // return register value (although it's at the target's option whether
8697         // to do so)
8698         // TODO: allow code generation to take advantage of partially preserved
8699         // registers rather than clobbering the entire register when the
8700         // parameter extension method is not compatible with the return
8701         // extension method
8702         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
8703             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
8704              CLI.RetZExt == Args[i].IsZExt))
8705           Flags.setReturned();
8706       }
8707 
8708       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
8709                      CLI.CS.getInstruction(), CLI.CallConv, ExtendKind);
8710 
8711       for (unsigned j = 0; j != NumParts; ++j) {
8712         // if it isn't first piece, alignment must be 1
8713         ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
8714                                i < CLI.NumFixedArgs,
8715                                i, j*Parts[j].getValueType().getStoreSize());
8716         if (NumParts > 1 && j == 0)
8717           MyFlags.Flags.setSplit();
8718         else if (j != 0) {
8719           MyFlags.Flags.setOrigAlign(1);
8720           if (j == NumParts - 1)
8721             MyFlags.Flags.setSplitEnd();
8722         }
8723 
8724         CLI.Outs.push_back(MyFlags);
8725         CLI.OutVals.push_back(Parts[j]);
8726       }
8727 
8728       if (NeedsRegBlock && Value == NumValues - 1)
8729         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
8730     }
8731   }
8732 
8733   SmallVector<SDValue, 4> InVals;
8734   CLI.Chain = LowerCall(CLI, InVals);
8735 
8736   // Update CLI.InVals to use outside of this function.
8737   CLI.InVals = InVals;
8738 
8739   // Verify that the target's LowerCall behaved as expected.
8740   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
8741          "LowerCall didn't return a valid chain!");
8742   assert((!CLI.IsTailCall || InVals.empty()) &&
8743          "LowerCall emitted a return value for a tail call!");
8744   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
8745          "LowerCall didn't emit the correct number of values!");
8746 
8747   // For a tail call, the return value is merely live-out and there aren't
8748   // any nodes in the DAG representing it. Return a special value to
8749   // indicate that a tail call has been emitted and no more Instructions
8750   // should be processed in the current block.
8751   if (CLI.IsTailCall) {
8752     CLI.DAG.setRoot(CLI.Chain);
8753     return std::make_pair(SDValue(), SDValue());
8754   }
8755 
8756 #ifndef NDEBUG
8757   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
8758     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
8759     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
8760            "LowerCall emitted a value with the wrong type!");
8761   }
8762 #endif
8763 
8764   SmallVector<SDValue, 4> ReturnValues;
8765   if (!CanLowerReturn) {
8766     // The instruction result is the result of loading from the
8767     // hidden sret parameter.
8768     SmallVector<EVT, 1> PVTs;
8769     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
8770 
8771     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
8772     assert(PVTs.size() == 1 && "Pointers should fit in one register");
8773     EVT PtrVT = PVTs[0];
8774 
8775     unsigned NumValues = RetTys.size();
8776     ReturnValues.resize(NumValues);
8777     SmallVector<SDValue, 4> Chains(NumValues);
8778 
8779     // An aggregate return value cannot wrap around the address space, so
8780     // offsets to its parts don't wrap either.
8781     SDNodeFlags Flags;
8782     Flags.setNoUnsignedWrap(true);
8783 
8784     for (unsigned i = 0; i < NumValues; ++i) {
8785       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
8786                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
8787                                                         PtrVT), Flags);
8788       SDValue L = CLI.DAG.getLoad(
8789           RetTys[i], CLI.DL, CLI.Chain, Add,
8790           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
8791                                             DemoteStackIdx, Offsets[i]),
8792           /* Alignment = */ 1);
8793       ReturnValues[i] = L;
8794       Chains[i] = L.getValue(1);
8795     }
8796 
8797     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
8798   } else {
8799     // Collect the legal value parts into potentially illegal values
8800     // that correspond to the original function's return values.
8801     Optional<ISD::NodeType> AssertOp;
8802     if (CLI.RetSExt)
8803       AssertOp = ISD::AssertSext;
8804     else if (CLI.RetZExt)
8805       AssertOp = ISD::AssertZext;
8806     unsigned CurReg = 0;
8807     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8808       EVT VT = RetTys[I];
8809       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8810                                                      CLI.CallConv, VT);
8811       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8812                                                        CLI.CallConv, VT);
8813 
8814       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
8815                                               NumRegs, RegisterVT, VT, nullptr,
8816                                               CLI.CallConv, AssertOp));
8817       CurReg += NumRegs;
8818     }
8819 
8820     // For a function returning void, there is no return value. We can't create
8821     // such a node, so we just return a null return value in that case. In
8822     // that case, nothing will actually look at the value.
8823     if (ReturnValues.empty())
8824       return std::make_pair(SDValue(), CLI.Chain);
8825   }
8826 
8827   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
8828                                 CLI.DAG.getVTList(RetTys), ReturnValues);
8829   return std::make_pair(Res, CLI.Chain);
8830 }
8831 
8832 void TargetLowering::LowerOperationWrapper(SDNode *N,
8833                                            SmallVectorImpl<SDValue> &Results,
8834                                            SelectionDAG &DAG) const {
8835   if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
8836     Results.push_back(Res);
8837 }
8838 
8839 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
8840   llvm_unreachable("LowerOperation not implemented for this target!");
8841 }
8842 
8843 void
8844 SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
8845   SDValue Op = getNonRegisterValue(V);
8846   assert((Op.getOpcode() != ISD::CopyFromReg ||
8847           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
8848          "Copy from a reg to the same reg!");
8849   assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
8850 
8851   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8852   // If this is an InlineAsm we have to match the registers required, not the
8853   // notional registers required by the type.
8854 
8855   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
8856                    None); // This is not an ABI copy.
8857   SDValue Chain = DAG.getEntryNode();
8858 
8859   ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
8860                               FuncInfo.PreferredExtendType.end())
8861                                  ? ISD::ANY_EXTEND
8862                                  : FuncInfo.PreferredExtendType[V];
8863   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
8864   PendingExports.push_back(Chain);
8865 }
8866 
8867 #include "llvm/CodeGen/SelectionDAGISel.h"
8868 
8869 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
8870 /// entry block, return true.  This includes arguments used by switches, since
8871 /// the switch may expand into multiple basic blocks.
8872 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
8873   // With FastISel active, we may be splitting blocks, so force creation
8874   // of virtual registers for all non-dead arguments.
8875   if (FastISel)
8876     return A->use_empty();
8877 
8878   const BasicBlock &Entry = A->getParent()->front();
8879   for (const User *U : A->users())
8880     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
8881       return false;  // Use not in entry block.
8882 
8883   return true;
8884 }
8885 
8886 using ArgCopyElisionMapTy =
8887     DenseMap<const Argument *,
8888              std::pair<const AllocaInst *, const StoreInst *>>;
8889 
8890 /// Scan the entry block of the function in FuncInfo for arguments that look
8891 /// like copies into a local alloca. Record any copied arguments in
8892 /// ArgCopyElisionCandidates.
8893 static void
8894 findArgumentCopyElisionCandidates(const DataLayout &DL,
8895                                   FunctionLoweringInfo *FuncInfo,
8896                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
8897   // Record the state of every static alloca used in the entry block. Argument
8898   // allocas are all used in the entry block, so we need approximately as many
8899   // entries as we have arguments.
8900   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
8901   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
8902   unsigned NumArgs = FuncInfo->Fn->arg_size();
8903   StaticAllocas.reserve(NumArgs * 2);
8904 
8905   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
8906     if (!V)
8907       return nullptr;
8908     V = V->stripPointerCasts();
8909     const auto *AI = dyn_cast<AllocaInst>(V);
8910     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
8911       return nullptr;
8912     auto Iter = StaticAllocas.insert({AI, Unknown});
8913     return &Iter.first->second;
8914   };
8915 
8916   // Look for stores of arguments to static allocas. Look through bitcasts and
8917   // GEPs to handle type coercions, as long as the alloca is fully initialized
8918   // by the store. Any non-store use of an alloca escapes it and any subsequent
8919   // unanalyzed store might write it.
8920   // FIXME: Handle structs initialized with multiple stores.
8921   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
8922     // Look for stores, and handle non-store uses conservatively.
8923     const auto *SI = dyn_cast<StoreInst>(&I);
8924     if (!SI) {
8925       // We will look through cast uses, so ignore them completely.
8926       if (I.isCast())
8927         continue;
8928       // Ignore debug info intrinsics, they don't escape or store to allocas.
8929       if (isa<DbgInfoIntrinsic>(I))
8930         continue;
8931       // This is an unknown instruction. Assume it escapes or writes to all
8932       // static alloca operands.
8933       for (const Use &U : I.operands()) {
8934         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
8935           *Info = StaticAllocaInfo::Clobbered;
8936       }
8937       continue;
8938     }
8939 
8940     // If the stored value is a static alloca, mark it as escaped.
8941     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
8942       *Info = StaticAllocaInfo::Clobbered;
8943 
8944     // Check if the destination is a static alloca.
8945     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
8946     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
8947     if (!Info)
8948       continue;
8949     const AllocaInst *AI = cast<AllocaInst>(Dst);
8950 
8951     // Skip allocas that have been initialized or clobbered.
8952     if (*Info != StaticAllocaInfo::Unknown)
8953       continue;
8954 
8955     // Check if the stored value is an argument, and that this store fully
8956     // initializes the alloca. Don't elide copies from the same argument twice.
8957     const Value *Val = SI->getValueOperand()->stripPointerCasts();
8958     const auto *Arg = dyn_cast<Argument>(Val);
8959     if (!Arg || Arg->hasInAllocaAttr() || Arg->hasByValAttr() ||
8960         Arg->getType()->isEmptyTy() ||
8961         DL.getTypeStoreSize(Arg->getType()) !=
8962             DL.getTypeAllocSize(AI->getAllocatedType()) ||
8963         ArgCopyElisionCandidates.count(Arg)) {
8964       *Info = StaticAllocaInfo::Clobbered;
8965       continue;
8966     }
8967 
8968     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
8969                       << '\n');
8970 
8971     // Mark this alloca and store for argument copy elision.
8972     *Info = StaticAllocaInfo::Elidable;
8973     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
8974 
8975     // Stop scanning if we've seen all arguments. This will happen early in -O0
8976     // builds, which is useful, because -O0 builds have large entry blocks and
8977     // many allocas.
8978     if (ArgCopyElisionCandidates.size() == NumArgs)
8979       break;
8980   }
8981 }
8982 
8983 /// Try to elide argument copies from memory into a local alloca. Succeeds if
8984 /// ArgVal is a load from a suitable fixed stack object.
8985 static void tryToElideArgumentCopy(
8986     FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
8987     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
8988     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
8989     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
8990     SDValue ArgVal, bool &ArgHasUses) {
8991   // Check if this is a load from a fixed stack object.
8992   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
8993   if (!LNode)
8994     return;
8995   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
8996   if (!FINode)
8997     return;
8998 
8999   // Check that the fixed stack object is the right size and alignment.
9000   // Look at the alignment that the user wrote on the alloca instead of looking
9001   // at the stack object.
9002   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
9003   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
9004   const AllocaInst *AI = ArgCopyIter->second.first;
9005   int FixedIndex = FINode->getIndex();
9006   int &AllocaIndex = FuncInfo->StaticAllocaMap[AI];
9007   int OldIndex = AllocaIndex;
9008   MachineFrameInfo &MFI = FuncInfo->MF->getFrameInfo();
9009   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
9010     LLVM_DEBUG(
9011         dbgs() << "  argument copy elision failed due to bad fixed stack "
9012                   "object size\n");
9013     return;
9014   }
9015   unsigned RequiredAlignment = AI->getAlignment();
9016   if (!RequiredAlignment) {
9017     RequiredAlignment = FuncInfo->MF->getDataLayout().getABITypeAlignment(
9018         AI->getAllocatedType());
9019   }
9020   if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
9021     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
9022                          "greater than stack argument alignment ("
9023                       << RequiredAlignment << " vs "
9024                       << MFI.getObjectAlignment(FixedIndex) << ")\n");
9025     return;
9026   }
9027 
9028   // Perform the elision. Delete the old stack object and replace its only use
9029   // in the variable info map. Mark the stack object as mutable.
9030   LLVM_DEBUG({
9031     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
9032            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
9033            << '\n';
9034   });
9035   MFI.RemoveStackObject(OldIndex);
9036   MFI.setIsImmutableObjectIndex(FixedIndex, false);
9037   AllocaIndex = FixedIndex;
9038   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
9039   Chains.push_back(ArgVal.getValue(1));
9040 
9041   // Avoid emitting code for the store implementing the copy.
9042   const StoreInst *SI = ArgCopyIter->second.second;
9043   ElidedArgCopyInstrs.insert(SI);
9044 
9045   // Check for uses of the argument again so that we can avoid exporting ArgVal
9046   // if it is't used by anything other than the store.
9047   for (const Value *U : Arg.users()) {
9048     if (U != SI) {
9049       ArgHasUses = true;
9050       break;
9051     }
9052   }
9053 }
9054 
9055 void SelectionDAGISel::LowerArguments(const Function &F) {
9056   SelectionDAG &DAG = SDB->DAG;
9057   SDLoc dl = SDB->getCurSDLoc();
9058   const DataLayout &DL = DAG.getDataLayout();
9059   SmallVector<ISD::InputArg, 16> Ins;
9060 
9061   if (!FuncInfo->CanLowerReturn) {
9062     // Put in an sret pointer parameter before all the other parameters.
9063     SmallVector<EVT, 1> ValueVTs;
9064     ComputeValueVTs(*TLI, DAG.getDataLayout(),
9065                     F.getReturnType()->getPointerTo(
9066                         DAG.getDataLayout().getAllocaAddrSpace()),
9067                     ValueVTs);
9068 
9069     // NOTE: Assuming that a pointer will never break down to more than one VT
9070     // or one register.
9071     ISD::ArgFlagsTy Flags;
9072     Flags.setSRet();
9073     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
9074     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
9075                          ISD::InputArg::NoArgIndex, 0);
9076     Ins.push_back(RetArg);
9077   }
9078 
9079   // Look for stores of arguments to static allocas. Mark such arguments with a
9080   // flag to ask the target to give us the memory location of that argument if
9081   // available.
9082   ArgCopyElisionMapTy ArgCopyElisionCandidates;
9083   findArgumentCopyElisionCandidates(DL, FuncInfo, ArgCopyElisionCandidates);
9084 
9085   // Set up the incoming argument description vector.
9086   for (const Argument &Arg : F.args()) {
9087     unsigned ArgNo = Arg.getArgNo();
9088     SmallVector<EVT, 4> ValueVTs;
9089     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
9090     bool isArgValueUsed = !Arg.use_empty();
9091     unsigned PartBase = 0;
9092     Type *FinalType = Arg.getType();
9093     if (Arg.hasAttribute(Attribute::ByVal))
9094       FinalType = cast<PointerType>(FinalType)->getElementType();
9095     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
9096         FinalType, F.getCallingConv(), F.isVarArg());
9097     for (unsigned Value = 0, NumValues = ValueVTs.size();
9098          Value != NumValues; ++Value) {
9099       EVT VT = ValueVTs[Value];
9100       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
9101       ISD::ArgFlagsTy Flags;
9102 
9103       // Certain targets (such as MIPS), may have a different ABI alignment
9104       // for a type depending on the context. Give the target a chance to
9105       // specify the alignment it wants.
9106       unsigned OriginalAlignment =
9107           TLI->getABIAlignmentForCallingConv(ArgTy, DL);
9108 
9109       if (Arg.hasAttribute(Attribute::ZExt))
9110         Flags.setZExt();
9111       if (Arg.hasAttribute(Attribute::SExt))
9112         Flags.setSExt();
9113       if (Arg.hasAttribute(Attribute::InReg)) {
9114         // If we are using vectorcall calling convention, a structure that is
9115         // passed InReg - is surely an HVA
9116         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
9117             isa<StructType>(Arg.getType())) {
9118           // The first value of a structure is marked
9119           if (0 == Value)
9120             Flags.setHvaStart();
9121           Flags.setHva();
9122         }
9123         // Set InReg Flag
9124         Flags.setInReg();
9125       }
9126       if (Arg.hasAttribute(Attribute::StructRet))
9127         Flags.setSRet();
9128       if (Arg.hasAttribute(Attribute::SwiftSelf))
9129         Flags.setSwiftSelf();
9130       if (Arg.hasAttribute(Attribute::SwiftError))
9131         Flags.setSwiftError();
9132       if (Arg.hasAttribute(Attribute::ByVal))
9133         Flags.setByVal();
9134       if (Arg.hasAttribute(Attribute::InAlloca)) {
9135         Flags.setInAlloca();
9136         // Set the byval flag for CCAssignFn callbacks that don't know about
9137         // inalloca.  This way we can know how many bytes we should've allocated
9138         // and how many bytes a callee cleanup function will pop.  If we port
9139         // inalloca to more targets, we'll have to add custom inalloca handling
9140         // in the various CC lowering callbacks.
9141         Flags.setByVal();
9142       }
9143       if (F.getCallingConv() == CallingConv::X86_INTR) {
9144         // IA Interrupt passes frame (1st parameter) by value in the stack.
9145         if (ArgNo == 0)
9146           Flags.setByVal();
9147       }
9148       if (Flags.isByVal() || Flags.isInAlloca()) {
9149         PointerType *Ty = cast<PointerType>(Arg.getType());
9150         Type *ElementTy = Ty->getElementType();
9151         Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
9152         // For ByVal, alignment should be passed from FE.  BE will guess if
9153         // this info is not there but there are cases it cannot get right.
9154         unsigned FrameAlign;
9155         if (Arg.getParamAlignment())
9156           FrameAlign = Arg.getParamAlignment();
9157         else
9158           FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
9159         Flags.setByValAlign(FrameAlign);
9160       }
9161       if (Arg.hasAttribute(Attribute::Nest))
9162         Flags.setNest();
9163       if (NeedsRegBlock)
9164         Flags.setInConsecutiveRegs();
9165       Flags.setOrigAlign(OriginalAlignment);
9166       if (ArgCopyElisionCandidates.count(&Arg))
9167         Flags.setCopyElisionCandidate();
9168 
9169       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
9170           *CurDAG->getContext(), F.getCallingConv(), VT);
9171       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
9172           *CurDAG->getContext(), F.getCallingConv(), VT);
9173       for (unsigned i = 0; i != NumRegs; ++i) {
9174         ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
9175                               ArgNo, PartBase+i*RegisterVT.getStoreSize());
9176         if (NumRegs > 1 && i == 0)
9177           MyFlags.Flags.setSplit();
9178         // if it isn't first piece, alignment must be 1
9179         else if (i > 0) {
9180           MyFlags.Flags.setOrigAlign(1);
9181           if (i == NumRegs - 1)
9182             MyFlags.Flags.setSplitEnd();
9183         }
9184         Ins.push_back(MyFlags);
9185       }
9186       if (NeedsRegBlock && Value == NumValues - 1)
9187         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
9188       PartBase += VT.getStoreSize();
9189     }
9190   }
9191 
9192   // Call the target to set up the argument values.
9193   SmallVector<SDValue, 8> InVals;
9194   SDValue NewRoot = TLI->LowerFormalArguments(
9195       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
9196 
9197   // Verify that the target's LowerFormalArguments behaved as expected.
9198   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
9199          "LowerFormalArguments didn't return a valid chain!");
9200   assert(InVals.size() == Ins.size() &&
9201          "LowerFormalArguments didn't emit the correct number of values!");
9202   LLVM_DEBUG({
9203     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
9204       assert(InVals[i].getNode() &&
9205              "LowerFormalArguments emitted a null value!");
9206       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
9207              "LowerFormalArguments emitted a value with the wrong type!");
9208     }
9209   });
9210 
9211   // Update the DAG with the new chain value resulting from argument lowering.
9212   DAG.setRoot(NewRoot);
9213 
9214   // Set up the argument values.
9215   unsigned i = 0;
9216   if (!FuncInfo->CanLowerReturn) {
9217     // Create a virtual register for the sret pointer, and put in a copy
9218     // from the sret argument into it.
9219     SmallVector<EVT, 1> ValueVTs;
9220     ComputeValueVTs(*TLI, DAG.getDataLayout(),
9221                     F.getReturnType()->getPointerTo(
9222                         DAG.getDataLayout().getAllocaAddrSpace()),
9223                     ValueVTs);
9224     MVT VT = ValueVTs[0].getSimpleVT();
9225     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
9226     Optional<ISD::NodeType> AssertOp = None;
9227     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
9228                                         nullptr, F.getCallingConv(), AssertOp);
9229 
9230     MachineFunction& MF = SDB->DAG.getMachineFunction();
9231     MachineRegisterInfo& RegInfo = MF.getRegInfo();
9232     unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
9233     FuncInfo->DemoteRegister = SRetReg;
9234     NewRoot =
9235         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
9236     DAG.setRoot(NewRoot);
9237 
9238     // i indexes lowered arguments.  Bump it past the hidden sret argument.
9239     ++i;
9240   }
9241 
9242   SmallVector<SDValue, 4> Chains;
9243   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
9244   for (const Argument &Arg : F.args()) {
9245     SmallVector<SDValue, 4> ArgValues;
9246     SmallVector<EVT, 4> ValueVTs;
9247     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
9248     unsigned NumValues = ValueVTs.size();
9249     if (NumValues == 0)
9250       continue;
9251 
9252     bool ArgHasUses = !Arg.use_empty();
9253 
9254     // Elide the copying store if the target loaded this argument from a
9255     // suitable fixed stack object.
9256     if (Ins[i].Flags.isCopyElisionCandidate()) {
9257       tryToElideArgumentCopy(FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
9258                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
9259                              InVals[i], ArgHasUses);
9260     }
9261 
9262     // If this argument is unused then remember its value. It is used to generate
9263     // debugging information.
9264     bool isSwiftErrorArg =
9265         TLI->supportSwiftError() &&
9266         Arg.hasAttribute(Attribute::SwiftError);
9267     if (!ArgHasUses && !isSwiftErrorArg) {
9268       SDB->setUnusedArgValue(&Arg, InVals[i]);
9269 
9270       // Also remember any frame index for use in FastISel.
9271       if (FrameIndexSDNode *FI =
9272           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
9273         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9274     }
9275 
9276     for (unsigned Val = 0; Val != NumValues; ++Val) {
9277       EVT VT = ValueVTs[Val];
9278       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
9279                                                       F.getCallingConv(), VT);
9280       unsigned NumParts = TLI->getNumRegistersForCallingConv(
9281           *CurDAG->getContext(), F.getCallingConv(), VT);
9282 
9283       // Even an apparant 'unused' swifterror argument needs to be returned. So
9284       // we do generate a copy for it that can be used on return from the
9285       // function.
9286       if (ArgHasUses || isSwiftErrorArg) {
9287         Optional<ISD::NodeType> AssertOp;
9288         if (Arg.hasAttribute(Attribute::SExt))
9289           AssertOp = ISD::AssertSext;
9290         else if (Arg.hasAttribute(Attribute::ZExt))
9291           AssertOp = ISD::AssertZext;
9292 
9293         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
9294                                              PartVT, VT, nullptr,
9295                                              F.getCallingConv(), AssertOp));
9296       }
9297 
9298       i += NumParts;
9299     }
9300 
9301     // We don't need to do anything else for unused arguments.
9302     if (ArgValues.empty())
9303       continue;
9304 
9305     // Note down frame index.
9306     if (FrameIndexSDNode *FI =
9307         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
9308       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9309 
9310     SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
9311                                      SDB->getCurSDLoc());
9312 
9313     SDB->setValue(&Arg, Res);
9314     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
9315       // We want to associate the argument with the frame index, among
9316       // involved operands, that correspond to the lowest address. The
9317       // getCopyFromParts function, called earlier, is swapping the order of
9318       // the operands to BUILD_PAIR depending on endianness. The result of
9319       // that swapping is that the least significant bits of the argument will
9320       // be in the first operand of the BUILD_PAIR node, and the most
9321       // significant bits will be in the second operand.
9322       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9323       if (LoadSDNode *LNode =
9324           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
9325         if (FrameIndexSDNode *FI =
9326             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
9327           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9328     }
9329 
9330     // Update the SwiftErrorVRegDefMap.
9331     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
9332       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9333       if (TargetRegisterInfo::isVirtualRegister(Reg))
9334         FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB,
9335                                            FuncInfo->SwiftErrorArg, Reg);
9336     }
9337 
9338     // If this argument is live outside of the entry block, insert a copy from
9339     // wherever we got it to the vreg that other BB's will reference it as.
9340     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
9341       // If we can, though, try to skip creating an unnecessary vreg.
9342       // FIXME: This isn't very clean... it would be nice to make this more
9343       // general.  It's also subtly incompatible with the hacks FastISel
9344       // uses with vregs.
9345       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9346       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
9347         FuncInfo->ValueMap[&Arg] = Reg;
9348         continue;
9349       }
9350     }
9351     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
9352       FuncInfo->InitializeRegForValue(&Arg);
9353       SDB->CopyToExportRegsIfNeeded(&Arg);
9354     }
9355   }
9356 
9357   if (!Chains.empty()) {
9358     Chains.push_back(NewRoot);
9359     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
9360   }
9361 
9362   DAG.setRoot(NewRoot);
9363 
9364   assert(i == InVals.size() && "Argument register count mismatch!");
9365 
9366   // If any argument copy elisions occurred and we have debug info, update the
9367   // stale frame indices used in the dbg.declare variable info table.
9368   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
9369   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
9370     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
9371       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
9372       if (I != ArgCopyElisionFrameIndexMap.end())
9373         VI.Slot = I->second;
9374     }
9375   }
9376 
9377   // Finally, if the target has anything special to do, allow it to do so.
9378   EmitFunctionEntryCode();
9379 }
9380 
9381 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
9382 /// ensure constants are generated when needed.  Remember the virtual registers
9383 /// that need to be added to the Machine PHI nodes as input.  We cannot just
9384 /// directly add them, because expansion might result in multiple MBB's for one
9385 /// BB.  As such, the start of the BB might correspond to a different MBB than
9386 /// the end.
9387 void
9388 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
9389   const Instruction *TI = LLVMBB->getTerminator();
9390 
9391   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
9392 
9393   // Check PHI nodes in successors that expect a value to be available from this
9394   // block.
9395   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
9396     const BasicBlock *SuccBB = TI->getSuccessor(succ);
9397     if (!isa<PHINode>(SuccBB->begin())) continue;
9398     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
9399 
9400     // If this terminator has multiple identical successors (common for
9401     // switches), only handle each succ once.
9402     if (!SuccsHandled.insert(SuccMBB).second)
9403       continue;
9404 
9405     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
9406 
9407     // At this point we know that there is a 1-1 correspondence between LLVM PHI
9408     // nodes and Machine PHI nodes, but the incoming operands have not been
9409     // emitted yet.
9410     for (const PHINode &PN : SuccBB->phis()) {
9411       // Ignore dead phi's.
9412       if (PN.use_empty())
9413         continue;
9414 
9415       // Skip empty types
9416       if (PN.getType()->isEmptyTy())
9417         continue;
9418 
9419       unsigned Reg;
9420       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
9421 
9422       if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
9423         unsigned &RegOut = ConstantsOut[C];
9424         if (RegOut == 0) {
9425           RegOut = FuncInfo.CreateRegs(C->getType());
9426           CopyValueToVirtualRegister(C, RegOut);
9427         }
9428         Reg = RegOut;
9429       } else {
9430         DenseMap<const Value *, unsigned>::iterator I =
9431           FuncInfo.ValueMap.find(PHIOp);
9432         if (I != FuncInfo.ValueMap.end())
9433           Reg = I->second;
9434         else {
9435           assert(isa<AllocaInst>(PHIOp) &&
9436                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
9437                  "Didn't codegen value into a register!??");
9438           Reg = FuncInfo.CreateRegs(PHIOp->getType());
9439           CopyValueToVirtualRegister(PHIOp, Reg);
9440         }
9441       }
9442 
9443       // Remember that this register needs to added to the machine PHI node as
9444       // the input for this MBB.
9445       SmallVector<EVT, 4> ValueVTs;
9446       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9447       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
9448       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
9449         EVT VT = ValueVTs[vti];
9450         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
9451         for (unsigned i = 0, e = NumRegisters; i != e; ++i)
9452           FuncInfo.PHINodesToUpdate.push_back(
9453               std::make_pair(&*MBBI++, Reg + i));
9454         Reg += NumRegisters;
9455       }
9456     }
9457   }
9458 
9459   ConstantsOut.clear();
9460 }
9461 
9462 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
9463 /// is 0.
9464 MachineBasicBlock *
9465 SelectionDAGBuilder::StackProtectorDescriptor::
9466 AddSuccessorMBB(const BasicBlock *BB,
9467                 MachineBasicBlock *ParentMBB,
9468                 bool IsLikely,
9469                 MachineBasicBlock *SuccMBB) {
9470   // If SuccBB has not been created yet, create it.
9471   if (!SuccMBB) {
9472     MachineFunction *MF = ParentMBB->getParent();
9473     MachineFunction::iterator BBI(ParentMBB);
9474     SuccMBB = MF->CreateMachineBasicBlock(BB);
9475     MF->insert(++BBI, SuccMBB);
9476   }
9477   // Add it as a successor of ParentMBB.
9478   ParentMBB->addSuccessor(
9479       SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
9480   return SuccMBB;
9481 }
9482 
9483 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
9484   MachineFunction::iterator I(MBB);
9485   if (++I == FuncInfo.MF->end())
9486     return nullptr;
9487   return &*I;
9488 }
9489 
9490 /// During lowering new call nodes can be created (such as memset, etc.).
9491 /// Those will become new roots of the current DAG, but complications arise
9492 /// when they are tail calls. In such cases, the call lowering will update
9493 /// the root, but the builder still needs to know that a tail call has been
9494 /// lowered in order to avoid generating an additional return.
9495 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
9496   // If the node is null, we do have a tail call.
9497   if (MaybeTC.getNode() != nullptr)
9498     DAG.setRoot(MaybeTC);
9499   else
9500     HasTailCall = true;
9501 }
9502 
9503 uint64_t
9504 SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters,
9505                                        unsigned First, unsigned Last) const {
9506   assert(Last >= First);
9507   const APInt &LowCase = Clusters[First].Low->getValue();
9508   const APInt &HighCase = Clusters[Last].High->getValue();
9509   assert(LowCase.getBitWidth() == HighCase.getBitWidth());
9510 
9511   // FIXME: A range of consecutive cases has 100% density, but only requires one
9512   // comparison to lower. We should discriminate against such consecutive ranges
9513   // in jump tables.
9514 
9515   return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1;
9516 }
9517 
9518 uint64_t SelectionDAGBuilder::getJumpTableNumCases(
9519     const SmallVectorImpl<unsigned> &TotalCases, unsigned First,
9520     unsigned Last) const {
9521   assert(Last >= First);
9522   assert(TotalCases[Last] >= TotalCases[First]);
9523   uint64_t NumCases =
9524       TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
9525   return NumCases;
9526 }
9527 
9528 bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
9529                                          unsigned First, unsigned Last,
9530                                          const SwitchInst *SI,
9531                                          MachineBasicBlock *DefaultMBB,
9532                                          CaseCluster &JTCluster) {
9533   assert(First <= Last);
9534 
9535   auto Prob = BranchProbability::getZero();
9536   unsigned NumCmps = 0;
9537   std::vector<MachineBasicBlock*> Table;
9538   DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
9539 
9540   // Initialize probabilities in JTProbs.
9541   for (unsigned I = First; I <= Last; ++I)
9542     JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
9543 
9544   for (unsigned I = First; I <= Last; ++I) {
9545     assert(Clusters[I].Kind == CC_Range);
9546     Prob += Clusters[I].Prob;
9547     const APInt &Low = Clusters[I].Low->getValue();
9548     const APInt &High = Clusters[I].High->getValue();
9549     NumCmps += (Low == High) ? 1 : 2;
9550     if (I != First) {
9551       // Fill the gap between this and the previous cluster.
9552       const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
9553       assert(PreviousHigh.slt(Low));
9554       uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
9555       for (uint64_t J = 0; J < Gap; J++)
9556         Table.push_back(DefaultMBB);
9557     }
9558     uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
9559     for (uint64_t J = 0; J < ClusterSize; ++J)
9560       Table.push_back(Clusters[I].MBB);
9561     JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
9562   }
9563 
9564   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9565   unsigned NumDests = JTProbs.size();
9566   if (TLI.isSuitableForBitTests(
9567           NumDests, NumCmps, Clusters[First].Low->getValue(),
9568           Clusters[Last].High->getValue(), DAG.getDataLayout())) {
9569     // Clusters[First..Last] should be lowered as bit tests instead.
9570     return false;
9571   }
9572 
9573   // Create the MBB that will load from and jump through the table.
9574   // Note: We create it here, but it's not inserted into the function yet.
9575   MachineFunction *CurMF = FuncInfo.MF;
9576   MachineBasicBlock *JumpTableMBB =
9577       CurMF->CreateMachineBasicBlock(SI->getParent());
9578 
9579   // Add successors. Note: use table order for determinism.
9580   SmallPtrSet<MachineBasicBlock *, 8> Done;
9581   for (MachineBasicBlock *Succ : Table) {
9582     if (Done.count(Succ))
9583       continue;
9584     addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
9585     Done.insert(Succ);
9586   }
9587   JumpTableMBB->normalizeSuccProbs();
9588 
9589   unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
9590                      ->createJumpTableIndex(Table);
9591 
9592   // Set up the jump table info.
9593   JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
9594   JumpTableHeader JTH(Clusters[First].Low->getValue(),
9595                       Clusters[Last].High->getValue(), SI->getCondition(),
9596                       nullptr, false);
9597   JTCases.emplace_back(std::move(JTH), std::move(JT));
9598 
9599   JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
9600                                      JTCases.size() - 1, Prob);
9601   return true;
9602 }
9603 
9604 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
9605                                          const SwitchInst *SI,
9606                                          MachineBasicBlock *DefaultMBB) {
9607 #ifndef NDEBUG
9608   // Clusters must be non-empty, sorted, and only contain Range clusters.
9609   assert(!Clusters.empty());
9610   for (CaseCluster &C : Clusters)
9611     assert(C.Kind == CC_Range);
9612   for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
9613     assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
9614 #endif
9615 
9616   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9617   if (!TLI.areJTsAllowed(SI->getParent()->getParent()))
9618     return;
9619 
9620   const int64_t N = Clusters.size();
9621   const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
9622   const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
9623 
9624   if (N < 2 || N < MinJumpTableEntries)
9625     return;
9626 
9627   // TotalCases[i]: Total nbr of cases in Clusters[0..i].
9628   SmallVector<unsigned, 8> TotalCases(N);
9629   for (unsigned i = 0; i < N; ++i) {
9630     const APInt &Hi = Clusters[i].High->getValue();
9631     const APInt &Lo = Clusters[i].Low->getValue();
9632     TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
9633     if (i != 0)
9634       TotalCases[i] += TotalCases[i - 1];
9635   }
9636 
9637   // Cheap case: the whole range may be suitable for jump table.
9638   uint64_t Range = getJumpTableRange(Clusters,0, N - 1);
9639   uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1);
9640   assert(NumCases < UINT64_MAX / 100);
9641   assert(Range >= NumCases);
9642   if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9643     CaseCluster JTCluster;
9644     if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
9645       Clusters[0] = JTCluster;
9646       Clusters.resize(1);
9647       return;
9648     }
9649   }
9650 
9651   // The algorithm below is not suitable for -O0.
9652   if (TM.getOptLevel() == CodeGenOpt::None)
9653     return;
9654 
9655   // Split Clusters into minimum number of dense partitions. The algorithm uses
9656   // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
9657   // for the Case Statement'" (1994), but builds the MinPartitions array in
9658   // reverse order to make it easier to reconstruct the partitions in ascending
9659   // order. In the choice between two optimal partitionings, it picks the one
9660   // which yields more jump tables.
9661 
9662   // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9663   SmallVector<unsigned, 8> MinPartitions(N);
9664   // LastElement[i] is the last element of the partition starting at i.
9665   SmallVector<unsigned, 8> LastElement(N);
9666   // PartitionsScore[i] is used to break ties when choosing between two
9667   // partitionings resulting in the same number of partitions.
9668   SmallVector<unsigned, 8> PartitionsScore(N);
9669   // For PartitionsScore, a small number of comparisons is considered as good as
9670   // a jump table and a single comparison is considered better than a jump
9671   // table.
9672   enum PartitionScores : unsigned {
9673     NoTable = 0,
9674     Table = 1,
9675     FewCases = 1,
9676     SingleCase = 2
9677   };
9678 
9679   // Base case: There is only one way to partition Clusters[N-1].
9680   MinPartitions[N - 1] = 1;
9681   LastElement[N - 1] = N - 1;
9682   PartitionsScore[N - 1] = PartitionScores::SingleCase;
9683 
9684   // Note: loop indexes are signed to avoid underflow.
9685   for (int64_t i = N - 2; i >= 0; i--) {
9686     // Find optimal partitioning of Clusters[i..N-1].
9687     // Baseline: Put Clusters[i] into a partition on its own.
9688     MinPartitions[i] = MinPartitions[i + 1] + 1;
9689     LastElement[i] = i;
9690     PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
9691 
9692     // Search for a solution that results in fewer partitions.
9693     for (int64_t j = N - 1; j > i; j--) {
9694       // Try building a partition from Clusters[i..j].
9695       uint64_t Range = getJumpTableRange(Clusters, i, j);
9696       uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j);
9697       assert(NumCases < UINT64_MAX / 100);
9698       assert(Range >= NumCases);
9699       if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9700         unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9701         unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
9702         int64_t NumEntries = j - i + 1;
9703 
9704         if (NumEntries == 1)
9705           Score += PartitionScores::SingleCase;
9706         else if (NumEntries <= SmallNumberOfEntries)
9707           Score += PartitionScores::FewCases;
9708         else if (NumEntries >= MinJumpTableEntries)
9709           Score += PartitionScores::Table;
9710 
9711         // If this leads to fewer partitions, or to the same number of
9712         // partitions with better score, it is a better partitioning.
9713         if (NumPartitions < MinPartitions[i] ||
9714             (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
9715           MinPartitions[i] = NumPartitions;
9716           LastElement[i] = j;
9717           PartitionsScore[i] = Score;
9718         }
9719       }
9720     }
9721   }
9722 
9723   // Iterate over the partitions, replacing some with jump tables in-place.
9724   unsigned DstIndex = 0;
9725   for (unsigned First = 0, Last; First < N; First = Last + 1) {
9726     Last = LastElement[First];
9727     assert(Last >= First);
9728     assert(DstIndex <= First);
9729     unsigned NumClusters = Last - First + 1;
9730 
9731     CaseCluster JTCluster;
9732     if (NumClusters >= MinJumpTableEntries &&
9733         buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
9734       Clusters[DstIndex++] = JTCluster;
9735     } else {
9736       for (unsigned I = First; I <= Last; ++I)
9737         std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
9738     }
9739   }
9740   Clusters.resize(DstIndex);
9741 }
9742 
9743 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
9744                                         unsigned First, unsigned Last,
9745                                         const SwitchInst *SI,
9746                                         CaseCluster &BTCluster) {
9747   assert(First <= Last);
9748   if (First == Last)
9749     return false;
9750 
9751   BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9752   unsigned NumCmps = 0;
9753   for (int64_t I = First; I <= Last; ++I) {
9754     assert(Clusters[I].Kind == CC_Range);
9755     Dests.set(Clusters[I].MBB->getNumber());
9756     NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
9757   }
9758   unsigned NumDests = Dests.count();
9759 
9760   APInt Low = Clusters[First].Low->getValue();
9761   APInt High = Clusters[Last].High->getValue();
9762   assert(Low.slt(High));
9763 
9764   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9765   const DataLayout &DL = DAG.getDataLayout();
9766   if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL))
9767     return false;
9768 
9769   APInt LowBound;
9770   APInt CmpRange;
9771 
9772   const int BitWidth = TLI.getPointerTy(DL).getSizeInBits();
9773   assert(TLI.rangeFitsInWord(Low, High, DL) &&
9774          "Case range must fit in bit mask!");
9775 
9776   // Check if the clusters cover a contiguous range such that no value in the
9777   // range will jump to the default statement.
9778   bool ContiguousRange = true;
9779   for (int64_t I = First + 1; I <= Last; ++I) {
9780     if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
9781       ContiguousRange = false;
9782       break;
9783     }
9784   }
9785 
9786   if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
9787     // Optimize the case where all the case values fit in a word without having
9788     // to subtract minValue. In this case, we can optimize away the subtraction.
9789     LowBound = APInt::getNullValue(Low.getBitWidth());
9790     CmpRange = High;
9791     ContiguousRange = false;
9792   } else {
9793     LowBound = Low;
9794     CmpRange = High - Low;
9795   }
9796 
9797   CaseBitsVector CBV;
9798   auto TotalProb = BranchProbability::getZero();
9799   for (unsigned i = First; i <= Last; ++i) {
9800     // Find the CaseBits for this destination.
9801     unsigned j;
9802     for (j = 0; j < CBV.size(); ++j)
9803       if (CBV[j].BB == Clusters[i].MBB)
9804         break;
9805     if (j == CBV.size())
9806       CBV.push_back(
9807           CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
9808     CaseBits *CB = &CBV[j];
9809 
9810     // Update Mask, Bits and ExtraProb.
9811     uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
9812     uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
9813     assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
9814     CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
9815     CB->Bits += Hi - Lo + 1;
9816     CB->ExtraProb += Clusters[i].Prob;
9817     TotalProb += Clusters[i].Prob;
9818   }
9819 
9820   BitTestInfo BTI;
9821   llvm::sort(CBV, [](const CaseBits &a, const CaseBits &b) {
9822     // Sort by probability first, number of bits second, bit mask third.
9823     if (a.ExtraProb != b.ExtraProb)
9824       return a.ExtraProb > b.ExtraProb;
9825     if (a.Bits != b.Bits)
9826       return a.Bits > b.Bits;
9827     return a.Mask < b.Mask;
9828   });
9829 
9830   for (auto &CB : CBV) {
9831     MachineBasicBlock *BitTestBB =
9832         FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
9833     BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
9834   }
9835   BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
9836                             SI->getCondition(), -1U, MVT::Other, false,
9837                             ContiguousRange, nullptr, nullptr, std::move(BTI),
9838                             TotalProb);
9839 
9840   BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
9841                                     BitTestCases.size() - 1, TotalProb);
9842   return true;
9843 }
9844 
9845 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
9846                                               const SwitchInst *SI) {
9847 // Partition Clusters into as few subsets as possible, where each subset has a
9848 // range that fits in a machine word and has <= 3 unique destinations.
9849 
9850 #ifndef NDEBUG
9851   // Clusters must be sorted and contain Range or JumpTable clusters.
9852   assert(!Clusters.empty());
9853   assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
9854   for (const CaseCluster &C : Clusters)
9855     assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
9856   for (unsigned i = 1; i < Clusters.size(); ++i)
9857     assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
9858 #endif
9859 
9860   // The algorithm below is not suitable for -O0.
9861   if (TM.getOptLevel() == CodeGenOpt::None)
9862     return;
9863 
9864   // If target does not have legal shift left, do not emit bit tests at all.
9865   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9866   const DataLayout &DL = DAG.getDataLayout();
9867 
9868   EVT PTy = TLI.getPointerTy(DL);
9869   if (!TLI.isOperationLegal(ISD::SHL, PTy))
9870     return;
9871 
9872   int BitWidth = PTy.getSizeInBits();
9873   const int64_t N = Clusters.size();
9874 
9875   // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9876   SmallVector<unsigned, 8> MinPartitions(N);
9877   // LastElement[i] is the last element of the partition starting at i.
9878   SmallVector<unsigned, 8> LastElement(N);
9879 
9880   // FIXME: This might not be the best algorithm for finding bit test clusters.
9881 
9882   // Base case: There is only one way to partition Clusters[N-1].
9883   MinPartitions[N - 1] = 1;
9884   LastElement[N - 1] = N - 1;
9885 
9886   // Note: loop indexes are signed to avoid underflow.
9887   for (int64_t i = N - 2; i >= 0; --i) {
9888     // Find optimal partitioning of Clusters[i..N-1].
9889     // Baseline: Put Clusters[i] into a partition on its own.
9890     MinPartitions[i] = MinPartitions[i + 1] + 1;
9891     LastElement[i] = i;
9892 
9893     // Search for a solution that results in fewer partitions.
9894     // Note: the search is limited by BitWidth, reducing time complexity.
9895     for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
9896       // Try building a partition from Clusters[i..j].
9897 
9898       // Check the range.
9899       if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(),
9900                                Clusters[j].High->getValue(), DL))
9901         continue;
9902 
9903       // Check nbr of destinations and cluster types.
9904       // FIXME: This works, but doesn't seem very efficient.
9905       bool RangesOnly = true;
9906       BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9907       for (int64_t k = i; k <= j; k++) {
9908         if (Clusters[k].Kind != CC_Range) {
9909           RangesOnly = false;
9910           break;
9911         }
9912         Dests.set(Clusters[k].MBB->getNumber());
9913       }
9914       if (!RangesOnly || Dests.count() > 3)
9915         break;
9916 
9917       // Check if it's a better partition.
9918       unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9919       if (NumPartitions < MinPartitions[i]) {
9920         // Found a better partition.
9921         MinPartitions[i] = NumPartitions;
9922         LastElement[i] = j;
9923       }
9924     }
9925   }
9926 
9927   // Iterate over the partitions, replacing with bit-test clusters in-place.
9928   unsigned DstIndex = 0;
9929   for (unsigned First = 0, Last; First < N; First = Last + 1) {
9930     Last = LastElement[First];
9931     assert(First <= Last);
9932     assert(DstIndex <= First);
9933 
9934     CaseCluster BitTestCluster;
9935     if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
9936       Clusters[DstIndex++] = BitTestCluster;
9937     } else {
9938       size_t NumClusters = Last - First + 1;
9939       std::memmove(&Clusters[DstIndex], &Clusters[First],
9940                    sizeof(Clusters[0]) * NumClusters);
9941       DstIndex += NumClusters;
9942     }
9943   }
9944   Clusters.resize(DstIndex);
9945 }
9946 
9947 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
9948                                         MachineBasicBlock *SwitchMBB,
9949                                         MachineBasicBlock *DefaultMBB) {
9950   MachineFunction *CurMF = FuncInfo.MF;
9951   MachineBasicBlock *NextMBB = nullptr;
9952   MachineFunction::iterator BBI(W.MBB);
9953   if (++BBI != FuncInfo.MF->end())
9954     NextMBB = &*BBI;
9955 
9956   unsigned Size = W.LastCluster - W.FirstCluster + 1;
9957 
9958   BranchProbabilityInfo *BPI = FuncInfo.BPI;
9959 
9960   if (Size == 2 && W.MBB == SwitchMBB) {
9961     // If any two of the cases has the same destination, and if one value
9962     // is the same as the other, but has one bit unset that the other has set,
9963     // use bit manipulation to do two compares at once.  For example:
9964     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
9965     // TODO: This could be extended to merge any 2 cases in switches with 3
9966     // cases.
9967     // TODO: Handle cases where W.CaseBB != SwitchBB.
9968     CaseCluster &Small = *W.FirstCluster;
9969     CaseCluster &Big = *W.LastCluster;
9970 
9971     if (Small.Low == Small.High && Big.Low == Big.High &&
9972         Small.MBB == Big.MBB) {
9973       const APInt &SmallValue = Small.Low->getValue();
9974       const APInt &BigValue = Big.Low->getValue();
9975 
9976       // Check that there is only one bit different.
9977       APInt CommonBit = BigValue ^ SmallValue;
9978       if (CommonBit.isPowerOf2()) {
9979         SDValue CondLHS = getValue(Cond);
9980         EVT VT = CondLHS.getValueType();
9981         SDLoc DL = getCurSDLoc();
9982 
9983         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
9984                                  DAG.getConstant(CommonBit, DL, VT));
9985         SDValue Cond = DAG.getSetCC(
9986             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
9987             ISD::SETEQ);
9988 
9989         // Update successor info.
9990         // Both Small and Big will jump to Small.BB, so we sum up the
9991         // probabilities.
9992         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
9993         if (BPI)
9994           addSuccessorWithProb(
9995               SwitchMBB, DefaultMBB,
9996               // The default destination is the first successor in IR.
9997               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
9998         else
9999           addSuccessorWithProb(SwitchMBB, DefaultMBB);
10000 
10001         // Insert the true branch.
10002         SDValue BrCond =
10003             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
10004                         DAG.getBasicBlock(Small.MBB));
10005         // Insert the false branch.
10006         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
10007                              DAG.getBasicBlock(DefaultMBB));
10008 
10009         DAG.setRoot(BrCond);
10010         return;
10011       }
10012     }
10013   }
10014 
10015   if (TM.getOptLevel() != CodeGenOpt::None) {
10016     // Here, we order cases by probability so the most likely case will be
10017     // checked first. However, two clusters can have the same probability in
10018     // which case their relative ordering is non-deterministic. So we use Low
10019     // as a tie-breaker as clusters are guaranteed to never overlap.
10020     llvm::sort(W.FirstCluster, W.LastCluster + 1,
10021                [](const CaseCluster &a, const CaseCluster &b) {
10022       return a.Prob != b.Prob ?
10023              a.Prob > b.Prob :
10024              a.Low->getValue().slt(b.Low->getValue());
10025     });
10026 
10027     // Rearrange the case blocks so that the last one falls through if possible
10028     // without changing the order of probabilities.
10029     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
10030       --I;
10031       if (I->Prob > W.LastCluster->Prob)
10032         break;
10033       if (I->Kind == CC_Range && I->MBB == NextMBB) {
10034         std::swap(*I, *W.LastCluster);
10035         break;
10036       }
10037     }
10038   }
10039 
10040   // Compute total probability.
10041   BranchProbability DefaultProb = W.DefaultProb;
10042   BranchProbability UnhandledProbs = DefaultProb;
10043   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
10044     UnhandledProbs += I->Prob;
10045 
10046   MachineBasicBlock *CurMBB = W.MBB;
10047   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
10048     MachineBasicBlock *Fallthrough;
10049     if (I == W.LastCluster) {
10050       // For the last cluster, fall through to the default destination.
10051       Fallthrough = DefaultMBB;
10052     } else {
10053       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
10054       CurMF->insert(BBI, Fallthrough);
10055       // Put Cond in a virtual register to make it available from the new blocks.
10056       ExportFromCurrentBlock(Cond);
10057     }
10058     UnhandledProbs -= I->Prob;
10059 
10060     switch (I->Kind) {
10061       case CC_JumpTable: {
10062         // FIXME: Optimize away range check based on pivot comparisons.
10063         JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
10064         JumpTable *JT = &JTCases[I->JTCasesIndex].second;
10065 
10066         // The jump block hasn't been inserted yet; insert it here.
10067         MachineBasicBlock *JumpMBB = JT->MBB;
10068         CurMF->insert(BBI, JumpMBB);
10069 
10070         auto JumpProb = I->Prob;
10071         auto FallthroughProb = UnhandledProbs;
10072 
10073         // If the default statement is a target of the jump table, we evenly
10074         // distribute the default probability to successors of CurMBB. Also
10075         // update the probability on the edge from JumpMBB to Fallthrough.
10076         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
10077                                               SE = JumpMBB->succ_end();
10078              SI != SE; ++SI) {
10079           if (*SI == DefaultMBB) {
10080             JumpProb += DefaultProb / 2;
10081             FallthroughProb -= DefaultProb / 2;
10082             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
10083             JumpMBB->normalizeSuccProbs();
10084             break;
10085           }
10086         }
10087 
10088         addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
10089         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
10090         CurMBB->normalizeSuccProbs();
10091 
10092         // The jump table header will be inserted in our current block, do the
10093         // range check, and fall through to our fallthrough block.
10094         JTH->HeaderBB = CurMBB;
10095         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
10096 
10097         // If we're in the right place, emit the jump table header right now.
10098         if (CurMBB == SwitchMBB) {
10099           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
10100           JTH->Emitted = true;
10101         }
10102         break;
10103       }
10104       case CC_BitTests: {
10105         // FIXME: Optimize away range check based on pivot comparisons.
10106         BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
10107 
10108         // The bit test blocks haven't been inserted yet; insert them here.
10109         for (BitTestCase &BTC : BTB->Cases)
10110           CurMF->insert(BBI, BTC.ThisBB);
10111 
10112         // Fill in fields of the BitTestBlock.
10113         BTB->Parent = CurMBB;
10114         BTB->Default = Fallthrough;
10115 
10116         BTB->DefaultProb = UnhandledProbs;
10117         // If the cases in bit test don't form a contiguous range, we evenly
10118         // distribute the probability on the edge to Fallthrough to two
10119         // successors of CurMBB.
10120         if (!BTB->ContiguousRange) {
10121           BTB->Prob += DefaultProb / 2;
10122           BTB->DefaultProb -= DefaultProb / 2;
10123         }
10124 
10125         // If we're in the right place, emit the bit test header right now.
10126         if (CurMBB == SwitchMBB) {
10127           visitBitTestHeader(*BTB, SwitchMBB);
10128           BTB->Emitted = true;
10129         }
10130         break;
10131       }
10132       case CC_Range: {
10133         const Value *RHS, *LHS, *MHS;
10134         ISD::CondCode CC;
10135         if (I->Low == I->High) {
10136           // Check Cond == I->Low.
10137           CC = ISD::SETEQ;
10138           LHS = Cond;
10139           RHS=I->Low;
10140           MHS = nullptr;
10141         } else {
10142           // Check I->Low <= Cond <= I->High.
10143           CC = ISD::SETLE;
10144           LHS = I->Low;
10145           MHS = Cond;
10146           RHS = I->High;
10147         }
10148 
10149         // The false probability is the sum of all unhandled cases.
10150         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
10151                      getCurSDLoc(), I->Prob, UnhandledProbs);
10152 
10153         if (CurMBB == SwitchMBB)
10154           visitSwitchCase(CB, SwitchMBB);
10155         else
10156           SwitchCases.push_back(CB);
10157 
10158         break;
10159       }
10160     }
10161     CurMBB = Fallthrough;
10162   }
10163 }
10164 
10165 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
10166                                               CaseClusterIt First,
10167                                               CaseClusterIt Last) {
10168   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
10169     if (X.Prob != CC.Prob)
10170       return X.Prob > CC.Prob;
10171 
10172     // Ties are broken by comparing the case value.
10173     return X.Low->getValue().slt(CC.Low->getValue());
10174   });
10175 }
10176 
10177 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
10178                                         const SwitchWorkListItem &W,
10179                                         Value *Cond,
10180                                         MachineBasicBlock *SwitchMBB) {
10181   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
10182          "Clusters not sorted?");
10183 
10184   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
10185 
10186   // Balance the tree based on branch probabilities to create a near-optimal (in
10187   // terms of search time given key frequency) binary search tree. See e.g. Kurt
10188   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
10189   CaseClusterIt LastLeft = W.FirstCluster;
10190   CaseClusterIt FirstRight = W.LastCluster;
10191   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
10192   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
10193 
10194   // Move LastLeft and FirstRight towards each other from opposite directions to
10195   // find a partitioning of the clusters which balances the probability on both
10196   // sides. If LeftProb and RightProb are equal, alternate which side is
10197   // taken to ensure 0-probability nodes are distributed evenly.
10198   unsigned I = 0;
10199   while (LastLeft + 1 < FirstRight) {
10200     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
10201       LeftProb += (++LastLeft)->Prob;
10202     else
10203       RightProb += (--FirstRight)->Prob;
10204     I++;
10205   }
10206 
10207   while (true) {
10208     // Our binary search tree differs from a typical BST in that ours can have up
10209     // to three values in each leaf. The pivot selection above doesn't take that
10210     // into account, which means the tree might require more nodes and be less
10211     // efficient. We compensate for this here.
10212 
10213     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
10214     unsigned NumRight = W.LastCluster - FirstRight + 1;
10215 
10216     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
10217       // If one side has less than 3 clusters, and the other has more than 3,
10218       // consider taking a cluster from the other side.
10219 
10220       if (NumLeft < NumRight) {
10221         // Consider moving the first cluster on the right to the left side.
10222         CaseCluster &CC = *FirstRight;
10223         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10224         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10225         if (LeftSideRank <= RightSideRank) {
10226           // Moving the cluster to the left does not demote it.
10227           ++LastLeft;
10228           ++FirstRight;
10229           continue;
10230         }
10231       } else {
10232         assert(NumRight < NumLeft);
10233         // Consider moving the last element on the left to the right side.
10234         CaseCluster &CC = *LastLeft;
10235         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10236         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10237         if (RightSideRank <= LeftSideRank) {
10238           // Moving the cluster to the right does not demot it.
10239           --LastLeft;
10240           --FirstRight;
10241           continue;
10242         }
10243       }
10244     }
10245     break;
10246   }
10247 
10248   assert(LastLeft + 1 == FirstRight);
10249   assert(LastLeft >= W.FirstCluster);
10250   assert(FirstRight <= W.LastCluster);
10251 
10252   // Use the first element on the right as pivot since we will make less-than
10253   // comparisons against it.
10254   CaseClusterIt PivotCluster = FirstRight;
10255   assert(PivotCluster > W.FirstCluster);
10256   assert(PivotCluster <= W.LastCluster);
10257 
10258   CaseClusterIt FirstLeft = W.FirstCluster;
10259   CaseClusterIt LastRight = W.LastCluster;
10260 
10261   const ConstantInt *Pivot = PivotCluster->Low;
10262 
10263   // New blocks will be inserted immediately after the current one.
10264   MachineFunction::iterator BBI(W.MBB);
10265   ++BBI;
10266 
10267   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
10268   // we can branch to its destination directly if it's squeezed exactly in
10269   // between the known lower bound and Pivot - 1.
10270   MachineBasicBlock *LeftMBB;
10271   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
10272       FirstLeft->Low == W.GE &&
10273       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
10274     LeftMBB = FirstLeft->MBB;
10275   } else {
10276     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10277     FuncInfo.MF->insert(BBI, LeftMBB);
10278     WorkList.push_back(
10279         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
10280     // Put Cond in a virtual register to make it available from the new blocks.
10281     ExportFromCurrentBlock(Cond);
10282   }
10283 
10284   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
10285   // single cluster, RHS.Low == Pivot, and we can branch to its destination
10286   // directly if RHS.High equals the current upper bound.
10287   MachineBasicBlock *RightMBB;
10288   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
10289       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
10290     RightMBB = FirstRight->MBB;
10291   } else {
10292     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10293     FuncInfo.MF->insert(BBI, RightMBB);
10294     WorkList.push_back(
10295         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
10296     // Put Cond in a virtual register to make it available from the new blocks.
10297     ExportFromCurrentBlock(Cond);
10298   }
10299 
10300   // Create the CaseBlock record that will be used to lower the branch.
10301   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
10302                getCurSDLoc(), LeftProb, RightProb);
10303 
10304   if (W.MBB == SwitchMBB)
10305     visitSwitchCase(CB, SwitchMBB);
10306   else
10307     SwitchCases.push_back(CB);
10308 }
10309 
10310 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
10311 // from the swith statement.
10312 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
10313                                             BranchProbability PeeledCaseProb) {
10314   if (PeeledCaseProb == BranchProbability::getOne())
10315     return BranchProbability::getZero();
10316   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
10317 
10318   uint32_t Numerator = CaseProb.getNumerator();
10319   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
10320   return BranchProbability(Numerator, std::max(Numerator, Denominator));
10321 }
10322 
10323 // Try to peel the top probability case if it exceeds the threshold.
10324 // Return current MachineBasicBlock for the switch statement if the peeling
10325 // does not occur.
10326 // If the peeling is performed, return the newly created MachineBasicBlock
10327 // for the peeled switch statement. Also update Clusters to remove the peeled
10328 // case. PeeledCaseProb is the BranchProbability for the peeled case.
10329 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
10330     const SwitchInst &SI, CaseClusterVector &Clusters,
10331     BranchProbability &PeeledCaseProb) {
10332   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10333   // Don't perform if there is only one cluster or optimizing for size.
10334   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
10335       TM.getOptLevel() == CodeGenOpt::None ||
10336       SwitchMBB->getParent()->getFunction().optForMinSize())
10337     return SwitchMBB;
10338 
10339   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
10340   unsigned PeeledCaseIndex = 0;
10341   bool SwitchPeeled = false;
10342   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
10343     CaseCluster &CC = Clusters[Index];
10344     if (CC.Prob < TopCaseProb)
10345       continue;
10346     TopCaseProb = CC.Prob;
10347     PeeledCaseIndex = Index;
10348     SwitchPeeled = true;
10349   }
10350   if (!SwitchPeeled)
10351     return SwitchMBB;
10352 
10353   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
10354                     << TopCaseProb << "\n");
10355 
10356   // Record the MBB for the peeled switch statement.
10357   MachineFunction::iterator BBI(SwitchMBB);
10358   ++BBI;
10359   MachineBasicBlock *PeeledSwitchMBB =
10360       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
10361   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
10362 
10363   ExportFromCurrentBlock(SI.getCondition());
10364   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
10365   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
10366                           nullptr,   nullptr,      TopCaseProb.getCompl()};
10367   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
10368 
10369   Clusters.erase(PeeledCaseIt);
10370   for (CaseCluster &CC : Clusters) {
10371     LLVM_DEBUG(
10372         dbgs() << "Scale the probablity for one cluster, before scaling: "
10373                << CC.Prob << "\n");
10374     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
10375     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
10376   }
10377   PeeledCaseProb = TopCaseProb;
10378   return PeeledSwitchMBB;
10379 }
10380 
10381 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
10382   // Extract cases from the switch.
10383   BranchProbabilityInfo *BPI = FuncInfo.BPI;
10384   CaseClusterVector Clusters;
10385   Clusters.reserve(SI.getNumCases());
10386   for (auto I : SI.cases()) {
10387     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
10388     const ConstantInt *CaseVal = I.getCaseValue();
10389     BranchProbability Prob =
10390         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
10391             : BranchProbability(1, SI.getNumCases() + 1);
10392     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
10393   }
10394 
10395   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
10396 
10397   // Cluster adjacent cases with the same destination. We do this at all
10398   // optimization levels because it's cheap to do and will make codegen faster
10399   // if there are many clusters.
10400   sortAndRangeify(Clusters);
10401 
10402   if (TM.getOptLevel() != CodeGenOpt::None) {
10403     // Replace an unreachable default with the most popular destination.
10404     // FIXME: Exploit unreachable default more aggressively.
10405     bool UnreachableDefault =
10406         isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
10407     if (UnreachableDefault && !Clusters.empty()) {
10408       DenseMap<const BasicBlock *, unsigned> Popularity;
10409       unsigned MaxPop = 0;
10410       const BasicBlock *MaxBB = nullptr;
10411       for (auto I : SI.cases()) {
10412         const BasicBlock *BB = I.getCaseSuccessor();
10413         if (++Popularity[BB] > MaxPop) {
10414           MaxPop = Popularity[BB];
10415           MaxBB = BB;
10416         }
10417       }
10418       // Set new default.
10419       assert(MaxPop > 0 && MaxBB);
10420       DefaultMBB = FuncInfo.MBBMap[MaxBB];
10421 
10422       // Remove cases that were pointing to the destination that is now the
10423       // default.
10424       CaseClusterVector New;
10425       New.reserve(Clusters.size());
10426       for (CaseCluster &CC : Clusters) {
10427         if (CC.MBB != DefaultMBB)
10428           New.push_back(CC);
10429       }
10430       Clusters = std::move(New);
10431     }
10432   }
10433 
10434   // The branch probablity of the peeled case.
10435   BranchProbability PeeledCaseProb = BranchProbability::getZero();
10436   MachineBasicBlock *PeeledSwitchMBB =
10437       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
10438 
10439   // If there is only the default destination, jump there directly.
10440   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10441   if (Clusters.empty()) {
10442     assert(PeeledSwitchMBB == SwitchMBB);
10443     SwitchMBB->addSuccessor(DefaultMBB);
10444     if (DefaultMBB != NextBlock(SwitchMBB)) {
10445       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
10446                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
10447     }
10448     return;
10449   }
10450 
10451   findJumpTables(Clusters, &SI, DefaultMBB);
10452   findBitTestClusters(Clusters, &SI);
10453 
10454   LLVM_DEBUG({
10455     dbgs() << "Case clusters: ";
10456     for (const CaseCluster &C : Clusters) {
10457       if (C.Kind == CC_JumpTable)
10458         dbgs() << "JT:";
10459       if (C.Kind == CC_BitTests)
10460         dbgs() << "BT:";
10461 
10462       C.Low->getValue().print(dbgs(), true);
10463       if (C.Low != C.High) {
10464         dbgs() << '-';
10465         C.High->getValue().print(dbgs(), true);
10466       }
10467       dbgs() << ' ';
10468     }
10469     dbgs() << '\n';
10470   });
10471 
10472   assert(!Clusters.empty());
10473   SwitchWorkList WorkList;
10474   CaseClusterIt First = Clusters.begin();
10475   CaseClusterIt Last = Clusters.end() - 1;
10476   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
10477   // Scale the branchprobability for DefaultMBB if the peel occurs and
10478   // DefaultMBB is not replaced.
10479   if (PeeledCaseProb != BranchProbability::getZero() &&
10480       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
10481     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
10482   WorkList.push_back(
10483       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
10484 
10485   while (!WorkList.empty()) {
10486     SwitchWorkListItem W = WorkList.back();
10487     WorkList.pop_back();
10488     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
10489 
10490     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
10491         !DefaultMBB->getParent()->getFunction().optForMinSize()) {
10492       // For optimized builds, lower large range as a balanced binary tree.
10493       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
10494       continue;
10495     }
10496 
10497     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
10498   }
10499 }
10500