xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp (revision 297eecfb02bb25902531dbb5c3b9a88caf8adf29) 
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that X86 uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "X86ISelLowering.h"
15 #include "MCTargetDesc/X86ShuffleDecode.h"
16 #include "X86.h"
17 #include "X86CallingConv.h"
18 #include "X86FrameLowering.h"
19 #include "X86InstrBuilder.h"
20 #include "X86IntrinsicsInfo.h"
21 #include "X86MachineFunctionInfo.h"
22 #include "X86TargetMachine.h"
23 #include "X86TargetObjectFile.h"
24 #include "llvm/ADT/SmallBitVector.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/Statistic.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/ADT/StringSwitch.h"
29 #include "llvm/Analysis/BlockFrequencyInfo.h"
30 #include "llvm/Analysis/ObjCARCUtil.h"
31 #include "llvm/Analysis/ProfileSummaryInfo.h"
32 #include "llvm/Analysis/VectorUtils.h"
33 #include "llvm/CodeGen/IntrinsicLowering.h"
34 #include "llvm/CodeGen/MachineFrameInfo.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineInstrBuilder.h"
37 #include "llvm/CodeGen/MachineJumpTableInfo.h"
38 #include "llvm/CodeGen/MachineLoopInfo.h"
39 #include "llvm/CodeGen/MachineModuleInfo.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/TargetLowering.h"
42 #include "llvm/CodeGen/WinEHFuncInfo.h"
43 #include "llvm/IR/CallingConv.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DerivedTypes.h"
46 #include "llvm/IR/EHPersonalities.h"
47 #include "llvm/IR/Function.h"
48 #include "llvm/IR/GlobalAlias.h"
49 #include "llvm/IR/GlobalVariable.h"
50 #include "llvm/IR/IRBuilder.h"
51 #include "llvm/IR/Instructions.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/PatternMatch.h"
54 #include "llvm/MC/MCAsmInfo.h"
55 #include "llvm/MC/MCContext.h"
56 #include "llvm/MC/MCExpr.h"
57 #include "llvm/MC/MCSymbol.h"
58 #include "llvm/Support/CommandLine.h"
59 #include "llvm/Support/Debug.h"
60 #include "llvm/Support/ErrorHandling.h"
61 #include "llvm/Support/KnownBits.h"
62 #include "llvm/Support/MathExtras.h"
63 #include "llvm/Target/TargetOptions.h"
64 #include <algorithm>
65 #include <bitset>
66 #include <cctype>
67 #include <numeric>
68 using namespace llvm;
69 
70 #define DEBUG_TYPE "x86-isel"
71 
72 static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
73     "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
74     cl::desc(
75         "Sets the preferable loop alignment for experiments (as log2 bytes) "
76         "for innermost loops only. If specified, this option overrides "
77         "alignment set by x86-experimental-pref-loop-alignment."),
78     cl::Hidden);
79 
80 static cl::opt<bool> MulConstantOptimization(
81     "mul-constant-optimization", cl::init(true),
82     cl::desc("Replace 'mul x, Const' with more effective instructions like "
83              "SHIFT, LEA, etc."),
84     cl::Hidden);
85 
86 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
87                                      const X86Subtarget &STI)
88     : TargetLowering(TM), Subtarget(STI) {
89   bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
90   MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
91 
92   // Set up the TargetLowering object.
93 
94   // X86 is weird. It always uses i8 for shift amounts and setcc results.
95   setBooleanContents(ZeroOrOneBooleanContent);
96   // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
97   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
98 
99   // For 64-bit, since we have so many registers, use the ILP scheduler.
100   // For 32-bit, use the register pressure specific scheduling.
101   // For Atom, always use ILP scheduling.
102   if (Subtarget.isAtom())
103     setSchedulingPreference(Sched::ILP);
104   else if (Subtarget.is64Bit())
105     setSchedulingPreference(Sched::ILP);
106   else
107     setSchedulingPreference(Sched::RegPressure);
108   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
109   setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
110 
111   // Bypass expensive divides and use cheaper ones.
112   if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
113     if (Subtarget.hasSlowDivide32())
114       addBypassSlowDiv(32, 8);
115     if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
116       addBypassSlowDiv(64, 32);
117   }
118 
119   // Setup Windows compiler runtime calls.
120   if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
121     static const struct {
122       const RTLIB::Libcall Op;
123       const char * const Name;
124       const CallingConv::ID CC;
125     } LibraryCalls[] = {
126       { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
127       { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
128       { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
129       { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
130       { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
131     };
132 
133     for (const auto &LC : LibraryCalls) {
134       setLibcallName(LC.Op, LC.Name);
135       setLibcallCallingConv(LC.Op, LC.CC);
136     }
137   }
138 
139   if (Subtarget.getTargetTriple().isOSMSVCRT()) {
140     // MSVCRT doesn't have powi; fall back to pow
141     setLibcallName(RTLIB::POWI_F32, nullptr);
142     setLibcallName(RTLIB::POWI_F64, nullptr);
143   }
144 
145   if (Subtarget.canUseCMPXCHG16B())
146     setMaxAtomicSizeInBitsSupported(128);
147   else if (Subtarget.canUseCMPXCHG8B())
148     setMaxAtomicSizeInBitsSupported(64);
149   else
150     setMaxAtomicSizeInBitsSupported(32);
151 
152   setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
153 
154   setMaxLargeFPConvertBitWidthSupported(128);
155 
156   // Set up the register classes.
157   addRegisterClass(MVT::i8, &X86::GR8RegClass);
158   addRegisterClass(MVT::i16, &X86::GR16RegClass);
159   addRegisterClass(MVT::i32, &X86::GR32RegClass);
160   if (Subtarget.is64Bit())
161     addRegisterClass(MVT::i64, &X86::GR64RegClass);
162 
163   for (MVT VT : MVT::integer_valuetypes())
164     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
165 
166   // We don't accept any truncstore of integer registers.
167   setTruncStoreAction(MVT::i64, MVT::i32, Expand);
168   setTruncStoreAction(MVT::i64, MVT::i16, Expand);
169   setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
170   setTruncStoreAction(MVT::i32, MVT::i16, Expand);
171   setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
172   setTruncStoreAction(MVT::i16, MVT::i8,  Expand);
173 
174   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
175 
176   // SETOEQ and SETUNE require checking two conditions.
177   for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
178     setCondCodeAction(ISD::SETOEQ, VT, Expand);
179     setCondCodeAction(ISD::SETUNE, VT, Expand);
180   }
181 
182   // Integer absolute.
183   if (Subtarget.canUseCMOV()) {
184     setOperationAction(ISD::ABS            , MVT::i16  , Custom);
185     setOperationAction(ISD::ABS            , MVT::i32  , Custom);
186     if (Subtarget.is64Bit())
187       setOperationAction(ISD::ABS          , MVT::i64  , Custom);
188   }
189 
190   // Absolute difference.
191   for (auto Op : {ISD::ABDS, ISD::ABDU}) {
192     setOperationAction(Op                  , MVT::i8   , Custom);
193     setOperationAction(Op                  , MVT::i16  , Custom);
194     setOperationAction(Op                  , MVT::i32  , Custom);
195     if (Subtarget.is64Bit())
196      setOperationAction(Op                 , MVT::i64  , Custom);
197   }
198 
199   // Signed saturation subtraction.
200   setOperationAction(ISD::SSUBSAT          , MVT::i8   , Custom);
201   setOperationAction(ISD::SSUBSAT          , MVT::i16  , Custom);
202   setOperationAction(ISD::SSUBSAT          , MVT::i32  , Custom);
203   if (Subtarget.is64Bit())
204     setOperationAction(ISD::SSUBSAT        , MVT::i64  , Custom);
205 
206   // Funnel shifts.
207   for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
208     // For slow shld targets we only lower for code size.
209     LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
210 
211     setOperationAction(ShiftOp             , MVT::i8   , Custom);
212     setOperationAction(ShiftOp             , MVT::i16  , Custom);
213     setOperationAction(ShiftOp             , MVT::i32  , ShiftDoubleAction);
214     if (Subtarget.is64Bit())
215       setOperationAction(ShiftOp           , MVT::i64  , ShiftDoubleAction);
216   }
217 
218   if (!Subtarget.useSoftFloat()) {
219     // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
220     // operation.
221     setOperationAction(ISD::UINT_TO_FP,        MVT::i8, Promote);
222     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
223     setOperationAction(ISD::UINT_TO_FP,        MVT::i16, Promote);
224     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
225     // We have an algorithm for SSE2, and we turn this into a 64-bit
226     // FILD or VCVTUSI2SS/SD for other targets.
227     setOperationAction(ISD::UINT_TO_FP,        MVT::i32, Custom);
228     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
229     // We have an algorithm for SSE2->double, and we turn this into a
230     // 64-bit FILD followed by conditional FADD for other targets.
231     setOperationAction(ISD::UINT_TO_FP,        MVT::i64, Custom);
232     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
233 
234     // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
235     // this operation.
236     setOperationAction(ISD::SINT_TO_FP,        MVT::i8, Promote);
237     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
238     // SSE has no i16 to fp conversion, only i32. We promote in the handler
239     // to allow f80 to use i16 and f64 to use i16 with sse1 only
240     setOperationAction(ISD::SINT_TO_FP,        MVT::i16, Custom);
241     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
242     // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
243     setOperationAction(ISD::SINT_TO_FP,        MVT::i32, Custom);
244     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
245     // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
246     // are Legal, f80 is custom lowered.
247     setOperationAction(ISD::SINT_TO_FP,        MVT::i64, Custom);
248     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
249 
250     // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
251     // this operation.
252     setOperationAction(ISD::FP_TO_SINT,        MVT::i8,  Promote);
253     // FIXME: This doesn't generate invalid exception when it should. PR44019.
254     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8,  Promote);
255     setOperationAction(ISD::FP_TO_SINT,        MVT::i16, Custom);
256     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
257     setOperationAction(ISD::FP_TO_SINT,        MVT::i32, Custom);
258     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
259     // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
260     // are Legal, f80 is custom lowered.
261     setOperationAction(ISD::FP_TO_SINT,        MVT::i64, Custom);
262     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
263 
264     // Handle FP_TO_UINT by promoting the destination to a larger signed
265     // conversion.
266     setOperationAction(ISD::FP_TO_UINT,        MVT::i8,  Promote);
267     // FIXME: This doesn't generate invalid exception when it should. PR44019.
268     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8,  Promote);
269     setOperationAction(ISD::FP_TO_UINT,        MVT::i16, Promote);
270     // FIXME: This doesn't generate invalid exception when it should. PR44019.
271     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
272     setOperationAction(ISD::FP_TO_UINT,        MVT::i32, Custom);
273     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
274     setOperationAction(ISD::FP_TO_UINT,        MVT::i64, Custom);
275     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
276 
277     setOperationAction(ISD::LRINT,             MVT::f32, Custom);
278     setOperationAction(ISD::LRINT,             MVT::f64, Custom);
279     setOperationAction(ISD::LLRINT,            MVT::f32, Custom);
280     setOperationAction(ISD::LLRINT,            MVT::f64, Custom);
281 
282     if (!Subtarget.is64Bit()) {
283       setOperationAction(ISD::LRINT,  MVT::i64, Custom);
284       setOperationAction(ISD::LLRINT, MVT::i64, Custom);
285     }
286   }
287 
288   if (Subtarget.hasSSE2()) {
289     // Custom lowering for saturating float to int conversions.
290     // We handle promotion to larger result types manually.
291     for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
292       setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
293       setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
294     }
295     if (Subtarget.is64Bit()) {
296       setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
297       setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
298     }
299   }
300 
301   // Handle address space casts between mixed sized pointers.
302   setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
303   setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
304 
305   // TODO: when we have SSE, these could be more efficient, by using movd/movq.
306   if (!Subtarget.hasSSE2()) {
307     setOperationAction(ISD::BITCAST        , MVT::f32  , Expand);
308     setOperationAction(ISD::BITCAST        , MVT::i32  , Expand);
309     if (Subtarget.is64Bit()) {
310       setOperationAction(ISD::BITCAST      , MVT::f64  , Expand);
311       // Without SSE, i64->f64 goes through memory.
312       setOperationAction(ISD::BITCAST      , MVT::i64  , Expand);
313     }
314   } else if (!Subtarget.is64Bit())
315     setOperationAction(ISD::BITCAST      , MVT::i64  , Custom);
316 
317   // Scalar integer divide and remainder are lowered to use operations that
318   // produce two results, to match the available instructions. This exposes
319   // the two-result form to trivial CSE, which is able to combine x/y and x%y
320   // into a single instruction.
321   //
322   // Scalar integer multiply-high is also lowered to use two-result
323   // operations, to match the available instructions. However, plain multiply
324   // (low) operations are left as Legal, as there are single-result
325   // instructions for this in x86. Using the two-result multiply instructions
326   // when both high and low results are needed must be arranged by dagcombine.
327   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
328     setOperationAction(ISD::MULHS, VT, Expand);
329     setOperationAction(ISD::MULHU, VT, Expand);
330     setOperationAction(ISD::SDIV, VT, Expand);
331     setOperationAction(ISD::UDIV, VT, Expand);
332     setOperationAction(ISD::SREM, VT, Expand);
333     setOperationAction(ISD::UREM, VT, Expand);
334   }
335 
336   setOperationAction(ISD::BR_JT            , MVT::Other, Expand);
337   setOperationAction(ISD::BRCOND           , MVT::Other, Custom);
338   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
339                    MVT::i8,  MVT::i16, MVT::i32, MVT::i64 }) {
340     setOperationAction(ISD::BR_CC,     VT, Expand);
341     setOperationAction(ISD::SELECT_CC, VT, Expand);
342   }
343   if (Subtarget.is64Bit())
344     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
345   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Legal);
346   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Legal);
347   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
348 
349   setOperationAction(ISD::FREM             , MVT::f32  , Expand);
350   setOperationAction(ISD::FREM             , MVT::f64  , Expand);
351   setOperationAction(ISD::FREM             , MVT::f80  , Expand);
352   setOperationAction(ISD::FREM             , MVT::f128 , Expand);
353 
354   if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
355     setOperationAction(ISD::GET_ROUNDING   , MVT::i32  , Custom);
356     setOperationAction(ISD::SET_ROUNDING   , MVT::Other, Custom);
357     setOperationAction(ISD::GET_FPENV_MEM  , MVT::Other, Custom);
358     setOperationAction(ISD::SET_FPENV_MEM  , MVT::Other, Custom);
359     setOperationAction(ISD::RESET_FPENV    , MVT::Other, Custom);
360   }
361 
362   // Promote the i8 variants and force them on up to i32 which has a shorter
363   // encoding.
364   setOperationPromotedToType(ISD::CTTZ           , MVT::i8   , MVT::i32);
365   setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8   , MVT::i32);
366   // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
367   // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
368   // promote that too.
369   setOperationPromotedToType(ISD::CTTZ           , MVT::i16  , MVT::i32);
370   setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16  , MVT::i32);
371 
372   if (!Subtarget.hasBMI()) {
373     setOperationAction(ISD::CTTZ           , MVT::i32  , Custom);
374     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32  , Legal);
375     if (Subtarget.is64Bit()) {
376       setOperationAction(ISD::CTTZ         , MVT::i64  , Custom);
377       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
378     }
379   }
380 
381   if (Subtarget.hasLZCNT()) {
382     // When promoting the i8 variants, force them to i32 for a shorter
383     // encoding.
384     setOperationPromotedToType(ISD::CTLZ           , MVT::i8   , MVT::i32);
385     setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8   , MVT::i32);
386   } else {
387     for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
388       if (VT == MVT::i64 && !Subtarget.is64Bit())
389         continue;
390       setOperationAction(ISD::CTLZ           , VT, Custom);
391       setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
392     }
393   }
394 
395   for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
396                   ISD::STRICT_FP_TO_FP16}) {
397     // Special handling for half-precision floating point conversions.
398     // If we don't have F16C support, then lower half float conversions
399     // into library calls.
400     setOperationAction(
401         Op, MVT::f32,
402         (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
403     // There's never any support for operations beyond MVT::f32.
404     setOperationAction(Op, MVT::f64, Expand);
405     setOperationAction(Op, MVT::f80, Expand);
406     setOperationAction(Op, MVT::f128, Expand);
407   }
408 
409   for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
410     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
411     setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
412     setTruncStoreAction(VT, MVT::f16, Expand);
413     setTruncStoreAction(VT, MVT::bf16, Expand);
414 
415     setOperationAction(ISD::BF16_TO_FP, VT, Expand);
416     setOperationAction(ISD::FP_TO_BF16, VT, Custom);
417   }
418 
419   setOperationAction(ISD::PARITY, MVT::i8, Custom);
420   setOperationAction(ISD::PARITY, MVT::i16, Custom);
421   setOperationAction(ISD::PARITY, MVT::i32, Custom);
422   if (Subtarget.is64Bit())
423     setOperationAction(ISD::PARITY, MVT::i64, Custom);
424   if (Subtarget.hasPOPCNT()) {
425     setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
426     // popcntw is longer to encode than popcntl and also has a false dependency
427     // on the dest that popcntl hasn't had since Cannon Lake.
428     setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
429   } else {
430     setOperationAction(ISD::CTPOP          , MVT::i8   , Expand);
431     setOperationAction(ISD::CTPOP          , MVT::i16  , Expand);
432     setOperationAction(ISD::CTPOP          , MVT::i32  , Expand);
433     if (Subtarget.is64Bit())
434       setOperationAction(ISD::CTPOP        , MVT::i64  , Expand);
435     else
436       setOperationAction(ISD::CTPOP        , MVT::i64  , Custom);
437   }
438 
439   setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
440 
441   if (!Subtarget.hasMOVBE())
442     setOperationAction(ISD::BSWAP          , MVT::i16  , Expand);
443 
444   // X86 wants to expand cmov itself.
445   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
446     setOperationAction(ISD::SELECT, VT, Custom);
447     setOperationAction(ISD::SETCC, VT, Custom);
448     setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
449     setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
450   }
451   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
452     if (VT == MVT::i64 && !Subtarget.is64Bit())
453       continue;
454     setOperationAction(ISD::SELECT, VT, Custom);
455     setOperationAction(ISD::SETCC,  VT, Custom);
456   }
457 
458   // Custom action for SELECT MMX and expand action for SELECT_CC MMX
459   setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
460   setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
461 
462   setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);
463   // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
464   // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
465   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
466   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
467   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
468   if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
469     setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
470 
471   // Darwin ABI issue.
472   for (auto VT : { MVT::i32, MVT::i64 }) {
473     if (VT == MVT::i64 && !Subtarget.is64Bit())
474       continue;
475     setOperationAction(ISD::ConstantPool    , VT, Custom);
476     setOperationAction(ISD::JumpTable       , VT, Custom);
477     setOperationAction(ISD::GlobalAddress   , VT, Custom);
478     setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
479     setOperationAction(ISD::ExternalSymbol  , VT, Custom);
480     setOperationAction(ISD::BlockAddress    , VT, Custom);
481   }
482 
483   // 64-bit shl, sra, srl (iff 32-bit x86)
484   for (auto VT : { MVT::i32, MVT::i64 }) {
485     if (VT == MVT::i64 && !Subtarget.is64Bit())
486       continue;
487     setOperationAction(ISD::SHL_PARTS, VT, Custom);
488     setOperationAction(ISD::SRA_PARTS, VT, Custom);
489     setOperationAction(ISD::SRL_PARTS, VT, Custom);
490   }
491 
492   if (Subtarget.hasSSEPrefetch() || Subtarget.hasThreeDNow())
493     setOperationAction(ISD::PREFETCH      , MVT::Other, Custom);
494 
495   setOperationAction(ISD::ATOMIC_FENCE  , MVT::Other, Custom);
496 
497   // Expand certain atomics
498   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
499     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
500     setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
501     setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
502     setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
503     setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
504     setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
505     setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
506   }
507 
508   if (!Subtarget.is64Bit())
509     setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
510 
511   if (Subtarget.canUseCMPXCHG16B())
512     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
513 
514   // FIXME - use subtarget debug flags
515   if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
516       !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
517       TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
518     setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
519   }
520 
521   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
522   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
523 
524   setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
525   setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
526 
527   setOperationAction(ISD::TRAP, MVT::Other, Legal);
528   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
529   if (Subtarget.isTargetPS())
530     setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
531   else
532     setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
533 
534   // VASTART needs to be custom lowered to use the VarArgsFrameIndex
535   setOperationAction(ISD::VASTART           , MVT::Other, Custom);
536   setOperationAction(ISD::VAEND             , MVT::Other, Expand);
537   bool Is64Bit = Subtarget.is64Bit();
538   setOperationAction(ISD::VAARG,  MVT::Other, Is64Bit ? Custom : Expand);
539   setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
540 
541   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
542   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
543 
544   setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
545 
546   // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
547   setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
548   setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
549 
550   setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
551 
552   auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
553     setOperationAction(ISD::FABS, VT, Action);
554     setOperationAction(ISD::FNEG, VT, Action);
555     setOperationAction(ISD::FCOPYSIGN, VT, Expand);
556     setOperationAction(ISD::FREM, VT, Action);
557     setOperationAction(ISD::FMA, VT, Action);
558     setOperationAction(ISD::FMINNUM, VT, Action);
559     setOperationAction(ISD::FMAXNUM, VT, Action);
560     setOperationAction(ISD::FMINIMUM, VT, Action);
561     setOperationAction(ISD::FMAXIMUM, VT, Action);
562     setOperationAction(ISD::FSIN, VT, Action);
563     setOperationAction(ISD::FCOS, VT, Action);
564     setOperationAction(ISD::FSINCOS, VT, Action);
565     setOperationAction(ISD::FSQRT, VT, Action);
566     setOperationAction(ISD::FPOW, VT, Action);
567     setOperationAction(ISD::FLOG, VT, Action);
568     setOperationAction(ISD::FLOG2, VT, Action);
569     setOperationAction(ISD::FLOG10, VT, Action);
570     setOperationAction(ISD::FEXP, VT, Action);
571     setOperationAction(ISD::FEXP2, VT, Action);
572     setOperationAction(ISD::FEXP10, VT, Action);
573     setOperationAction(ISD::FCEIL, VT, Action);
574     setOperationAction(ISD::FFLOOR, VT, Action);
575     setOperationAction(ISD::FNEARBYINT, VT, Action);
576     setOperationAction(ISD::FRINT, VT, Action);
577     setOperationAction(ISD::BR_CC, VT, Action);
578     setOperationAction(ISD::SETCC, VT, Action);
579     setOperationAction(ISD::SELECT, VT, Custom);
580     setOperationAction(ISD::SELECT_CC, VT, Action);
581     setOperationAction(ISD::FROUND, VT, Action);
582     setOperationAction(ISD::FROUNDEVEN, VT, Action);
583     setOperationAction(ISD::FTRUNC, VT, Action);
584     setOperationAction(ISD::FLDEXP, VT, Action);
585   };
586 
587   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
588     // f16, f32 and f64 use SSE.
589     // Set up the FP register classes.
590     addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
591                                                      : &X86::FR16RegClass);
592     addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
593                                                      : &X86::FR32RegClass);
594     addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
595                                                      : &X86::FR64RegClass);
596 
597     // Disable f32->f64 extload as we can only generate this in one instruction
598     // under optsize. So its easier to pattern match (fpext (load)) for that
599     // case instead of needing to emit 2 instructions for extload in the
600     // non-optsize case.
601     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
602 
603     for (auto VT : { MVT::f32, MVT::f64 }) {
604       // Use ANDPD to simulate FABS.
605       setOperationAction(ISD::FABS, VT, Custom);
606 
607       // Use XORP to simulate FNEG.
608       setOperationAction(ISD::FNEG, VT, Custom);
609 
610       // Use ANDPD and ORPD to simulate FCOPYSIGN.
611       setOperationAction(ISD::FCOPYSIGN, VT, Custom);
612 
613       // These might be better off as horizontal vector ops.
614       setOperationAction(ISD::FADD, VT, Custom);
615       setOperationAction(ISD::FSUB, VT, Custom);
616 
617       // We don't support sin/cos/fmod
618       setOperationAction(ISD::FSIN   , VT, Expand);
619       setOperationAction(ISD::FCOS   , VT, Expand);
620       setOperationAction(ISD::FSINCOS, VT, Expand);
621     }
622 
623     // Half type will be promoted by default.
624     setF16Action(MVT::f16, Promote);
625     setOperationAction(ISD::FADD, MVT::f16, Promote);
626     setOperationAction(ISD::FSUB, MVT::f16, Promote);
627     setOperationAction(ISD::FMUL, MVT::f16, Promote);
628     setOperationAction(ISD::FDIV, MVT::f16, Promote);
629     setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
630     setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
631     setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
632 
633     setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
634     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
635     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
636     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
637     setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
638     setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
639     setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
640     setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
641     setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
642     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
643     setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
644     setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
645     setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
646     setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
647     setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
648     setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
649     setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
650     setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
651     setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
652     setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
653     setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
654     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
655     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
656     setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
657     setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
658     setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
659     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
660     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
661     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
662 
663     setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
664     setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
665 
666     // Lower this to MOVMSK plus an AND.
667     setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
668     setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
669 
670   } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
671              (UseX87 || Is64Bit)) {
672     // Use SSE for f32, x87 for f64.
673     // Set up the FP register classes.
674     addRegisterClass(MVT::f32, &X86::FR32RegClass);
675     if (UseX87)
676       addRegisterClass(MVT::f64, &X86::RFP64RegClass);
677 
678     // Use ANDPS to simulate FABS.
679     setOperationAction(ISD::FABS , MVT::f32, Custom);
680 
681     // Use XORP to simulate FNEG.
682     setOperationAction(ISD::FNEG , MVT::f32, Custom);
683 
684     if (UseX87)
685       setOperationAction(ISD::UNDEF, MVT::f64, Expand);
686 
687     // Use ANDPS and ORPS to simulate FCOPYSIGN.
688     if (UseX87)
689       setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
690     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
691 
692     // We don't support sin/cos/fmod
693     setOperationAction(ISD::FSIN   , MVT::f32, Expand);
694     setOperationAction(ISD::FCOS   , MVT::f32, Expand);
695     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
696 
697     if (UseX87) {
698       // Always expand sin/cos functions even though x87 has an instruction.
699       setOperationAction(ISD::FSIN, MVT::f64, Expand);
700       setOperationAction(ISD::FCOS, MVT::f64, Expand);
701       setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
702     }
703   } else if (UseX87) {
704     // f32 and f64 in x87.
705     // Set up the FP register classes.
706     addRegisterClass(MVT::f64, &X86::RFP64RegClass);
707     addRegisterClass(MVT::f32, &X86::RFP32RegClass);
708 
709     for (auto VT : { MVT::f32, MVT::f64 }) {
710       setOperationAction(ISD::UNDEF,     VT, Expand);
711       setOperationAction(ISD::FCOPYSIGN, VT, Expand);
712 
713       // Always expand sin/cos functions even though x87 has an instruction.
714       setOperationAction(ISD::FSIN   , VT, Expand);
715       setOperationAction(ISD::FCOS   , VT, Expand);
716       setOperationAction(ISD::FSINCOS, VT, Expand);
717     }
718   }
719 
720   // Expand FP32 immediates into loads from the stack, save special cases.
721   if (isTypeLegal(MVT::f32)) {
722     if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
723       addLegalFPImmediate(APFloat(+0.0f)); // FLD0
724       addLegalFPImmediate(APFloat(+1.0f)); // FLD1
725       addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
726       addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
727     } else // SSE immediates.
728       addLegalFPImmediate(APFloat(+0.0f)); // xorps
729   }
730   // Expand FP64 immediates into loads from the stack, save special cases.
731   if (isTypeLegal(MVT::f64)) {
732     if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
733       addLegalFPImmediate(APFloat(+0.0)); // FLD0
734       addLegalFPImmediate(APFloat(+1.0)); // FLD1
735       addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
736       addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
737     } else // SSE immediates.
738       addLegalFPImmediate(APFloat(+0.0)); // xorpd
739   }
740   // Support fp16 0 immediate.
741   if (isTypeLegal(MVT::f16))
742     addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
743 
744   // Handle constrained floating-point operations of scalar.
745   setOperationAction(ISD::STRICT_FADD,      MVT::f32, Legal);
746   setOperationAction(ISD::STRICT_FADD,      MVT::f64, Legal);
747   setOperationAction(ISD::STRICT_FSUB,      MVT::f32, Legal);
748   setOperationAction(ISD::STRICT_FSUB,      MVT::f64, Legal);
749   setOperationAction(ISD::STRICT_FMUL,      MVT::f32, Legal);
750   setOperationAction(ISD::STRICT_FMUL,      MVT::f64, Legal);
751   setOperationAction(ISD::STRICT_FDIV,      MVT::f32, Legal);
752   setOperationAction(ISD::STRICT_FDIV,      MVT::f64, Legal);
753   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f32, Legal);
754   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f64, Legal);
755   setOperationAction(ISD::STRICT_FSQRT,     MVT::f32, Legal);
756   setOperationAction(ISD::STRICT_FSQRT,     MVT::f64, Legal);
757 
758   // We don't support FMA.
759   setOperationAction(ISD::FMA, MVT::f64, Expand);
760   setOperationAction(ISD::FMA, MVT::f32, Expand);
761 
762   // f80 always uses X87.
763   if (UseX87) {
764     addRegisterClass(MVT::f80, &X86::RFP80RegClass);
765     setOperationAction(ISD::UNDEF,     MVT::f80, Expand);
766     setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
767     {
768       APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
769       addLegalFPImmediate(TmpFlt);  // FLD0
770       TmpFlt.changeSign();
771       addLegalFPImmediate(TmpFlt);  // FLD0/FCHS
772 
773       bool ignored;
774       APFloat TmpFlt2(+1.0);
775       TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
776                       &ignored);
777       addLegalFPImmediate(TmpFlt2);  // FLD1
778       TmpFlt2.changeSign();
779       addLegalFPImmediate(TmpFlt2);  // FLD1/FCHS
780     }
781 
782     // Always expand sin/cos functions even though x87 has an instruction.
783     setOperationAction(ISD::FSIN   , MVT::f80, Expand);
784     setOperationAction(ISD::FCOS   , MVT::f80, Expand);
785     setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
786 
787     setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
788     setOperationAction(ISD::FCEIL,  MVT::f80, Expand);
789     setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
790     setOperationAction(ISD::FRINT,  MVT::f80, Expand);
791     setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
792     setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
793     setOperationAction(ISD::FMA, MVT::f80, Expand);
794     setOperationAction(ISD::LROUND, MVT::f80, Expand);
795     setOperationAction(ISD::LLROUND, MVT::f80, Expand);
796     setOperationAction(ISD::LRINT, MVT::f80, Custom);
797     setOperationAction(ISD::LLRINT, MVT::f80, Custom);
798 
799     // Handle constrained floating-point operations of scalar.
800     setOperationAction(ISD::STRICT_FADD     , MVT::f80, Legal);
801     setOperationAction(ISD::STRICT_FSUB     , MVT::f80, Legal);
802     setOperationAction(ISD::STRICT_FMUL     , MVT::f80, Legal);
803     setOperationAction(ISD::STRICT_FDIV     , MVT::f80, Legal);
804     setOperationAction(ISD::STRICT_FSQRT    , MVT::f80, Legal);
805     if (isTypeLegal(MVT::f16)) {
806       setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
807       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
808     } else {
809       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
810     }
811     // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
812     // as Custom.
813     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
814   }
815 
816   // f128 uses xmm registers, but most operations require libcalls.
817   if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
818     addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
819                                                    : &X86::VR128RegClass);
820 
821     addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
822 
823     setOperationAction(ISD::FADD,        MVT::f128, LibCall);
824     setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
825     setOperationAction(ISD::FSUB,        MVT::f128, LibCall);
826     setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
827     setOperationAction(ISD::FDIV,        MVT::f128, LibCall);
828     setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
829     setOperationAction(ISD::FMUL,        MVT::f128, LibCall);
830     setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
831     setOperationAction(ISD::FMA,         MVT::f128, LibCall);
832     setOperationAction(ISD::STRICT_FMA,  MVT::f128, LibCall);
833 
834     setOperationAction(ISD::FABS, MVT::f128, Custom);
835     setOperationAction(ISD::FNEG, MVT::f128, Custom);
836     setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
837 
838     setOperationAction(ISD::FSIN,         MVT::f128, LibCall);
839     setOperationAction(ISD::STRICT_FSIN,  MVT::f128, LibCall);
840     setOperationAction(ISD::FCOS,         MVT::f128, LibCall);
841     setOperationAction(ISD::STRICT_FCOS,  MVT::f128, LibCall);
842     setOperationAction(ISD::FSINCOS,      MVT::f128, LibCall);
843     // No STRICT_FSINCOS
844     setOperationAction(ISD::FSQRT,        MVT::f128, LibCall);
845     setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
846 
847     setOperationAction(ISD::FP_EXTEND,        MVT::f128, Custom);
848     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
849     // We need to custom handle any FP_ROUND with an f128 input, but
850     // LegalizeDAG uses the result type to know when to run a custom handler.
851     // So we have to list all legal floating point result types here.
852     if (isTypeLegal(MVT::f32)) {
853       setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
854       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
855     }
856     if (isTypeLegal(MVT::f64)) {
857       setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
858       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
859     }
860     if (isTypeLegal(MVT::f80)) {
861       setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
862       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
863     }
864 
865     setOperationAction(ISD::SETCC, MVT::f128, Custom);
866 
867     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
868     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
869     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
870     setTruncStoreAction(MVT::f128, MVT::f32, Expand);
871     setTruncStoreAction(MVT::f128, MVT::f64, Expand);
872     setTruncStoreAction(MVT::f128, MVT::f80, Expand);
873   }
874 
875   // Always use a library call for pow.
876   setOperationAction(ISD::FPOW             , MVT::f32  , Expand);
877   setOperationAction(ISD::FPOW             , MVT::f64  , Expand);
878   setOperationAction(ISD::FPOW             , MVT::f80  , Expand);
879   setOperationAction(ISD::FPOW             , MVT::f128 , Expand);
880 
881   setOperationAction(ISD::FLOG, MVT::f80, Expand);
882   setOperationAction(ISD::FLOG2, MVT::f80, Expand);
883   setOperationAction(ISD::FLOG10, MVT::f80, Expand);
884   setOperationAction(ISD::FEXP, MVT::f80, Expand);
885   setOperationAction(ISD::FEXP2, MVT::f80, Expand);
886   setOperationAction(ISD::FEXP10, MVT::f80, Expand);
887   setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
888   setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
889 
890   // Some FP actions are always expanded for vector types.
891   for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
892                    MVT::v4f32, MVT::v8f32,  MVT::v16f32,
893                    MVT::v2f64, MVT::v4f64,  MVT::v8f64 }) {
894     setOperationAction(ISD::FSIN,      VT, Expand);
895     setOperationAction(ISD::FSINCOS,   VT, Expand);
896     setOperationAction(ISD::FCOS,      VT, Expand);
897     setOperationAction(ISD::FREM,      VT, Expand);
898     setOperationAction(ISD::FCOPYSIGN, VT, Expand);
899     setOperationAction(ISD::FPOW,      VT, Expand);
900     setOperationAction(ISD::FLOG,      VT, Expand);
901     setOperationAction(ISD::FLOG2,     VT, Expand);
902     setOperationAction(ISD::FLOG10,    VT, Expand);
903     setOperationAction(ISD::FEXP,      VT, Expand);
904     setOperationAction(ISD::FEXP2,     VT, Expand);
905     setOperationAction(ISD::FEXP10,    VT, Expand);
906   }
907 
908   // First set operation action for all vector types to either promote
909   // (for widening) or expand (for scalarization). Then we will selectively
910   // turn on ones that can be effectively codegen'd.
911   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
912     setOperationAction(ISD::SDIV, VT, Expand);
913     setOperationAction(ISD::UDIV, VT, Expand);
914     setOperationAction(ISD::SREM, VT, Expand);
915     setOperationAction(ISD::UREM, VT, Expand);
916     setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
917     setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
918     setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
919     setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
920     setOperationAction(ISD::FMA,  VT, Expand);
921     setOperationAction(ISD::FFLOOR, VT, Expand);
922     setOperationAction(ISD::FCEIL, VT, Expand);
923     setOperationAction(ISD::FTRUNC, VT, Expand);
924     setOperationAction(ISD::FRINT, VT, Expand);
925     setOperationAction(ISD::FNEARBYINT, VT, Expand);
926     setOperationAction(ISD::FROUNDEVEN, VT, Expand);
927     setOperationAction(ISD::SMUL_LOHI, VT, Expand);
928     setOperationAction(ISD::MULHS, VT, Expand);
929     setOperationAction(ISD::UMUL_LOHI, VT, Expand);
930     setOperationAction(ISD::MULHU, VT, Expand);
931     setOperationAction(ISD::SDIVREM, VT, Expand);
932     setOperationAction(ISD::UDIVREM, VT, Expand);
933     setOperationAction(ISD::CTPOP, VT, Expand);
934     setOperationAction(ISD::CTTZ, VT, Expand);
935     setOperationAction(ISD::CTLZ, VT, Expand);
936     setOperationAction(ISD::ROTL, VT, Expand);
937     setOperationAction(ISD::ROTR, VT, Expand);
938     setOperationAction(ISD::BSWAP, VT, Expand);
939     setOperationAction(ISD::SETCC, VT, Expand);
940     setOperationAction(ISD::FP_TO_UINT, VT, Expand);
941     setOperationAction(ISD::FP_TO_SINT, VT, Expand);
942     setOperationAction(ISD::UINT_TO_FP, VT, Expand);
943     setOperationAction(ISD::SINT_TO_FP, VT, Expand);
944     setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
945     setOperationAction(ISD::TRUNCATE, VT, Expand);
946     setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
947     setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
948     setOperationAction(ISD::ANY_EXTEND, VT, Expand);
949     setOperationAction(ISD::SELECT_CC, VT, Expand);
950     for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
951       setTruncStoreAction(InnerVT, VT, Expand);
952 
953       setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
954       setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
955 
956       // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
957       // types, we have to deal with them whether we ask for Expansion or not.
958       // Setting Expand causes its own optimisation problems though, so leave
959       // them legal.
960       if (VT.getVectorElementType() == MVT::i1)
961         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
962 
963       // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
964       // split/scalarized right now.
965       if (VT.getVectorElementType() == MVT::f16 ||
966           VT.getVectorElementType() == MVT::bf16)
967         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
968     }
969   }
970 
971   // FIXME: In order to prevent SSE instructions being expanded to MMX ones
972   // with -msoft-float, disable use of MMX as well.
973   if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
974     addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
975     // No operations on x86mmx supported, everything uses intrinsics.
976   }
977 
978   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
979     addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
980                                                     : &X86::VR128RegClass);
981 
982     setOperationAction(ISD::FMAXIMUM,           MVT::f32, Custom);
983     setOperationAction(ISD::FMINIMUM,           MVT::f32, Custom);
984 
985     setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
986     setOperationAction(ISD::FABS,               MVT::v4f32, Custom);
987     setOperationAction(ISD::FCOPYSIGN,          MVT::v4f32, Custom);
988     setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);
989     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
990     setOperationAction(ISD::VSELECT,            MVT::v4f32, Custom);
991     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
992     setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);
993 
994     setOperationAction(ISD::LOAD,               MVT::v2f32, Custom);
995     setOperationAction(ISD::STORE,              MVT::v2f32, Custom);
996 
997     setOperationAction(ISD::STRICT_FADD,        MVT::v4f32, Legal);
998     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f32, Legal);
999     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f32, Legal);
1000     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f32, Legal);
1001     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f32, Legal);
1002   }
1003 
1004   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
1005     addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1006                                                     : &X86::VR128RegClass);
1007 
1008     // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
1009     // registers cannot be used even for integer operations.
1010     addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1011                                                     : &X86::VR128RegClass);
1012     addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1013                                                     : &X86::VR128RegClass);
1014     addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1015                                                     : &X86::VR128RegClass);
1016     addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1017                                                     : &X86::VR128RegClass);
1018     addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1019                                                     : &X86::VR128RegClass);
1020 
1021     for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) {
1022       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1023       setOperationAction(ISD::FMINIMUM, VT, Custom);
1024     }
1025 
1026     for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
1027                      MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
1028       setOperationAction(ISD::SDIV, VT, Custom);
1029       setOperationAction(ISD::SREM, VT, Custom);
1030       setOperationAction(ISD::UDIV, VT, Custom);
1031       setOperationAction(ISD::UREM, VT, Custom);
1032     }
1033 
1034     setOperationAction(ISD::MUL,                MVT::v2i8,  Custom);
1035     setOperationAction(ISD::MUL,                MVT::v4i8,  Custom);
1036     setOperationAction(ISD::MUL,                MVT::v8i8,  Custom);
1037 
1038     setOperationAction(ISD::MUL,                MVT::v16i8, Custom);
1039     setOperationAction(ISD::MUL,                MVT::v4i32, Custom);
1040     setOperationAction(ISD::MUL,                MVT::v2i64, Custom);
1041     setOperationAction(ISD::MULHU,              MVT::v4i32, Custom);
1042     setOperationAction(ISD::MULHS,              MVT::v4i32, Custom);
1043     setOperationAction(ISD::MULHU,              MVT::v16i8, Custom);
1044     setOperationAction(ISD::MULHS,              MVT::v16i8, Custom);
1045     setOperationAction(ISD::MULHU,              MVT::v8i16, Legal);
1046     setOperationAction(ISD::MULHS,              MVT::v8i16, Legal);
1047     setOperationAction(ISD::MUL,                MVT::v8i16, Legal);
1048     setOperationAction(ISD::AVGCEILU,           MVT::v16i8, Legal);
1049     setOperationAction(ISD::AVGCEILU,           MVT::v8i16, Legal);
1050 
1051     setOperationAction(ISD::SMULO,              MVT::v16i8, Custom);
1052     setOperationAction(ISD::UMULO,              MVT::v16i8, Custom);
1053     setOperationAction(ISD::UMULO,              MVT::v2i32, Custom);
1054 
1055     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
1056     setOperationAction(ISD::FABS,               MVT::v2f64, Custom);
1057     setOperationAction(ISD::FCOPYSIGN,          MVT::v2f64, Custom);
1058 
1059     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1060       setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
1061       setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
1062       setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
1063       setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
1064     }
1065 
1066     setOperationAction(ISD::ABDU,               MVT::v16i8, Custom);
1067     setOperationAction(ISD::ABDS,               MVT::v16i8, Custom);
1068     setOperationAction(ISD::ABDU,               MVT::v8i16, Custom);
1069     setOperationAction(ISD::ABDS,               MVT::v8i16, Custom);
1070     setOperationAction(ISD::ABDU,               MVT::v4i32, Custom);
1071     setOperationAction(ISD::ABDS,               MVT::v4i32, Custom);
1072 
1073     setOperationAction(ISD::UADDSAT,            MVT::v16i8, Legal);
1074     setOperationAction(ISD::SADDSAT,            MVT::v16i8, Legal);
1075     setOperationAction(ISD::USUBSAT,            MVT::v16i8, Legal);
1076     setOperationAction(ISD::SSUBSAT,            MVT::v16i8, Legal);
1077     setOperationAction(ISD::UADDSAT,            MVT::v8i16, Legal);
1078     setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);
1079     setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);
1080     setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);
1081     setOperationAction(ISD::USUBSAT,            MVT::v4i32, Custom);
1082     setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);
1083 
1084     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i8, Custom);
1085     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
1086     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
1087     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
1088 
1089     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1090       setOperationAction(ISD::SETCC,              VT, Custom);
1091       setOperationAction(ISD::CTPOP,              VT, Custom);
1092       setOperationAction(ISD::ABS,                VT, Custom);
1093 
1094       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1095       // setcc all the way to isel and prefer SETGT in some isel patterns.
1096       setCondCodeAction(ISD::SETLT, VT, Custom);
1097       setCondCodeAction(ISD::SETLE, VT, Custom);
1098     }
1099 
1100     setOperationAction(ISD::SETCC,          MVT::v2f64, Custom);
1101     setOperationAction(ISD::SETCC,          MVT::v4f32, Custom);
1102     setOperationAction(ISD::STRICT_FSETCC,  MVT::v2f64, Custom);
1103     setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f32, Custom);
1104     setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
1105     setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
1106 
1107     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1108       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1109       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1110       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1111       setOperationAction(ISD::VSELECT,            VT, Custom);
1112       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1113     }
1114 
1115     for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
1116       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1117       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1118       setOperationAction(ISD::VSELECT,            VT, Custom);
1119 
1120       if (VT == MVT::v2i64 && !Subtarget.is64Bit())
1121         continue;
1122 
1123       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1124       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1125     }
1126     setF16Action(MVT::v8f16, Expand);
1127     setOperationAction(ISD::FADD, MVT::v8f16, Expand);
1128     setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
1129     setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
1130     setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
1131     setOperationAction(ISD::FNEG, MVT::v8f16, Custom);
1132     setOperationAction(ISD::FABS, MVT::v8f16, Custom);
1133     setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
1134 
1135     // Custom lower v2i64 and v2f64 selects.
1136     setOperationAction(ISD::SELECT,             MVT::v2f64, Custom);
1137     setOperationAction(ISD::SELECT,             MVT::v2i64, Custom);
1138     setOperationAction(ISD::SELECT,             MVT::v4i32, Custom);
1139     setOperationAction(ISD::SELECT,             MVT::v8i16, Custom);
1140     setOperationAction(ISD::SELECT,             MVT::v8f16, Custom);
1141     setOperationAction(ISD::SELECT,             MVT::v16i8, Custom);
1142 
1143     setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Custom);
1144     setOperationAction(ISD::FP_TO_UINT,         MVT::v4i32, Custom);
1145     setOperationAction(ISD::FP_TO_SINT,         MVT::v2i32, Custom);
1146     setOperationAction(ISD::FP_TO_UINT,         MVT::v2i32, Custom);
1147     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v4i32, Custom);
1148     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v2i32, Custom);
1149 
1150     // Custom legalize these to avoid over promotion or custom promotion.
1151     for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
1152       setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
1153       setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
1154       setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1155       setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1156     }
1157 
1158     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Custom);
1159     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v4i32, Custom);
1160     setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);
1161     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2i32, Custom);
1162 
1163     setOperationAction(ISD::UINT_TO_FP,         MVT::v2i32, Custom);
1164     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2i32, Custom);
1165 
1166     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Custom);
1167     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v4i32, Custom);
1168 
1169     // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
1170     setOperationAction(ISD::SINT_TO_FP,         MVT::v2f32, Custom);
1171     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2f32, Custom);
1172     setOperationAction(ISD::UINT_TO_FP,         MVT::v2f32, Custom);
1173     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2f32, Custom);
1174 
1175     setOperationAction(ISD::FP_EXTEND,          MVT::v2f32, Custom);
1176     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v2f32, Custom);
1177     setOperationAction(ISD::FP_ROUND,           MVT::v2f32, Custom);
1178     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v2f32, Custom);
1179 
1180     // We want to legalize this to an f64 load rather than an i64 load on
1181     // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1182     // store.
1183     setOperationAction(ISD::LOAD,               MVT::v2i32, Custom);
1184     setOperationAction(ISD::LOAD,               MVT::v4i16, Custom);
1185     setOperationAction(ISD::LOAD,               MVT::v8i8,  Custom);
1186     setOperationAction(ISD::STORE,              MVT::v2i32, Custom);
1187     setOperationAction(ISD::STORE,              MVT::v4i16, Custom);
1188     setOperationAction(ISD::STORE,              MVT::v8i8,  Custom);
1189 
1190     // Add 32-bit vector stores to help vectorization opportunities.
1191     setOperationAction(ISD::STORE,              MVT::v2i16, Custom);
1192     setOperationAction(ISD::STORE,              MVT::v4i8,  Custom);
1193 
1194     setOperationAction(ISD::BITCAST,            MVT::v2i32, Custom);
1195     setOperationAction(ISD::BITCAST,            MVT::v4i16, Custom);
1196     setOperationAction(ISD::BITCAST,            MVT::v8i8,  Custom);
1197     if (!Subtarget.hasAVX512())
1198       setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1199 
1200     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1201     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1202     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1203 
1204     setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1205 
1206     setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);
1207     setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);
1208     setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);
1209     setOperationAction(ISD::TRUNCATE,    MVT::v2i64, Custom);
1210     setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
1211     setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
1212     setOperationAction(ISD::TRUNCATE,    MVT::v4i32, Custom);
1213     setOperationAction(ISD::TRUNCATE,    MVT::v4i64, Custom);
1214     setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
1215     setOperationAction(ISD::TRUNCATE,    MVT::v8i16, Custom);
1216     setOperationAction(ISD::TRUNCATE,    MVT::v8i32, Custom);
1217     setOperationAction(ISD::TRUNCATE,    MVT::v8i64, Custom);
1218     setOperationAction(ISD::TRUNCATE,    MVT::v16i8, Custom);
1219     setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Custom);
1220     setOperationAction(ISD::TRUNCATE,    MVT::v16i32, Custom);
1221     setOperationAction(ISD::TRUNCATE,    MVT::v16i64, Custom);
1222 
1223     // In the customized shift lowering, the legal v4i32/v2i64 cases
1224     // in AVX2 will be recognized.
1225     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1226       setOperationAction(ISD::SRL,              VT, Custom);
1227       setOperationAction(ISD::SHL,              VT, Custom);
1228       setOperationAction(ISD::SRA,              VT, Custom);
1229       if (VT == MVT::v2i64) continue;
1230       setOperationAction(ISD::ROTL,             VT, Custom);
1231       setOperationAction(ISD::ROTR,             VT, Custom);
1232       setOperationAction(ISD::FSHL,             VT, Custom);
1233       setOperationAction(ISD::FSHR,             VT, Custom);
1234     }
1235 
1236     setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);
1237     setOperationAction(ISD::STRICT_FADD,        MVT::v2f64, Legal);
1238     setOperationAction(ISD::STRICT_FSUB,        MVT::v2f64, Legal);
1239     setOperationAction(ISD::STRICT_FMUL,        MVT::v2f64, Legal);
1240     setOperationAction(ISD::STRICT_FDIV,        MVT::v2f64, Legal);
1241   }
1242 
1243   if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1244     setOperationAction(ISD::ABS,                MVT::v16i8, Legal);
1245     setOperationAction(ISD::ABS,                MVT::v8i16, Legal);
1246     setOperationAction(ISD::ABS,                MVT::v4i32, Legal);
1247     setOperationAction(ISD::BITREVERSE,         MVT::v16i8, Custom);
1248     setOperationAction(ISD::CTLZ,               MVT::v16i8, Custom);
1249     setOperationAction(ISD::CTLZ,               MVT::v8i16, Custom);
1250     setOperationAction(ISD::CTLZ,               MVT::v4i32, Custom);
1251     setOperationAction(ISD::CTLZ,               MVT::v2i64, Custom);
1252 
1253     // These might be better off as horizontal vector ops.
1254     setOperationAction(ISD::ADD,                MVT::i16, Custom);
1255     setOperationAction(ISD::ADD,                MVT::i32, Custom);
1256     setOperationAction(ISD::SUB,                MVT::i16, Custom);
1257     setOperationAction(ISD::SUB,                MVT::i32, Custom);
1258   }
1259 
1260   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1261     for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1262       setOperationAction(ISD::FFLOOR,            RoundedTy,  Legal);
1263       setOperationAction(ISD::STRICT_FFLOOR,     RoundedTy,  Legal);
1264       setOperationAction(ISD::FCEIL,             RoundedTy,  Legal);
1265       setOperationAction(ISD::STRICT_FCEIL,      RoundedTy,  Legal);
1266       setOperationAction(ISD::FTRUNC,            RoundedTy,  Legal);
1267       setOperationAction(ISD::STRICT_FTRUNC,     RoundedTy,  Legal);
1268       setOperationAction(ISD::FRINT,             RoundedTy,  Legal);
1269       setOperationAction(ISD::STRICT_FRINT,      RoundedTy,  Legal);
1270       setOperationAction(ISD::FNEARBYINT,        RoundedTy,  Legal);
1271       setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy,  Legal);
1272       setOperationAction(ISD::FROUNDEVEN,        RoundedTy,  Legal);
1273       setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy,  Legal);
1274 
1275       setOperationAction(ISD::FROUND,            RoundedTy,  Custom);
1276     }
1277 
1278     setOperationAction(ISD::SMAX,               MVT::v16i8, Legal);
1279     setOperationAction(ISD::SMAX,               MVT::v4i32, Legal);
1280     setOperationAction(ISD::UMAX,               MVT::v8i16, Legal);
1281     setOperationAction(ISD::UMAX,               MVT::v4i32, Legal);
1282     setOperationAction(ISD::SMIN,               MVT::v16i8, Legal);
1283     setOperationAction(ISD::SMIN,               MVT::v4i32, Legal);
1284     setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
1285     setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
1286 
1287     for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1288       setOperationAction(ISD::ABDS,             VT, Custom);
1289       setOperationAction(ISD::ABDU,             VT, Custom);
1290     }
1291 
1292     setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);
1293     setOperationAction(ISD::SADDSAT,            MVT::v2i64, Custom);
1294     setOperationAction(ISD::SSUBSAT,            MVT::v2i64, Custom);
1295 
1296     // FIXME: Do we need to handle scalar-to-vector here?
1297     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
1298     setOperationAction(ISD::SMULO,              MVT::v2i32, Custom);
1299 
1300     // We directly match byte blends in the backend as they match the VSELECT
1301     // condition form.
1302     setOperationAction(ISD::VSELECT,            MVT::v16i8, Legal);
1303 
1304     // SSE41 brings specific instructions for doing vector sign extend even in
1305     // cases where we don't have SRA.
1306     for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1307       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1308       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1309     }
1310 
1311     // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1312     for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1313       setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8,  Legal);
1314       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8,  Legal);
1315       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8,  Legal);
1316       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1317       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1318       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1319     }
1320 
1321     if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1322       // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1323       // do the pre and post work in the vector domain.
1324       setOperationAction(ISD::UINT_TO_FP,        MVT::v4i64, Custom);
1325       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1326       // We need to mark SINT_TO_FP as Custom even though we want to expand it
1327       // so that DAG combine doesn't try to turn it into uint_to_fp.
1328       setOperationAction(ISD::SINT_TO_FP,        MVT::v4i64, Custom);
1329       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1330     }
1331   }
1332 
1333   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
1334     setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
1335   }
1336 
1337   if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1338     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
1339                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1340       setOperationAction(ISD::ROTL, VT, Custom);
1341       setOperationAction(ISD::ROTR, VT, Custom);
1342     }
1343 
1344     // XOP can efficiently perform BITREVERSE with VPPERM.
1345     for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1346       setOperationAction(ISD::BITREVERSE, VT, Custom);
1347 
1348     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
1349                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1350       setOperationAction(ISD::BITREVERSE, VT, Custom);
1351   }
1352 
1353   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1354     bool HasInt256 = Subtarget.hasInt256();
1355 
1356     addRegisterClass(MVT::v32i8,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1357                                                      : &X86::VR256RegClass);
1358     addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1359                                                      : &X86::VR256RegClass);
1360     addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1361                                                      : &X86::VR256RegClass);
1362     addRegisterClass(MVT::v8i32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1363                                                      : &X86::VR256RegClass);
1364     addRegisterClass(MVT::v8f32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1365                                                      : &X86::VR256RegClass);
1366     addRegisterClass(MVT::v4i64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1367                                                      : &X86::VR256RegClass);
1368     addRegisterClass(MVT::v4f64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1369                                                      : &X86::VR256RegClass);
1370 
1371     for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1372       setOperationAction(ISD::FFLOOR,            VT, Legal);
1373       setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
1374       setOperationAction(ISD::FCEIL,             VT, Legal);
1375       setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
1376       setOperationAction(ISD::FTRUNC,            VT, Legal);
1377       setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
1378       setOperationAction(ISD::FRINT,             VT, Legal);
1379       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
1380       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
1381       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1382       setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
1383       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1384 
1385       setOperationAction(ISD::FROUND,            VT, Custom);
1386 
1387       setOperationAction(ISD::FNEG,              VT, Custom);
1388       setOperationAction(ISD::FABS,              VT, Custom);
1389       setOperationAction(ISD::FCOPYSIGN,         VT, Custom);
1390 
1391       setOperationAction(ISD::FMAXIMUM,          VT, Custom);
1392       setOperationAction(ISD::FMINIMUM,          VT, Custom);
1393     }
1394 
1395     // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1396     // even though v8i16 is a legal type.
1397     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i16, MVT::v8i32);
1398     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i16, MVT::v8i32);
1399     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1400     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1401     setOperationAction(ISD::FP_TO_SINT,                MVT::v8i32, Custom);
1402     setOperationAction(ISD::FP_TO_UINT,                MVT::v8i32, Custom);
1403     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v8i32, Custom);
1404 
1405     setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Custom);
1406     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i32, Custom);
1407     setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Expand);
1408     setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Expand);
1409     setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
1410     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Custom);
1411 
1412     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v4f32, Legal);
1413     setOperationAction(ISD::STRICT_FADD,        MVT::v8f32, Legal);
1414     setOperationAction(ISD::STRICT_FADD,        MVT::v4f64, Legal);
1415     setOperationAction(ISD::STRICT_FSUB,        MVT::v8f32, Legal);
1416     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f64, Legal);
1417     setOperationAction(ISD::STRICT_FMUL,        MVT::v8f32, Legal);
1418     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f64, Legal);
1419     setOperationAction(ISD::STRICT_FDIV,        MVT::v8f32, Legal);
1420     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f64, Legal);
1421     setOperationAction(ISD::STRICT_FSQRT,       MVT::v8f32, Legal);
1422     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f64, Legal);
1423 
1424     if (!Subtarget.hasAVX512())
1425       setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1426 
1427     // In the customized shift lowering, the legal v8i32/v4i64 cases
1428     // in AVX2 will be recognized.
1429     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1430       setOperationAction(ISD::SRL,             VT, Custom);
1431       setOperationAction(ISD::SHL,             VT, Custom);
1432       setOperationAction(ISD::SRA,             VT, Custom);
1433       setOperationAction(ISD::ABDS,            VT, Custom);
1434       setOperationAction(ISD::ABDU,            VT, Custom);
1435       if (VT == MVT::v4i64) continue;
1436       setOperationAction(ISD::ROTL,            VT, Custom);
1437       setOperationAction(ISD::ROTR,            VT, Custom);
1438       setOperationAction(ISD::FSHL,            VT, Custom);
1439       setOperationAction(ISD::FSHR,            VT, Custom);
1440     }
1441 
1442     // These types need custom splitting if their input is a 128-bit vector.
1443     setOperationAction(ISD::SIGN_EXTEND,       MVT::v8i64,  Custom);
1444     setOperationAction(ISD::SIGN_EXTEND,       MVT::v16i32, Custom);
1445     setOperationAction(ISD::ZERO_EXTEND,       MVT::v8i64,  Custom);
1446     setOperationAction(ISD::ZERO_EXTEND,       MVT::v16i32, Custom);
1447 
1448     setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
1449     setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);
1450     setOperationAction(ISD::SELECT,            MVT::v8i32, Custom);
1451     setOperationAction(ISD::SELECT,            MVT::v16i16, Custom);
1452     setOperationAction(ISD::SELECT,            MVT::v16f16, Custom);
1453     setOperationAction(ISD::SELECT,            MVT::v32i8, Custom);
1454     setOperationAction(ISD::SELECT,            MVT::v8f32, Custom);
1455 
1456     for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1457       setOperationAction(ISD::SIGN_EXTEND,     VT, Custom);
1458       setOperationAction(ISD::ZERO_EXTEND,     VT, Custom);
1459       setOperationAction(ISD::ANY_EXTEND,      VT, Custom);
1460     }
1461 
1462     setOperationAction(ISD::TRUNCATE,          MVT::v32i8, Custom);
1463     setOperationAction(ISD::TRUNCATE,          MVT::v32i16, Custom);
1464     setOperationAction(ISD::TRUNCATE,          MVT::v32i32, Custom);
1465     setOperationAction(ISD::TRUNCATE,          MVT::v32i64, Custom);
1466 
1467     setOperationAction(ISD::BITREVERSE,        MVT::v32i8, Custom);
1468 
1469     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1470       setOperationAction(ISD::SETCC,           VT, Custom);
1471       setOperationAction(ISD::CTPOP,           VT, Custom);
1472       setOperationAction(ISD::CTLZ,            VT, Custom);
1473 
1474       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1475       // setcc all the way to isel and prefer SETGT in some isel patterns.
1476       setCondCodeAction(ISD::SETLT, VT, Custom);
1477       setCondCodeAction(ISD::SETLE, VT, Custom);
1478     }
1479 
1480     setOperationAction(ISD::SETCC,          MVT::v4f64, Custom);
1481     setOperationAction(ISD::SETCC,          MVT::v8f32, Custom);
1482     setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f64, Custom);
1483     setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f32, Custom);
1484     setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
1485     setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
1486 
1487     if (Subtarget.hasAnyFMA()) {
1488       for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1489                        MVT::v2f64, MVT::v4f64 }) {
1490         setOperationAction(ISD::FMA, VT, Legal);
1491         setOperationAction(ISD::STRICT_FMA, VT, Legal);
1492       }
1493     }
1494 
1495     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1496       setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1497       setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1498     }
1499 
1500     setOperationAction(ISD::MUL,       MVT::v4i64,  Custom);
1501     setOperationAction(ISD::MUL,       MVT::v8i32,  HasInt256 ? Legal : Custom);
1502     setOperationAction(ISD::MUL,       MVT::v16i16, HasInt256 ? Legal : Custom);
1503     setOperationAction(ISD::MUL,       MVT::v32i8,  Custom);
1504 
1505     setOperationAction(ISD::MULHU,     MVT::v8i32,  Custom);
1506     setOperationAction(ISD::MULHS,     MVT::v8i32,  Custom);
1507     setOperationAction(ISD::MULHU,     MVT::v16i16, HasInt256 ? Legal : Custom);
1508     setOperationAction(ISD::MULHS,     MVT::v16i16, HasInt256 ? Legal : Custom);
1509     setOperationAction(ISD::MULHU,     MVT::v32i8,  Custom);
1510     setOperationAction(ISD::MULHS,     MVT::v32i8,  Custom);
1511     setOperationAction(ISD::AVGCEILU,  MVT::v16i16, HasInt256 ? Legal : Custom);
1512     setOperationAction(ISD::AVGCEILU,  MVT::v32i8,  HasInt256 ? Legal : Custom);
1513 
1514     setOperationAction(ISD::SMULO,     MVT::v32i8, Custom);
1515     setOperationAction(ISD::UMULO,     MVT::v32i8, Custom);
1516 
1517     setOperationAction(ISD::ABS,       MVT::v4i64,  Custom);
1518     setOperationAction(ISD::SMAX,      MVT::v4i64,  Custom);
1519     setOperationAction(ISD::UMAX,      MVT::v4i64,  Custom);
1520     setOperationAction(ISD::SMIN,      MVT::v4i64,  Custom);
1521     setOperationAction(ISD::UMIN,      MVT::v4i64,  Custom);
1522 
1523     setOperationAction(ISD::UADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1524     setOperationAction(ISD::SADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1525     setOperationAction(ISD::USUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1526     setOperationAction(ISD::SSUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1527     setOperationAction(ISD::UADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1528     setOperationAction(ISD::SADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1529     setOperationAction(ISD::USUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1530     setOperationAction(ISD::SSUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1531     setOperationAction(ISD::UADDSAT,   MVT::v8i32, Custom);
1532     setOperationAction(ISD::USUBSAT,   MVT::v8i32, Custom);
1533     setOperationAction(ISD::UADDSAT,   MVT::v4i64, Custom);
1534     setOperationAction(ISD::USUBSAT,   MVT::v4i64, Custom);
1535 
1536     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1537       setOperationAction(ISD::ABS,  VT, HasInt256 ? Legal : Custom);
1538       setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1539       setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1540       setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1541       setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1542     }
1543 
1544     for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1545       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1546       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1547     }
1548 
1549     if (HasInt256) {
1550       // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1551       // when we have a 256bit-wide blend with immediate.
1552       setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1553       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1554 
1555       // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1556       for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1557         setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1558         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i8,  Legal);
1559         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i8,  Legal);
1560         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i16, Legal);
1561         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i16, Legal);
1562         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i32, Legal);
1563       }
1564     }
1565 
1566     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1567                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1568       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
1569       setOperationAction(ISD::MSTORE, VT, Legal);
1570     }
1571 
1572     // Extract subvector is special because the value type
1573     // (result) is 128-bit but the source is 256-bit wide.
1574     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1575                      MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1576       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1577     }
1578 
1579     // Custom lower several nodes for 256-bit types.
1580     for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1581                     MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
1582       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1583       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1584       setOperationAction(ISD::VSELECT,            VT, Custom);
1585       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1586       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1587       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1588       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
1589       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
1590       setOperationAction(ISD::STORE,              VT, Custom);
1591     }
1592     setF16Action(MVT::v16f16, Expand);
1593     setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
1594     setOperationAction(ISD::FABS, MVT::v16f16, Custom);
1595     setOperationAction(ISD::FCOPYSIGN, MVT::v16f16, Custom);
1596     setOperationAction(ISD::FADD, MVT::v16f16, Expand);
1597     setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
1598     setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
1599     setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
1600 
1601     if (HasInt256) {
1602       setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1603 
1604       // Custom legalize 2x32 to get a little better code.
1605       setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1606       setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1607 
1608       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1609                        MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1610         setOperationAction(ISD::MGATHER,  VT, Custom);
1611     }
1612   }
1613 
1614   if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1615       Subtarget.hasF16C()) {
1616     for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
1617       setOperationAction(ISD::FP_ROUND,           VT, Custom);
1618       setOperationAction(ISD::STRICT_FP_ROUND,    VT, Custom);
1619     }
1620     for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
1621       setOperationAction(ISD::FP_EXTEND,          VT, Custom);
1622       setOperationAction(ISD::STRICT_FP_EXTEND,   VT, Custom);
1623     }
1624     for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1625       setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
1626       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1627     }
1628   }
1629 
1630   // This block controls legalization of the mask vector sizes that are
1631   // available with AVX512. 512-bit vectors are in a separate block controlled
1632   // by useAVX512Regs.
1633   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1634     addRegisterClass(MVT::v1i1,   &X86::VK1RegClass);
1635     addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);
1636     addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);
1637     addRegisterClass(MVT::v8i1,   &X86::VK8RegClass);
1638     addRegisterClass(MVT::v16i1,  &X86::VK16RegClass);
1639 
1640     setOperationAction(ISD::SELECT,             MVT::v1i1, Custom);
1641     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1642     setOperationAction(ISD::BUILD_VECTOR,       MVT::v1i1, Custom);
1643 
1644     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i1,  MVT::v8i32);
1645     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i1,  MVT::v8i32);
1646     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v4i1,  MVT::v4i32);
1647     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v4i1,  MVT::v4i32);
1648     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1,  MVT::v8i32);
1649     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1,  MVT::v8i32);
1650     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1,  MVT::v4i32);
1651     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1,  MVT::v4i32);
1652     setOperationAction(ISD::FP_TO_SINT,                MVT::v2i1,  Custom);
1653     setOperationAction(ISD::FP_TO_UINT,                MVT::v2i1,  Custom);
1654     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v2i1,  Custom);
1655     setOperationAction(ISD::STRICT_FP_TO_UINT,         MVT::v2i1,  Custom);
1656 
1657     // There is no byte sized k-register load or store without AVX512DQ.
1658     if (!Subtarget.hasDQI()) {
1659       setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1660       setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1661       setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1662       setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1663 
1664       setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1665       setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1666       setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1667       setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1668     }
1669 
1670     // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1671     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1672       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1673       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1674       setOperationAction(ISD::ANY_EXTEND,  VT, Custom);
1675     }
1676 
1677     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
1678       setOperationAction(ISD::VSELECT,          VT, Expand);
1679 
1680     for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1681       setOperationAction(ISD::SETCC,            VT, Custom);
1682       setOperationAction(ISD::SELECT,           VT, Custom);
1683       setOperationAction(ISD::TRUNCATE,         VT, Custom);
1684 
1685       setOperationAction(ISD::BUILD_VECTOR,     VT, Custom);
1686       setOperationAction(ISD::CONCAT_VECTORS,   VT, Custom);
1687       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1688       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1689       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1690       setOperationAction(ISD::VECTOR_SHUFFLE,   VT,  Custom);
1691     }
1692 
1693     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1694       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1695   }
1696 
1697   // This block controls legalization for 512-bit operations with 8/16/32/64 bit
1698   // elements. 512-bits can be disabled based on prefer-vector-width and
1699   // required-vector-width function attributes.
1700   if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1701     bool HasBWI = Subtarget.hasBWI();
1702 
1703     addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1704     addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1705     addRegisterClass(MVT::v8i64,  &X86::VR512RegClass);
1706     addRegisterClass(MVT::v8f64,  &X86::VR512RegClass);
1707     addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1708     addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1709     addRegisterClass(MVT::v64i8,  &X86::VR512RegClass);
1710 
1711     for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1712       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8,  Legal);
1713       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1714       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i8,   Legal);
1715       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i16,  Legal);
1716       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i32,  Legal);
1717       if (HasBWI)
1718         setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1719     }
1720 
1721     for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1722       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1723       setOperationAction(ISD::FMINIMUM, VT, Custom);
1724       setOperationAction(ISD::FNEG,  VT, Custom);
1725       setOperationAction(ISD::FABS,  VT, Custom);
1726       setOperationAction(ISD::FMA,   VT, Legal);
1727       setOperationAction(ISD::STRICT_FMA, VT, Legal);
1728       setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1729     }
1730 
1731     for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
1732       setOperationPromotedToType(ISD::FP_TO_SINT       , VT, MVT::v16i32);
1733       setOperationPromotedToType(ISD::FP_TO_UINT       , VT, MVT::v16i32);
1734       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1735       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1736     }
1737 
1738     for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
1739       setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
1740       setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
1741       setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1742       setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1743     }
1744 
1745     setOperationAction(ISD::SINT_TO_FP,        MVT::v16i32, Custom);
1746     setOperationAction(ISD::UINT_TO_FP,        MVT::v16i32, Custom);
1747     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
1748     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
1749     setOperationAction(ISD::FP_EXTEND,         MVT::v8f64,  Custom);
1750     setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v8f64,  Custom);
1751 
1752     setOperationAction(ISD::STRICT_FADD,      MVT::v16f32, Legal);
1753     setOperationAction(ISD::STRICT_FADD,      MVT::v8f64,  Legal);
1754     setOperationAction(ISD::STRICT_FSUB,      MVT::v16f32, Legal);
1755     setOperationAction(ISD::STRICT_FSUB,      MVT::v8f64,  Legal);
1756     setOperationAction(ISD::STRICT_FMUL,      MVT::v16f32, Legal);
1757     setOperationAction(ISD::STRICT_FMUL,      MVT::v8f64,  Legal);
1758     setOperationAction(ISD::STRICT_FDIV,      MVT::v16f32, Legal);
1759     setOperationAction(ISD::STRICT_FDIV,      MVT::v8f64,  Legal);
1760     setOperationAction(ISD::STRICT_FSQRT,     MVT::v16f32, Legal);
1761     setOperationAction(ISD::STRICT_FSQRT,     MVT::v8f64,  Legal);
1762     setOperationAction(ISD::STRICT_FP_ROUND,  MVT::v8f32,  Legal);
1763 
1764     setTruncStoreAction(MVT::v8i64,   MVT::v8i8,   Legal);
1765     setTruncStoreAction(MVT::v8i64,   MVT::v8i16,  Legal);
1766     setTruncStoreAction(MVT::v8i64,   MVT::v8i32,  Legal);
1767     setTruncStoreAction(MVT::v16i32,  MVT::v16i8,  Legal);
1768     setTruncStoreAction(MVT::v16i32,  MVT::v16i16, Legal);
1769     if (HasBWI)
1770       setTruncStoreAction(MVT::v32i16,  MVT::v32i8, Legal);
1771 
1772     // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1773     // to 512-bit rather than use the AVX2 instructions so that we can use
1774     // k-masks.
1775     if (!Subtarget.hasVLX()) {
1776       for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1777            MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1778         setOperationAction(ISD::MLOAD,  VT, Custom);
1779         setOperationAction(ISD::MSTORE, VT, Custom);
1780       }
1781     }
1782 
1783     setOperationAction(ISD::TRUNCATE,    MVT::v8i32,  Legal);
1784     setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Legal);
1785     setOperationAction(ISD::TRUNCATE,    MVT::v32i8,  HasBWI ? Legal : Custom);
1786     setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1787     setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1788     setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64,  Custom);
1789     setOperationAction(ISD::ANY_EXTEND,  MVT::v32i16, Custom);
1790     setOperationAction(ISD::ANY_EXTEND,  MVT::v16i32, Custom);
1791     setOperationAction(ISD::ANY_EXTEND,  MVT::v8i64,  Custom);
1792     setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1793     setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1794     setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64,  Custom);
1795 
1796     if (HasBWI) {
1797       // Extends from v64i1 masks to 512-bit vectors.
1798       setOperationAction(ISD::SIGN_EXTEND,        MVT::v64i8, Custom);
1799       setOperationAction(ISD::ZERO_EXTEND,        MVT::v64i8, Custom);
1800       setOperationAction(ISD::ANY_EXTEND,         MVT::v64i8, Custom);
1801     }
1802 
1803     for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1804       setOperationAction(ISD::FFLOOR,            VT, Legal);
1805       setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
1806       setOperationAction(ISD::FCEIL,             VT, Legal);
1807       setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
1808       setOperationAction(ISD::FTRUNC,            VT, Legal);
1809       setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
1810       setOperationAction(ISD::FRINT,             VT, Legal);
1811       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
1812       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
1813       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1814       setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
1815       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1816 
1817       setOperationAction(ISD::FROUND,            VT, Custom);
1818     }
1819 
1820     for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1821       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1822       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1823     }
1824 
1825     setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
1826     setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
1827     setOperationAction(ISD::ADD, MVT::v64i8,  HasBWI ? Legal : Custom);
1828     setOperationAction(ISD::SUB, MVT::v64i8,  HasBWI ? Legal : Custom);
1829 
1830     setOperationAction(ISD::MUL, MVT::v8i64,  Custom);
1831     setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1832     setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
1833     setOperationAction(ISD::MUL, MVT::v64i8,  Custom);
1834 
1835     setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1836     setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1837     setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
1838     setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
1839     setOperationAction(ISD::MULHS, MVT::v64i8,  Custom);
1840     setOperationAction(ISD::MULHU, MVT::v64i8,  Custom);
1841     setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
1842     setOperationAction(ISD::AVGCEILU, MVT::v64i8,  HasBWI ? Legal : Custom);
1843 
1844     setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
1845     setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
1846 
1847     setOperationAction(ISD::BITREVERSE, MVT::v64i8,  Custom);
1848 
1849     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1850       setOperationAction(ISD::SRL,              VT, Custom);
1851       setOperationAction(ISD::SHL,              VT, Custom);
1852       setOperationAction(ISD::SRA,              VT, Custom);
1853       setOperationAction(ISD::ROTL,             VT, Custom);
1854       setOperationAction(ISD::ROTR,             VT, Custom);
1855       setOperationAction(ISD::SETCC,            VT, Custom);
1856       setOperationAction(ISD::ABDS,             VT, Custom);
1857       setOperationAction(ISD::ABDU,             VT, Custom);
1858 
1859       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1860       // setcc all the way to isel and prefer SETGT in some isel patterns.
1861       setCondCodeAction(ISD::SETLT, VT, Custom);
1862       setCondCodeAction(ISD::SETLE, VT, Custom);
1863     }
1864 
1865     setOperationAction(ISD::SETCC,          MVT::v8f64, Custom);
1866     setOperationAction(ISD::SETCC,          MVT::v16f32, Custom);
1867     setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f64, Custom);
1868     setOperationAction(ISD::STRICT_FSETCC,  MVT::v16f32, Custom);
1869     setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
1870     setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
1871 
1872     for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1873       setOperationAction(ISD::SMAX,             VT, Legal);
1874       setOperationAction(ISD::UMAX,             VT, Legal);
1875       setOperationAction(ISD::SMIN,             VT, Legal);
1876       setOperationAction(ISD::UMIN,             VT, Legal);
1877       setOperationAction(ISD::ABS,              VT, Legal);
1878       setOperationAction(ISD::CTPOP,            VT, Custom);
1879     }
1880 
1881     for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1882       setOperationAction(ISD::ABS,     VT, HasBWI ? Legal : Custom);
1883       setOperationAction(ISD::CTPOP,   VT, Subtarget.hasBITALG() ? Legal : Custom);
1884       setOperationAction(ISD::CTLZ,    VT, Custom);
1885       setOperationAction(ISD::SMAX,    VT, HasBWI ? Legal : Custom);
1886       setOperationAction(ISD::UMAX,    VT, HasBWI ? Legal : Custom);
1887       setOperationAction(ISD::SMIN,    VT, HasBWI ? Legal : Custom);
1888       setOperationAction(ISD::UMIN,    VT, HasBWI ? Legal : Custom);
1889       setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
1890       setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
1891       setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
1892       setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
1893     }
1894 
1895     setOperationAction(ISD::FSHL,       MVT::v64i8, Custom);
1896     setOperationAction(ISD::FSHR,       MVT::v64i8, Custom);
1897     setOperationAction(ISD::FSHL,      MVT::v32i16, Custom);
1898     setOperationAction(ISD::FSHR,      MVT::v32i16, Custom);
1899     setOperationAction(ISD::FSHL,      MVT::v16i32, Custom);
1900     setOperationAction(ISD::FSHR,      MVT::v16i32, Custom);
1901 
1902     if (Subtarget.hasDQI()) {
1903       for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1904                        ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1905                        ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1906         setOperationAction(Opc,           MVT::v8i64, Custom);
1907       setOperationAction(ISD::MUL,        MVT::v8i64, Legal);
1908     }
1909 
1910     if (Subtarget.hasCDI()) {
1911       // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1912       for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1913         setOperationAction(ISD::CTLZ,            VT, Legal);
1914       }
1915     } // Subtarget.hasCDI()
1916 
1917     if (Subtarget.hasVPOPCNTDQ()) {
1918       for (auto VT : { MVT::v16i32, MVT::v8i64 })
1919         setOperationAction(ISD::CTPOP, VT, Legal);
1920     }
1921 
1922     // Extract subvector is special because the value type
1923     // (result) is 256-bit but the source is 512-bit wide.
1924     // 128-bit was made Legal under AVX1.
1925     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1926                      MVT::v16f16, MVT::v8f32, MVT::v4f64 })
1927       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1928 
1929     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
1930                      MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
1931       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
1932       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
1933       setOperationAction(ISD::SELECT,             VT, Custom);
1934       setOperationAction(ISD::VSELECT,            VT, Custom);
1935       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1936       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1937       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1938       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1939       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1940     }
1941     setF16Action(MVT::v32f16, Expand);
1942     setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
1943     setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
1944     setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
1945     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
1946     for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1947       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1948       setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
1949     }
1950 
1951     for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
1952       setOperationAction(ISD::MLOAD,               VT, Legal);
1953       setOperationAction(ISD::MSTORE,              VT, Legal);
1954       setOperationAction(ISD::MGATHER,             VT, Custom);
1955       setOperationAction(ISD::MSCATTER,            VT, Custom);
1956     }
1957     if (HasBWI) {
1958       for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1959         setOperationAction(ISD::MLOAD,        VT, Legal);
1960         setOperationAction(ISD::MSTORE,       VT, Legal);
1961       }
1962     } else {
1963       setOperationAction(ISD::STORE, MVT::v32i16, Custom);
1964       setOperationAction(ISD::STORE, MVT::v64i8,  Custom);
1965     }
1966 
1967     if (Subtarget.hasVBMI2()) {
1968       for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1969         setOperationAction(ISD::FSHL, VT, Custom);
1970         setOperationAction(ISD::FSHR, VT, Custom);
1971       }
1972 
1973       setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
1974       setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
1975     }
1976   }// useAVX512Regs
1977 
1978   if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
1979     for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
1980                     MVT::v4i64}) {
1981       setOperationAction(ISD::FSHL, VT, Custom);
1982       setOperationAction(ISD::FSHR, VT, Custom);
1983     }
1984   }
1985 
1986   // This block controls legalization for operations that don't have
1987   // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
1988   // narrower widths.
1989   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1990     // These operations are handled on non-VLX by artificially widening in
1991     // isel patterns.
1992 
1993     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i32, Custom);
1994     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v4i32, Custom);
1995     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v2i32, Custom);
1996 
1997     if (Subtarget.hasDQI()) {
1998       // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
1999       // v2f32 UINT_TO_FP is already custom under SSE2.
2000       assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
2001              isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
2002              "Unexpected operation action!");
2003       // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
2004       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f32, Custom);
2005       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f32, Custom);
2006       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
2007       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
2008     }
2009 
2010     for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
2011       setOperationAction(ISD::SMAX, VT, Legal);
2012       setOperationAction(ISD::UMAX, VT, Legal);
2013       setOperationAction(ISD::SMIN, VT, Legal);
2014       setOperationAction(ISD::UMIN, VT, Legal);
2015       setOperationAction(ISD::ABS,  VT, Legal);
2016     }
2017 
2018     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2019       setOperationAction(ISD::ROTL,     VT, Custom);
2020       setOperationAction(ISD::ROTR,     VT, Custom);
2021     }
2022 
2023     // Custom legalize 2x32 to get a little better code.
2024     setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
2025     setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
2026 
2027     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
2028                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
2029       setOperationAction(ISD::MSCATTER, VT, Custom);
2030 
2031     if (Subtarget.hasDQI()) {
2032       for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
2033                        ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2034                        ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
2035         setOperationAction(Opc, MVT::v2i64, Custom);
2036         setOperationAction(Opc, MVT::v4i64, Custom);
2037       }
2038       setOperationAction(ISD::MUL, MVT::v2i64, Legal);
2039       setOperationAction(ISD::MUL, MVT::v4i64, Legal);
2040     }
2041 
2042     if (Subtarget.hasCDI()) {
2043       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2044         setOperationAction(ISD::CTLZ,            VT, Legal);
2045       }
2046     } // Subtarget.hasCDI()
2047 
2048     if (Subtarget.hasVPOPCNTDQ()) {
2049       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
2050         setOperationAction(ISD::CTPOP, VT, Legal);
2051     }
2052     setOperationAction(ISD::FNEG, MVT::v32f16, Custom);
2053     setOperationAction(ISD::FABS, MVT::v32f16, Custom);
2054     setOperationAction(ISD::FCOPYSIGN, MVT::v32f16, Custom);
2055   }
2056 
2057   // This block control legalization of v32i1/v64i1 which are available with
2058   // AVX512BW..
2059   if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2060     addRegisterClass(MVT::v32i1,  &X86::VK32RegClass);
2061     addRegisterClass(MVT::v64i1,  &X86::VK64RegClass);
2062 
2063     for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
2064       setOperationAction(ISD::VSELECT,            VT, Expand);
2065       setOperationAction(ISD::TRUNCATE,           VT, Custom);
2066       setOperationAction(ISD::SETCC,              VT, Custom);
2067       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2068       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
2069       setOperationAction(ISD::SELECT,             VT, Custom);
2070       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
2071       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
2072       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
2073       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Custom);
2074     }
2075 
2076     for (auto VT : { MVT::v16i1, MVT::v32i1 })
2077       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
2078 
2079     // Extends from v32i1 masks to 256-bit vectors.
2080     setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);
2081     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
2082     setOperationAction(ISD::ANY_EXTEND,         MVT::v32i8, Custom);
2083 
2084     for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
2085       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
2086       setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
2087     }
2088 
2089     // These operations are handled on non-VLX by artificially widening in
2090     // isel patterns.
2091     // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
2092 
2093     if (Subtarget.hasBITALG()) {
2094       for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
2095         setOperationAction(ISD::CTPOP, VT, Legal);
2096     }
2097   }
2098 
2099   if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2100     auto setGroup = [&] (MVT VT) {
2101       setOperationAction(ISD::FADD,               VT, Legal);
2102       setOperationAction(ISD::STRICT_FADD,        VT, Legal);
2103       setOperationAction(ISD::FSUB,               VT, Legal);
2104       setOperationAction(ISD::STRICT_FSUB,        VT, Legal);
2105       setOperationAction(ISD::FMUL,               VT, Legal);
2106       setOperationAction(ISD::STRICT_FMUL,        VT, Legal);
2107       setOperationAction(ISD::FDIV,               VT, Legal);
2108       setOperationAction(ISD::STRICT_FDIV,        VT, Legal);
2109       setOperationAction(ISD::FSQRT,              VT, Legal);
2110       setOperationAction(ISD::STRICT_FSQRT,       VT, Legal);
2111 
2112       setOperationAction(ISD::FFLOOR,             VT, Legal);
2113       setOperationAction(ISD::STRICT_FFLOOR,      VT, Legal);
2114       setOperationAction(ISD::FCEIL,              VT, Legal);
2115       setOperationAction(ISD::STRICT_FCEIL,       VT, Legal);
2116       setOperationAction(ISD::FTRUNC,             VT, Legal);
2117       setOperationAction(ISD::STRICT_FTRUNC,      VT, Legal);
2118       setOperationAction(ISD::FRINT,              VT, Legal);
2119       setOperationAction(ISD::STRICT_FRINT,       VT, Legal);
2120       setOperationAction(ISD::FNEARBYINT,         VT, Legal);
2121       setOperationAction(ISD::STRICT_FNEARBYINT,  VT, Legal);
2122       setOperationAction(ISD::FROUNDEVEN, VT, Legal);
2123       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
2124 
2125       setOperationAction(ISD::FROUND,             VT, Custom);
2126 
2127       setOperationAction(ISD::LOAD,               VT, Legal);
2128       setOperationAction(ISD::STORE,              VT, Legal);
2129 
2130       setOperationAction(ISD::FMA,                VT, Legal);
2131       setOperationAction(ISD::STRICT_FMA,         VT, Legal);
2132       setOperationAction(ISD::VSELECT,            VT, Legal);
2133       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
2134       setOperationAction(ISD::SELECT,             VT, Custom);
2135 
2136       setOperationAction(ISD::FNEG,               VT, Custom);
2137       setOperationAction(ISD::FABS,               VT, Custom);
2138       setOperationAction(ISD::FCOPYSIGN,          VT, Custom);
2139       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2140       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
2141 
2142       setOperationAction(ISD::SETCC,              VT, Custom);
2143       setOperationAction(ISD::STRICT_FSETCC,      VT, Custom);
2144       setOperationAction(ISD::STRICT_FSETCCS,     VT, Custom);
2145     };
2146 
2147     // AVX512_FP16 scalar operations
2148     setGroup(MVT::f16);
2149     setOperationAction(ISD::FREM,                 MVT::f16, Promote);
2150     setOperationAction(ISD::STRICT_FREM,          MVT::f16, Promote);
2151     setOperationAction(ISD::SELECT_CC,            MVT::f16, Expand);
2152     setOperationAction(ISD::BR_CC,                MVT::f16, Expand);
2153     setOperationAction(ISD::STRICT_FROUND,        MVT::f16, Promote);
2154     setOperationAction(ISD::FROUNDEVEN,           MVT::f16, Legal);
2155     setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);
2156     setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);
2157     setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);
2158     setOperationAction(ISD::FMAXIMUM,             MVT::f16, Custom);
2159     setOperationAction(ISD::FMINIMUM,             MVT::f16, Custom);
2160     setOperationAction(ISD::FP_EXTEND,            MVT::f32, Legal);
2161     setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);
2162 
2163     setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
2164     setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
2165 
2166     if (Subtarget.useAVX512Regs()) {
2167       setGroup(MVT::v32f16);
2168       setOperationAction(ISD::SCALAR_TO_VECTOR,       MVT::v32f16, Custom);
2169       setOperationAction(ISD::SINT_TO_FP,             MVT::v32i16, Legal);
2170       setOperationAction(ISD::STRICT_SINT_TO_FP,      MVT::v32i16, Legal);
2171       setOperationAction(ISD::UINT_TO_FP,             MVT::v32i16, Legal);
2172       setOperationAction(ISD::STRICT_UINT_TO_FP,      MVT::v32i16, Legal);
2173       setOperationAction(ISD::FP_ROUND,               MVT::v16f16, Legal);
2174       setOperationAction(ISD::STRICT_FP_ROUND,        MVT::v16f16, Legal);
2175       setOperationAction(ISD::FP_EXTEND,              MVT::v16f32, Custom);
2176       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v16f32, Legal);
2177       setOperationAction(ISD::FP_EXTEND,              MVT::v8f64,  Custom);
2178       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v8f64,  Legal);
2179       setOperationAction(ISD::INSERT_VECTOR_ELT,      MVT::v32f16, Custom);
2180 
2181       setOperationAction(ISD::FP_TO_SINT,             MVT::v32i16, Custom);
2182       setOperationAction(ISD::STRICT_FP_TO_SINT,      MVT::v32i16, Custom);
2183       setOperationAction(ISD::FP_TO_UINT,             MVT::v32i16, Custom);
2184       setOperationAction(ISD::STRICT_FP_TO_UINT,      MVT::v32i16, Custom);
2185       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i8,  MVT::v32i16);
2186       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
2187                                  MVT::v32i16);
2188       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i8,  MVT::v32i16);
2189       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
2190                                  MVT::v32i16);
2191       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i1,  MVT::v32i16);
2192       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
2193                                  MVT::v32i16);
2194       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i1,  MVT::v32i16);
2195       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
2196                                  MVT::v32i16);
2197 
2198       setOperationAction(ISD::EXTRACT_SUBVECTOR,      MVT::v16f16, Legal);
2199       setOperationAction(ISD::INSERT_SUBVECTOR,       MVT::v32f16, Legal);
2200       setOperationAction(ISD::CONCAT_VECTORS,         MVT::v32f16, Custom);
2201 
2202       setLoadExtAction(ISD::EXTLOAD, MVT::v8f64,  MVT::v8f16,  Legal);
2203       setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
2204     }
2205 
2206     if (Subtarget.hasVLX()) {
2207       setGroup(MVT::v8f16);
2208       setGroup(MVT::v16f16);
2209 
2210       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8f16,  Legal);
2211       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16f16, Custom);
2212       setOperationAction(ISD::SINT_TO_FP,         MVT::v16i16, Legal);
2213       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v16i16, Legal);
2214       setOperationAction(ISD::SINT_TO_FP,         MVT::v8i16,  Legal);
2215       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i16,  Legal);
2216       setOperationAction(ISD::UINT_TO_FP,         MVT::v16i16, Legal);
2217       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v16i16, Legal);
2218       setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16,  Legal);
2219       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v8i16,  Legal);
2220 
2221       setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
2222       setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);
2223       setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);
2224       setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);
2225       setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Legal);
2226       setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v8f16, Legal);
2227       setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Custom);
2228       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v8f32, Legal);
2229       setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
2230       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Legal);
2231 
2232       // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
2233       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v8f16,  Custom);
2234       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v16f16, Custom);
2235 
2236       setOperationAction(ISD::EXTRACT_SUBVECTOR,    MVT::v8f16, Legal);
2237       setOperationAction(ISD::INSERT_SUBVECTOR,     MVT::v16f16, Legal);
2238       setOperationAction(ISD::CONCAT_VECTORS,       MVT::v16f16, Custom);
2239 
2240       setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
2241       setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
2242       setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
2243       setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
2244 
2245       // Need to custom widen these to prevent scalarization.
2246       setOperationAction(ISD::LOAD,  MVT::v4f16, Custom);
2247       setOperationAction(ISD::STORE, MVT::v4f16, Custom);
2248     }
2249   }
2250 
2251   if (!Subtarget.useSoftFloat() &&
2252       (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
2253     addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2254                                                         : &X86::VR128RegClass);
2255     addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2256                                                          : &X86::VR256RegClass);
2257     // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
2258     // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
2259     // Set the operation action Custom to do the customization later.
2260     setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
2261     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
2262     for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
2263       setF16Action(VT, Expand);
2264       setOperationAction(ISD::FADD, VT, Expand);
2265       setOperationAction(ISD::FSUB, VT, Expand);
2266       setOperationAction(ISD::FMUL, VT, Expand);
2267       setOperationAction(ISD::FDIV, VT, Expand);
2268       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2269       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2270       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2271       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2272     }
2273     setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
2274     addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
2275   }
2276 
2277   if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
2278     addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2279     setF16Action(MVT::v32bf16, Expand);
2280     setOperationAction(ISD::FADD, MVT::v32bf16, Expand);
2281     setOperationAction(ISD::FSUB, MVT::v32bf16, Expand);
2282     setOperationAction(ISD::FMUL, MVT::v32bf16, Expand);
2283     setOperationAction(ISD::FDIV, MVT::v32bf16, Expand);
2284     setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
2285     setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
2286     setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
2287     setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal);
2288     setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom);
2289   }
2290 
2291   if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2292     setTruncStoreAction(MVT::v4i64, MVT::v4i8,  Legal);
2293     setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
2294     setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
2295     setTruncStoreAction(MVT::v8i32, MVT::v8i8,  Legal);
2296     setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
2297 
2298     setTruncStoreAction(MVT::v2i64, MVT::v2i8,  Legal);
2299     setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
2300     setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
2301     setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
2302     setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
2303 
2304     if (Subtarget.hasBWI()) {
2305       setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);
2306       setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
2307     }
2308 
2309     if (Subtarget.hasFP16()) {
2310       // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
2311       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f16, Custom);
2312       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
2313       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f16, Custom);
2314       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
2315       setOperationAction(ISD::FP_TO_SINT,        MVT::v4f16, Custom);
2316       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
2317       setOperationAction(ISD::FP_TO_UINT,        MVT::v4f16, Custom);
2318       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
2319       // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
2320       setOperationAction(ISD::SINT_TO_FP,        MVT::v2f16, Custom);
2321       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
2322       setOperationAction(ISD::UINT_TO_FP,        MVT::v2f16, Custom);
2323       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
2324       setOperationAction(ISD::SINT_TO_FP,        MVT::v4f16, Custom);
2325       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
2326       setOperationAction(ISD::UINT_TO_FP,        MVT::v4f16, Custom);
2327       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
2328       // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
2329       setOperationAction(ISD::FP_ROUND,          MVT::v2f16, Custom);
2330       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v2f16, Custom);
2331       setOperationAction(ISD::FP_ROUND,          MVT::v4f16, Custom);
2332       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v4f16, Custom);
2333       // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
2334       setOperationAction(ISD::FP_EXTEND,         MVT::v2f16, Custom);
2335       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v2f16, Custom);
2336       setOperationAction(ISD::FP_EXTEND,         MVT::v4f16, Custom);
2337       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v4f16, Custom);
2338     }
2339   }
2340 
2341   if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
2342     addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2343   }
2344 
2345   // We want to custom lower some of our intrinsics.
2346   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
2347   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
2348   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
2349   if (!Subtarget.is64Bit()) {
2350     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
2351   }
2352 
2353   // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
2354   // handle type legalization for these operations here.
2355   //
2356   // FIXME: We really should do custom legalization for addition and
2357   // subtraction on x86-32 once PR3203 is fixed.  We really can't do much better
2358   // than generic legalization for 64-bit multiplication-with-overflow, though.
2359   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
2360     if (VT == MVT::i64 && !Subtarget.is64Bit())
2361       continue;
2362     // Add/Sub/Mul with overflow operations are custom lowered.
2363     setOperationAction(ISD::SADDO, VT, Custom);
2364     setOperationAction(ISD::UADDO, VT, Custom);
2365     setOperationAction(ISD::SSUBO, VT, Custom);
2366     setOperationAction(ISD::USUBO, VT, Custom);
2367     setOperationAction(ISD::SMULO, VT, Custom);
2368     setOperationAction(ISD::UMULO, VT, Custom);
2369 
2370     // Support carry in as value rather than glue.
2371     setOperationAction(ISD::UADDO_CARRY, VT, Custom);
2372     setOperationAction(ISD::USUBO_CARRY, VT, Custom);
2373     setOperationAction(ISD::SETCCCARRY, VT, Custom);
2374     setOperationAction(ISD::SADDO_CARRY, VT, Custom);
2375     setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
2376   }
2377 
2378   if (!Subtarget.is64Bit()) {
2379     // These libcalls are not available in 32-bit.
2380     setLibcallName(RTLIB::SHL_I128, nullptr);
2381     setLibcallName(RTLIB::SRL_I128, nullptr);
2382     setLibcallName(RTLIB::SRA_I128, nullptr);
2383     setLibcallName(RTLIB::MUL_I128, nullptr);
2384     // The MULO libcall is not part of libgcc, only compiler-rt.
2385     setLibcallName(RTLIB::MULO_I64, nullptr);
2386   }
2387   // The MULO libcall is not part of libgcc, only compiler-rt.
2388   setLibcallName(RTLIB::MULO_I128, nullptr);
2389 
2390   // Combine sin / cos into _sincos_stret if it is available.
2391   if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
2392       getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
2393     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
2394     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
2395   }
2396 
2397   if (Subtarget.isTargetWin64()) {
2398     setOperationAction(ISD::SDIV, MVT::i128, Custom);
2399     setOperationAction(ISD::UDIV, MVT::i128, Custom);
2400     setOperationAction(ISD::SREM, MVT::i128, Custom);
2401     setOperationAction(ISD::UREM, MVT::i128, Custom);
2402     setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
2403     setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
2404     setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
2405     setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
2406     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
2407     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
2408     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
2409     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
2410   }
2411 
2412   // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
2413   // is. We should promote the value to 64-bits to solve this.
2414   // This is what the CRT headers do - `fmodf` is an inline header
2415   // function casting to f64 and calling `fmod`.
2416   if (Subtarget.is32Bit() &&
2417       (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
2418     for (ISD::NodeType Op :
2419          {ISD::FCEIL,  ISD::STRICT_FCEIL,
2420           ISD::FCOS,   ISD::STRICT_FCOS,
2421           ISD::FEXP,   ISD::STRICT_FEXP,
2422           ISD::FFLOOR, ISD::STRICT_FFLOOR,
2423           ISD::FREM,   ISD::STRICT_FREM,
2424           ISD::FLOG,   ISD::STRICT_FLOG,
2425           ISD::FLOG10, ISD::STRICT_FLOG10,
2426           ISD::FPOW,   ISD::STRICT_FPOW,
2427           ISD::FSIN,   ISD::STRICT_FSIN})
2428       if (isOperationExpand(Op, MVT::f32))
2429         setOperationAction(Op, MVT::f32, Promote);
2430 
2431   // We have target-specific dag combine patterns for the following nodes:
2432   setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
2433                        ISD::SCALAR_TO_VECTOR,
2434                        ISD::INSERT_VECTOR_ELT,
2435                        ISD::EXTRACT_VECTOR_ELT,
2436                        ISD::CONCAT_VECTORS,
2437                        ISD::INSERT_SUBVECTOR,
2438                        ISD::EXTRACT_SUBVECTOR,
2439                        ISD::BITCAST,
2440                        ISD::VSELECT,
2441                        ISD::SELECT,
2442                        ISD::SHL,
2443                        ISD::SRA,
2444                        ISD::SRL,
2445                        ISD::OR,
2446                        ISD::AND,
2447                        ISD::BITREVERSE,
2448                        ISD::ADD,
2449                        ISD::FADD,
2450                        ISD::FSUB,
2451                        ISD::FNEG,
2452                        ISD::FMA,
2453                        ISD::STRICT_FMA,
2454                        ISD::FMINNUM,
2455                        ISD::FMAXNUM,
2456                        ISD::SUB,
2457                        ISD::LOAD,
2458                        ISD::MLOAD,
2459                        ISD::STORE,
2460                        ISD::MSTORE,
2461                        ISD::TRUNCATE,
2462                        ISD::ZERO_EXTEND,
2463                        ISD::ANY_EXTEND,
2464                        ISD::SIGN_EXTEND,
2465                        ISD::SIGN_EXTEND_INREG,
2466                        ISD::ANY_EXTEND_VECTOR_INREG,
2467                        ISD::SIGN_EXTEND_VECTOR_INREG,
2468                        ISD::ZERO_EXTEND_VECTOR_INREG,
2469                        ISD::SINT_TO_FP,
2470                        ISD::UINT_TO_FP,
2471                        ISD::STRICT_SINT_TO_FP,
2472                        ISD::STRICT_UINT_TO_FP,
2473                        ISD::SETCC,
2474                        ISD::MUL,
2475                        ISD::XOR,
2476                        ISD::MSCATTER,
2477                        ISD::MGATHER,
2478                        ISD::FP16_TO_FP,
2479                        ISD::FP_EXTEND,
2480                        ISD::STRICT_FP_EXTEND,
2481                        ISD::FP_ROUND,
2482                        ISD::STRICT_FP_ROUND});
2483 
2484   computeRegisterProperties(Subtarget.getRegisterInfo());
2485 
2486   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2487   MaxStoresPerMemsetOptSize = 8;
2488   MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2489   MaxStoresPerMemcpyOptSize = 4;
2490   MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2491   MaxStoresPerMemmoveOptSize = 4;
2492 
2493   // TODO: These control memcmp expansion in CGP and could be raised higher, but
2494   // that needs to benchmarked and balanced with the potential use of vector
2495   // load/store types (PR33329, PR33914).
2496   MaxLoadsPerMemcmp = 2;
2497   MaxLoadsPerMemcmpOptSize = 2;
2498 
2499   // Default loop alignment, which can be overridden by -align-loops.
2500   setPrefLoopAlignment(Align(16));
2501 
2502   // An out-of-order CPU can speculatively execute past a predictable branch,
2503   // but a conditional move could be stalled by an expensive earlier operation.
2504   PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2505   EnableExtLdPromotion = true;
2506   setPrefFunctionAlignment(Align(16));
2507 
2508   verifyIntrinsicTables();
2509 
2510   // Default to having -disable-strictnode-mutation on
2511   IsStrictFPEnabled = true;
2512 }
2513 
2514 // This has so far only been implemented for 64-bit MachO.
2515 bool X86TargetLowering::useLoadStackGuardNode() const {
2516   return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2517 }
2518 
2519 bool X86TargetLowering::useStackGuardXorFP() const {
2520   // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2521   return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2522 }
2523 
2524 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2525                                                const SDLoc &DL) const {
2526   EVT PtrTy = getPointerTy(DAG.getDataLayout());
2527   unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2528   MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2529   return SDValue(Node, 0);
2530 }
2531 
2532 TargetLoweringBase::LegalizeTypeAction
2533 X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2534   if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
2535       !Subtarget.hasBWI())
2536     return TypeSplitVector;
2537 
2538   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2539       !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
2540     return TypeSplitVector;
2541 
2542   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2543       VT.getVectorElementType() != MVT::i1)
2544     return TypeWidenVector;
2545 
2546   return TargetLoweringBase::getPreferredVectorAction(VT);
2547 }
2548 
2549 FastISel *
2550 X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2551                                   const TargetLibraryInfo *libInfo) const {
2552   return X86::createFastISel(funcInfo, libInfo);
2553 }
2554 
2555 //===----------------------------------------------------------------------===//
2556 //                           Other Lowering Hooks
2557 //===----------------------------------------------------------------------===//
2558 
2559 bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2560                       bool AssumeSingleUse) {
2561   if (!AssumeSingleUse && !Op.hasOneUse())
2562     return false;
2563   if (!ISD::isNormalLoad(Op.getNode()))
2564     return false;
2565 
2566   // If this is an unaligned vector, make sure the target supports folding it.
2567   auto *Ld = cast<LoadSDNode>(Op.getNode());
2568   if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() &&
2569       Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16))
2570     return false;
2571 
2572   // TODO: If this is a non-temporal load and the target has an instruction
2573   //       for it, it should not be folded. See "useNonTemporalLoad()".
2574 
2575   return true;
2576 }
2577 
2578 bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2579                                           const X86Subtarget &Subtarget,
2580                                           bool AssumeSingleUse) {
2581   assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
2582   if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2583     return false;
2584 
2585   // We can not replace a wide volatile load with a broadcast-from-memory,
2586   // because that would narrow the load, which isn't legal for volatiles.
2587   auto *Ld = cast<LoadSDNode>(Op.getNode());
2588   return !Ld->isVolatile() ||
2589          Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
2590 }
2591 
2592 bool X86::mayFoldIntoStore(SDValue Op) {
2593   return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
2594 }
2595 
2596 bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2597   if (Op.hasOneUse()) {
2598     unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
2599     return (ISD::ZERO_EXTEND == Opcode);
2600   }
2601   return false;
2602 }
2603 
2604 static bool isTargetShuffle(unsigned Opcode) {
2605   switch(Opcode) {
2606   default: return false;
2607   case X86ISD::BLENDI:
2608   case X86ISD::PSHUFB:
2609   case X86ISD::PSHUFD:
2610   case X86ISD::PSHUFHW:
2611   case X86ISD::PSHUFLW:
2612   case X86ISD::SHUFP:
2613   case X86ISD::INSERTPS:
2614   case X86ISD::EXTRQI:
2615   case X86ISD::INSERTQI:
2616   case X86ISD::VALIGN:
2617   case X86ISD::PALIGNR:
2618   case X86ISD::VSHLDQ:
2619   case X86ISD::VSRLDQ:
2620   case X86ISD::MOVLHPS:
2621   case X86ISD::MOVHLPS:
2622   case X86ISD::MOVSHDUP:
2623   case X86ISD::MOVSLDUP:
2624   case X86ISD::MOVDDUP:
2625   case X86ISD::MOVSS:
2626   case X86ISD::MOVSD:
2627   case X86ISD::MOVSH:
2628   case X86ISD::UNPCKL:
2629   case X86ISD::UNPCKH:
2630   case X86ISD::VBROADCAST:
2631   case X86ISD::VPERMILPI:
2632   case X86ISD::VPERMILPV:
2633   case X86ISD::VPERM2X128:
2634   case X86ISD::SHUF128:
2635   case X86ISD::VPERMIL2:
2636   case X86ISD::VPERMI:
2637   case X86ISD::VPPERM:
2638   case X86ISD::VPERMV:
2639   case X86ISD::VPERMV3:
2640   case X86ISD::VZEXT_MOVL:
2641     return true;
2642   }
2643 }
2644 
2645 static bool isTargetShuffleVariableMask(unsigned Opcode) {
2646   switch (Opcode) {
2647   default: return false;
2648   // Target Shuffles.
2649   case X86ISD::PSHUFB:
2650   case X86ISD::VPERMILPV:
2651   case X86ISD::VPERMIL2:
2652   case X86ISD::VPPERM:
2653   case X86ISD::VPERMV:
2654   case X86ISD::VPERMV3:
2655     return true;
2656   // 'Faux' Target Shuffles.
2657   case ISD::OR:
2658   case ISD::AND:
2659   case X86ISD::ANDNP:
2660     return true;
2661   }
2662 }
2663 
2664 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2665   MachineFunction &MF = DAG.getMachineFunction();
2666   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2667   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2668   int ReturnAddrIndex = FuncInfo->getRAIndex();
2669 
2670   if (ReturnAddrIndex == 0) {
2671     // Set up a frame object for the return address.
2672     unsigned SlotSize = RegInfo->getSlotSize();
2673     ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
2674                                                           -(int64_t)SlotSize,
2675                                                           false);
2676     FuncInfo->setRAIndex(ReturnAddrIndex);
2677   }
2678 
2679   return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2680 }
2681 
2682 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
2683                                        bool HasSymbolicDisplacement) {
2684   // Offset should fit into 32 bit immediate field.
2685   if (!isInt<32>(Offset))
2686     return false;
2687 
2688   // If we don't have a symbolic displacement - we don't have any extra
2689   // restrictions.
2690   if (!HasSymbolicDisplacement)
2691     return true;
2692 
2693   // We can fold large offsets in the large code model because we always use
2694   // 64-bit offsets.
2695   if (CM == CodeModel::Large)
2696     return true;
2697 
2698   // For kernel code model we know that all object resist in the negative half
2699   // of 32bits address space. We may not accept negative offsets, since they may
2700   // be just off and we may accept pretty large positive ones.
2701   if (CM == CodeModel::Kernel)
2702     return Offset >= 0;
2703 
2704   // For other non-large code models we assume that latest small object is 16MB
2705   // before end of 31 bits boundary. We may also accept pretty large negative
2706   // constants knowing that all objects are in the positive half of address
2707   // space.
2708   return Offset < 16 * 1024 * 1024;
2709 }
2710 
2711 /// Return true if the condition is an signed comparison operation.
2712 static bool isX86CCSigned(unsigned X86CC) {
2713   switch (X86CC) {
2714   default:
2715     llvm_unreachable("Invalid integer condition!");
2716   case X86::COND_E:
2717   case X86::COND_NE:
2718   case X86::COND_B:
2719   case X86::COND_A:
2720   case X86::COND_BE:
2721   case X86::COND_AE:
2722     return false;
2723   case X86::COND_G:
2724   case X86::COND_GE:
2725   case X86::COND_L:
2726   case X86::COND_LE:
2727     return true;
2728   }
2729 }
2730 
2731 static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2732   switch (SetCCOpcode) {
2733   default: llvm_unreachable("Invalid integer condition!");
2734   case ISD::SETEQ:  return X86::COND_E;
2735   case ISD::SETGT:  return X86::COND_G;
2736   case ISD::SETGE:  return X86::COND_GE;
2737   case ISD::SETLT:  return X86::COND_L;
2738   case ISD::SETLE:  return X86::COND_LE;
2739   case ISD::SETNE:  return X86::COND_NE;
2740   case ISD::SETULT: return X86::COND_B;
2741   case ISD::SETUGT: return X86::COND_A;
2742   case ISD::SETULE: return X86::COND_BE;
2743   case ISD::SETUGE: return X86::COND_AE;
2744   }
2745 }
2746 
2747 /// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2748 /// condition code, returning the condition code and the LHS/RHS of the
2749 /// comparison to make.
2750 static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2751                                     bool isFP, SDValue &LHS, SDValue &RHS,
2752                                     SelectionDAG &DAG) {
2753   if (!isFP) {
2754     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2755       if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
2756         // X > -1   -> X == 0, jump !sign.
2757         RHS = DAG.getConstant(0, DL, RHS.getValueType());
2758         return X86::COND_NS;
2759       }
2760       if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
2761         // X < 0   -> X == 0, jump on sign.
2762         return X86::COND_S;
2763       }
2764       if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
2765         // X >= 0   -> X == 0, jump on !sign.
2766         return X86::COND_NS;
2767       }
2768       if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
2769         // X < 1   -> X <= 0
2770         RHS = DAG.getConstant(0, DL, RHS.getValueType());
2771         return X86::COND_LE;
2772       }
2773     }
2774 
2775     return TranslateIntegerX86CC(SetCCOpcode);
2776   }
2777 
2778   // First determine if it is required or is profitable to flip the operands.
2779 
2780   // If LHS is a foldable load, but RHS is not, flip the condition.
2781   if (ISD::isNON_EXTLoad(LHS.getNode()) &&
2782       !ISD::isNON_EXTLoad(RHS.getNode())) {
2783     SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2784     std::swap(LHS, RHS);
2785   }
2786 
2787   switch (SetCCOpcode) {
2788   default: break;
2789   case ISD::SETOLT:
2790   case ISD::SETOLE:
2791   case ISD::SETUGT:
2792   case ISD::SETUGE:
2793     std::swap(LHS, RHS);
2794     break;
2795   }
2796 
2797   // On a floating point condition, the flags are set as follows:
2798   // ZF  PF  CF   op
2799   //  0 | 0 | 0 | X > Y
2800   //  0 | 0 | 1 | X < Y
2801   //  1 | 0 | 0 | X == Y
2802   //  1 | 1 | 1 | unordered
2803   switch (SetCCOpcode) {
2804   default: llvm_unreachable("Condcode should be pre-legalized away");
2805   case ISD::SETUEQ:
2806   case ISD::SETEQ:   return X86::COND_E;
2807   case ISD::SETOLT:              // flipped
2808   case ISD::SETOGT:
2809   case ISD::SETGT:   return X86::COND_A;
2810   case ISD::SETOLE:              // flipped
2811   case ISD::SETOGE:
2812   case ISD::SETGE:   return X86::COND_AE;
2813   case ISD::SETUGT:              // flipped
2814   case ISD::SETULT:
2815   case ISD::SETLT:   return X86::COND_B;
2816   case ISD::SETUGE:              // flipped
2817   case ISD::SETULE:
2818   case ISD::SETLE:   return X86::COND_BE;
2819   case ISD::SETONE:
2820   case ISD::SETNE:   return X86::COND_NE;
2821   case ISD::SETUO:   return X86::COND_P;
2822   case ISD::SETO:    return X86::COND_NP;
2823   case ISD::SETOEQ:
2824   case ISD::SETUNE:  return X86::COND_INVALID;
2825   }
2826 }
2827 
2828 /// Is there a floating point cmov for the specific X86 condition code?
2829 /// Current x86 isa includes the following FP cmov instructions:
2830 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2831 static bool hasFPCMov(unsigned X86CC) {
2832   switch (X86CC) {
2833   default:
2834     return false;
2835   case X86::COND_B:
2836   case X86::COND_BE:
2837   case X86::COND_E:
2838   case X86::COND_P:
2839   case X86::COND_A:
2840   case X86::COND_AE:
2841   case X86::COND_NE:
2842   case X86::COND_NP:
2843     return true;
2844   }
2845 }
2846 
2847 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
2848   return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
2849          VT.is512BitVector();
2850 }
2851 
2852 bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2853                                            const CallInst &I,
2854                                            MachineFunction &MF,
2855                                            unsigned Intrinsic) const {
2856   Info.flags = MachineMemOperand::MONone;
2857   Info.offset = 0;
2858 
2859   const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
2860   if (!IntrData) {
2861     switch (Intrinsic) {
2862     case Intrinsic::x86_aesenc128kl:
2863     case Intrinsic::x86_aesdec128kl:
2864       Info.opc = ISD::INTRINSIC_W_CHAIN;
2865       Info.ptrVal = I.getArgOperand(1);
2866       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2867       Info.align = Align(1);
2868       Info.flags |= MachineMemOperand::MOLoad;
2869       return true;
2870     case Intrinsic::x86_aesenc256kl:
2871     case Intrinsic::x86_aesdec256kl:
2872       Info.opc = ISD::INTRINSIC_W_CHAIN;
2873       Info.ptrVal = I.getArgOperand(1);
2874       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2875       Info.align = Align(1);
2876       Info.flags |= MachineMemOperand::MOLoad;
2877       return true;
2878     case Intrinsic::x86_aesencwide128kl:
2879     case Intrinsic::x86_aesdecwide128kl:
2880       Info.opc = ISD::INTRINSIC_W_CHAIN;
2881       Info.ptrVal = I.getArgOperand(0);
2882       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2883       Info.align = Align(1);
2884       Info.flags |= MachineMemOperand::MOLoad;
2885       return true;
2886     case Intrinsic::x86_aesencwide256kl:
2887     case Intrinsic::x86_aesdecwide256kl:
2888       Info.opc = ISD::INTRINSIC_W_CHAIN;
2889       Info.ptrVal = I.getArgOperand(0);
2890       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2891       Info.align = Align(1);
2892       Info.flags |= MachineMemOperand::MOLoad;
2893       return true;
2894     case Intrinsic::x86_cmpccxadd32:
2895     case Intrinsic::x86_cmpccxadd64:
2896     case Intrinsic::x86_atomic_bts:
2897     case Intrinsic::x86_atomic_btc:
2898     case Intrinsic::x86_atomic_btr: {
2899       Info.opc = ISD::INTRINSIC_W_CHAIN;
2900       Info.ptrVal = I.getArgOperand(0);
2901       unsigned Size = I.getType()->getScalarSizeInBits();
2902       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2903       Info.align = Align(Size);
2904       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2905                     MachineMemOperand::MOVolatile;
2906       return true;
2907     }
2908     case Intrinsic::x86_atomic_bts_rm:
2909     case Intrinsic::x86_atomic_btc_rm:
2910     case Intrinsic::x86_atomic_btr_rm: {
2911       Info.opc = ISD::INTRINSIC_W_CHAIN;
2912       Info.ptrVal = I.getArgOperand(0);
2913       unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2914       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2915       Info.align = Align(Size);
2916       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2917                     MachineMemOperand::MOVolatile;
2918       return true;
2919     }
2920     case Intrinsic::x86_aadd32:
2921     case Intrinsic::x86_aadd64:
2922     case Intrinsic::x86_aand32:
2923     case Intrinsic::x86_aand64:
2924     case Intrinsic::x86_aor32:
2925     case Intrinsic::x86_aor64:
2926     case Intrinsic::x86_axor32:
2927     case Intrinsic::x86_axor64:
2928     case Intrinsic::x86_atomic_add_cc:
2929     case Intrinsic::x86_atomic_sub_cc:
2930     case Intrinsic::x86_atomic_or_cc:
2931     case Intrinsic::x86_atomic_and_cc:
2932     case Intrinsic::x86_atomic_xor_cc: {
2933       Info.opc = ISD::INTRINSIC_W_CHAIN;
2934       Info.ptrVal = I.getArgOperand(0);
2935       unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2936       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2937       Info.align = Align(Size);
2938       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2939                     MachineMemOperand::MOVolatile;
2940       return true;
2941     }
2942     }
2943     return false;
2944   }
2945 
2946   switch (IntrData->Type) {
2947   case TRUNCATE_TO_MEM_VI8:
2948   case TRUNCATE_TO_MEM_VI16:
2949   case TRUNCATE_TO_MEM_VI32: {
2950     Info.opc = ISD::INTRINSIC_VOID;
2951     Info.ptrVal = I.getArgOperand(0);
2952     MVT VT  = MVT::getVT(I.getArgOperand(1)->getType());
2953     MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
2954     if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
2955       ScalarVT = MVT::i8;
2956     else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
2957       ScalarVT = MVT::i16;
2958     else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
2959       ScalarVT = MVT::i32;
2960 
2961     Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
2962     Info.align = Align(1);
2963     Info.flags |= MachineMemOperand::MOStore;
2964     break;
2965   }
2966   case GATHER:
2967   case GATHER_AVX2: {
2968     Info.opc = ISD::INTRINSIC_W_CHAIN;
2969     Info.ptrVal = nullptr;
2970     MVT DataVT = MVT::getVT(I.getType());
2971     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2972     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2973                                 IndexVT.getVectorNumElements());
2974     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2975     Info.align = Align(1);
2976     Info.flags |= MachineMemOperand::MOLoad;
2977     break;
2978   }
2979   case SCATTER: {
2980     Info.opc = ISD::INTRINSIC_VOID;
2981     Info.ptrVal = nullptr;
2982     MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
2983     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2984     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2985                                 IndexVT.getVectorNumElements());
2986     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2987     Info.align = Align(1);
2988     Info.flags |= MachineMemOperand::MOStore;
2989     break;
2990   }
2991   default:
2992     return false;
2993   }
2994 
2995   return true;
2996 }
2997 
2998 /// Returns true if the target can instruction select the
2999 /// specified FP immediate natively. If false, the legalizer will
3000 /// materialize the FP immediate as a load from a constant pool.
3001 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3002                                      bool ForCodeSize) const {
3003   for (const APFloat &FPImm : LegalFPImmediates)
3004     if (Imm.bitwiseIsEqual(FPImm))
3005       return true;
3006   return false;
3007 }
3008 
3009 bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3010                                               ISD::LoadExtType ExtTy,
3011                                               EVT NewVT) const {
3012   assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
3013 
3014   // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3015   // relocation target a movq or addq instruction: don't let the load shrink.
3016   SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3017   if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3018     if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3019       return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3020 
3021   // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
3022   // those uses are extracted directly into a store, then the extract + store
3023   // can be store-folded. Therefore, it's probably not worth splitting the load.
3024   EVT VT = Load->getValueType(0);
3025   if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
3026     for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
3027       // Skip uses of the chain value. Result 0 of the node is the load value.
3028       if (UI.getUse().getResNo() != 0)
3029         continue;
3030 
3031       // If this use is not an extract + store, it's probably worth splitting.
3032       if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
3033           UI->use_begin()->getOpcode() != ISD::STORE)
3034         return true;
3035     }
3036     // All non-chain uses are extract + store.
3037     return false;
3038   }
3039 
3040   return true;
3041 }
3042 
3043 /// Returns true if it is beneficial to convert a load of a constant
3044 /// to just the constant itself.
3045 bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3046                                                           Type *Ty) const {
3047   assert(Ty->isIntegerTy());
3048 
3049   unsigned BitSize = Ty->getPrimitiveSizeInBits();
3050   if (BitSize == 0 || BitSize > 64)
3051     return false;
3052   return true;
3053 }
3054 
3055 bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
3056   // If we are using XMM registers in the ABI and the condition of the select is
3057   // a floating-point compare and we have blendv or conditional move, then it is
3058   // cheaper to select instead of doing a cross-register move and creating a
3059   // load that depends on the compare result.
3060   bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
3061   return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
3062 }
3063 
3064 bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
3065   // TODO: It might be a win to ease or lift this restriction, but the generic
3066   // folds in DAGCombiner conflict with vector folds for an AVX512 target.
3067   if (VT.isVector() && Subtarget.hasAVX512())
3068     return false;
3069 
3070   return true;
3071 }
3072 
3073 bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
3074                                                SDValue C) const {
3075   // TODO: We handle scalars using custom code, but generic combining could make
3076   // that unnecessary.
3077   APInt MulC;
3078   if (!ISD::isConstantSplatVector(C.getNode(), MulC))
3079     return false;
3080 
3081   // Find the type this will be legalized too. Otherwise we might prematurely
3082   // convert this to shl+add/sub and then still have to type legalize those ops.
3083   // Another choice would be to defer the decision for illegal types until
3084   // after type legalization. But constant splat vectors of i64 can't make it
3085   // through type legalization on 32-bit targets so we would need to special
3086   // case vXi64.
3087   while (getTypeAction(Context, VT) != TypeLegal)
3088     VT = getTypeToTransformTo(Context, VT);
3089 
3090   // If vector multiply is legal, assume that's faster than shl + add/sub.
3091   // Multiply is a complex op with higher latency and lower throughput in
3092   // most implementations, sub-vXi32 vector multiplies are always fast,
3093   // vXi32 mustn't have a SlowMULLD implementation, and anything larger (vXi64)
3094   // is always going to be slow.
3095   unsigned EltSizeInBits = VT.getScalarSizeInBits();
3096   if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 &&
3097       (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow()))
3098     return false;
3099 
3100   // shl+add, shl+sub, shl+add+neg
3101   return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
3102          (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
3103 }
3104 
3105 bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
3106                                                 unsigned Index) const {
3107   if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3108     return false;
3109 
3110   // Mask vectors support all subregister combinations and operations that
3111   // extract half of vector.
3112   if (ResVT.getVectorElementType() == MVT::i1)
3113     return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) &&
3114                           (Index == ResVT.getVectorNumElements()));
3115 
3116   return (Index % ResVT.getVectorNumElements()) == 0;
3117 }
3118 
3119 bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
3120   unsigned Opc = VecOp.getOpcode();
3121 
3122   // Assume target opcodes can't be scalarized.
3123   // TODO - do we have any exceptions?
3124   if (Opc >= ISD::BUILTIN_OP_END)
3125     return false;
3126 
3127   // If the vector op is not supported, try to convert to scalar.
3128   EVT VecVT = VecOp.getValueType();
3129   if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
3130     return true;
3131 
3132   // If the vector op is supported, but the scalar op is not, the transform may
3133   // not be worthwhile.
3134   EVT ScalarVT = VecVT.getScalarType();
3135   return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
3136 }
3137 
3138 bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT,
3139                                              bool) const {
3140   // TODO: Allow vectors?
3141   if (VT.isVector())
3142     return false;
3143   return VT.isSimple() || !isOperationExpand(Opcode, VT);
3144 }
3145 
3146 bool X86TargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
3147   // Speculate cttz only if we can directly use TZCNT or can promote to i32.
3148   return Subtarget.hasBMI() ||
3149          (!Ty->isVectorTy() && Ty->getScalarSizeInBits() < 32);
3150 }
3151 
3152 bool X86TargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
3153   // Speculate ctlz only if we can directly use LZCNT.
3154   return Subtarget.hasLZCNT();
3155 }
3156 
3157 bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const {
3158   // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
3159   // expensive than a straight movsd. On the other hand, it's important to
3160   // shrink long double fp constant since fldt is very slow.
3161   return !Subtarget.hasSSE2() || VT == MVT::f80;
3162 }
3163 
3164 bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
3165   return (VT == MVT::f64 && Subtarget.hasSSE2()) ||
3166          (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
3167 }
3168 
3169 bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
3170                                                 const SelectionDAG &DAG,
3171                                                 const MachineMemOperand &MMO) const {
3172   if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
3173       BitcastVT.getVectorElementType() == MVT::i1)
3174     return false;
3175 
3176   if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
3177     return false;
3178 
3179   // If both types are legal vectors, it's always ok to convert them.
3180   if (LoadVT.isVector() && BitcastVT.isVector() &&
3181       isTypeLegal(LoadVT) && isTypeLegal(BitcastVT))
3182     return true;
3183 
3184   return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
3185 }
3186 
3187 bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
3188                                          const MachineFunction &MF) const {
3189   // Do not merge to float value size (128 bytes) if no implicit
3190   // float attribute is set.
3191   bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
3192 
3193   if (NoFloat) {
3194     unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
3195     return (MemVT.getSizeInBits() <= MaxIntSize);
3196   }
3197   // Make sure we don't merge greater than our preferred vector
3198   // width.
3199   if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
3200     return false;
3201 
3202   return true;
3203 }
3204 
3205 bool X86TargetLowering::isCtlzFast() const {
3206   return Subtarget.hasFastLZCNT();
3207 }
3208 
3209 bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
3210     const Instruction &AndI) const {
3211   return true;
3212 }
3213 
3214 bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
3215   EVT VT = Y.getValueType();
3216 
3217   if (VT.isVector())
3218     return false;
3219 
3220   if (!Subtarget.hasBMI())
3221     return false;
3222 
3223   // There are only 32-bit and 64-bit forms for 'andn'.
3224   if (VT != MVT::i32 && VT != MVT::i64)
3225     return false;
3226 
3227   return !isa<ConstantSDNode>(Y);
3228 }
3229 
3230 bool X86TargetLowering::hasAndNot(SDValue Y) const {
3231   EVT VT = Y.getValueType();
3232 
3233   if (!VT.isVector())
3234     return hasAndNotCompare(Y);
3235 
3236   // Vector.
3237 
3238   if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)
3239     return false;
3240 
3241   if (VT == MVT::v4i32)
3242     return true;
3243 
3244   return Subtarget.hasSSE2();
3245 }
3246 
3247 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
3248   return X.getValueType().isScalarInteger(); // 'bt'
3249 }
3250 
3251 bool X86TargetLowering::
3252     shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3253         SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
3254         unsigned OldShiftOpcode, unsigned NewShiftOpcode,
3255         SelectionDAG &DAG) const {
3256   // Does baseline recommend not to perform the fold by default?
3257   if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3258           X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
3259     return false;
3260   // For scalars this transform is always beneficial.
3261   if (X.getValueType().isScalarInteger())
3262     return true;
3263   // If all the shift amounts are identical, then transform is beneficial even
3264   // with rudimentary SSE2 shifts.
3265   if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
3266     return true;
3267   // If we have AVX2 with it's powerful shift operations, then it's also good.
3268   if (Subtarget.hasAVX2())
3269     return true;
3270   // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'.
3271   return NewShiftOpcode == ISD::SHL;
3272 }
3273 
3274 unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
3275     EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
3276     const APInt &ShiftOrRotateAmt, const std::optional<APInt> &AndMask) const {
3277   if (!VT.isInteger())
3278     return ShiftOpc;
3279 
3280   bool PreferRotate = false;
3281   if (VT.isVector()) {
3282     // For vectors, if we have rotate instruction support, then its definetly
3283     // best. Otherwise its not clear what the best so just don't make changed.
3284     PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 ||
3285                                              VT.getScalarType() == MVT::i64);
3286   } else {
3287     // For scalar, if we have bmi prefer rotate for rorx. Otherwise prefer
3288     // rotate unless we have a zext mask+shr.
3289     PreferRotate = Subtarget.hasBMI2();
3290     if (!PreferRotate) {
3291       unsigned MaskBits =
3292           VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue();
3293       PreferRotate = (MaskBits != 8) && (MaskBits != 16) && (MaskBits != 32);
3294     }
3295   }
3296 
3297   if (ShiftOpc == ISD::SHL || ShiftOpc == ISD::SRL) {
3298     assert(AndMask.has_value() && "Null andmask when querying about shift+and");
3299 
3300     if (PreferRotate && MayTransformRotate)
3301       return ISD::ROTL;
3302 
3303     // If vector we don't really get much benefit swapping around constants.
3304     // Maybe we could check if the DAG has the flipped node already in the
3305     // future.
3306     if (VT.isVector())
3307       return ShiftOpc;
3308 
3309     // See if the beneficial to swap shift type.
3310     if (ShiftOpc == ISD::SHL) {
3311       // If the current setup has imm64 mask, then inverse will have
3312       // at least imm32 mask (or be zext i32 -> i64).
3313       if (VT == MVT::i64)
3314         return AndMask->getSignificantBits() > 32 ? (unsigned)ISD::SRL
3315                                                   : ShiftOpc;
3316 
3317       // We can only benefit if req at least 7-bit for the mask. We
3318       // don't want to replace shl of 1,2,3 as they can be implemented
3319       // with lea/add.
3320       return ShiftOrRotateAmt.uge(7) ? (unsigned)ISD::SRL : ShiftOpc;
3321     }
3322 
3323     if (VT == MVT::i64)
3324       // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is
3325       // extremely efficient.
3326       return AndMask->getSignificantBits() > 33 ? (unsigned)ISD::SHL : ShiftOpc;
3327 
3328     // Keep small shifts as shl so we can generate add/lea.
3329     return ShiftOrRotateAmt.ult(7) ? (unsigned)ISD::SHL : ShiftOpc;
3330   }
3331 
3332   // We prefer rotate for vectors of if we won't get a zext mask with SRL
3333   // (PreferRotate will be set in the latter case).
3334   if (PreferRotate || VT.isVector())
3335     return ShiftOpc;
3336 
3337   // Non-vector type and we have a zext mask with SRL.
3338   return ISD::SRL;
3339 }
3340 
3341 bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const {
3342   return N->getOpcode() != ISD::FP_EXTEND;
3343 }
3344 
3345 bool X86TargetLowering::shouldFoldConstantShiftPairToMask(
3346     const SDNode *N, CombineLevel Level) const {
3347   assert(((N->getOpcode() == ISD::SHL &&
3348            N->getOperand(0).getOpcode() == ISD::SRL) ||
3349           (N->getOpcode() == ISD::SRL &&
3350            N->getOperand(0).getOpcode() == ISD::SHL)) &&
3351          "Expected shift-shift mask");
3352   // TODO: Should we always create i64 masks? Or only folded immediates?
3353   EVT VT = N->getValueType(0);
3354   if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) ||
3355       (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) {
3356     // Only fold if the shift values are equal - so it folds to AND.
3357     // TODO - we should fold if either is a non-uniform vector but we don't do
3358     // the fold for non-splats yet.
3359     return N->getOperand(1) == N->getOperand(0).getOperand(1);
3360   }
3361   return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N, Level);
3362 }
3363 
3364 bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
3365   EVT VT = Y.getValueType();
3366 
3367   // For vectors, we don't have a preference, but we probably want a mask.
3368   if (VT.isVector())
3369     return false;
3370 
3371   // 64-bit shifts on 32-bit targets produce really bad bloated code.
3372   if (VT == MVT::i64 && !Subtarget.is64Bit())
3373     return false;
3374 
3375   return true;
3376 }
3377 
3378 TargetLowering::ShiftLegalizationStrategy
3379 X86TargetLowering::preferredShiftLegalizationStrategy(
3380     SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
3381   if (DAG.getMachineFunction().getFunction().hasMinSize() &&
3382       !Subtarget.isOSWindows())
3383     return ShiftLegalizationStrategy::LowerToLibcall;
3384   return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
3385                                                             ExpansionFactor);
3386 }
3387 
3388 bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {
3389   // Any legal vector type can be splatted more efficiently than
3390   // loading/spilling from memory.
3391   return isTypeLegal(VT);
3392 }
3393 
3394 MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {
3395   MVT VT = MVT::getIntegerVT(NumBits);
3396   if (isTypeLegal(VT))
3397     return VT;
3398 
3399   // PMOVMSKB can handle this.
3400   if (NumBits == 128 && isTypeLegal(MVT::v16i8))
3401     return MVT::v16i8;
3402 
3403   // VPMOVMSKB can handle this.
3404   if (NumBits == 256 && isTypeLegal(MVT::v32i8))
3405     return MVT::v32i8;
3406 
3407   // TODO: Allow 64-bit type for 32-bit target.
3408   // TODO: 512-bit types should be allowed, but make sure that those
3409   // cases are handled in combineVectorSizedSetCCEquality().
3410 
3411   return MVT::INVALID_SIMPLE_VALUE_TYPE;
3412 }
3413 
3414 /// Val is the undef sentinel value or equal to the specified value.
3415 static bool isUndefOrEqual(int Val, int CmpVal) {
3416   return ((Val == SM_SentinelUndef) || (Val == CmpVal));
3417 }
3418 
3419 /// Return true if every element in Mask is the undef sentinel value or equal to
3420 /// the specified value.
3421 static bool isUndefOrEqual(ArrayRef<int> Mask, int CmpVal) {
3422   return llvm::all_of(Mask, [CmpVal](int M) {
3423     return (M == SM_SentinelUndef) || (M == CmpVal);
3424   });
3425 }
3426 
3427 /// Return true if every element in Mask, beginning from position Pos and ending
3428 /// in Pos+Size is the undef sentinel value or equal to the specified value.
3429 static bool isUndefOrEqualInRange(ArrayRef<int> Mask, int CmpVal, unsigned Pos,
3430                                   unsigned Size) {
3431   return llvm::all_of(Mask.slice(Pos, Size),
3432                       [CmpVal](int M) { return isUndefOrEqual(M, CmpVal); });
3433 }
3434 
3435 /// Val is either the undef or zero sentinel value.
3436 static bool isUndefOrZero(int Val) {
3437   return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
3438 }
3439 
3440 /// Return true if every element in Mask, beginning from position Pos and ending
3441 /// in Pos+Size is the undef sentinel value.
3442 static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
3443   return llvm::all_of(Mask.slice(Pos, Size),
3444                       [](int M) { return M == SM_SentinelUndef; });
3445 }
3446 
3447 /// Return true if the mask creates a vector whose lower half is undefined.
3448 static bool isUndefLowerHalf(ArrayRef<int> Mask) {
3449   unsigned NumElts = Mask.size();
3450   return isUndefInRange(Mask, 0, NumElts / 2);
3451 }
3452 
3453 /// Return true if the mask creates a vector whose upper half is undefined.
3454 static bool isUndefUpperHalf(ArrayRef<int> Mask) {
3455   unsigned NumElts = Mask.size();
3456   return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
3457 }
3458 
3459 /// Return true if Val falls within the specified range (L, H].
3460 static bool isInRange(int Val, int Low, int Hi) {
3461   return (Val >= Low && Val < Hi);
3462 }
3463 
3464 /// Return true if the value of any element in Mask falls within the specified
3465 /// range (L, H].
3466 static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
3467   return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
3468 }
3469 
3470 /// Return true if the value of any element in Mask is the zero sentinel value.
3471 static bool isAnyZero(ArrayRef<int> Mask) {
3472   return llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
3473 }
3474 
3475 /// Return true if the value of any element in Mask is the zero or undef
3476 /// sentinel values.
3477 static bool isAnyZeroOrUndef(ArrayRef<int> Mask) {
3478   return llvm::any_of(Mask, [](int M) {
3479     return M == SM_SentinelZero || M == SM_SentinelUndef;
3480   });
3481 }
3482 
3483 /// Return true if Val is undef or if its value falls within the
3484 /// specified range (L, H].
3485 static bool isUndefOrInRange(int Val, int Low, int Hi) {
3486   return (Val == SM_SentinelUndef) || isInRange(Val, Low, Hi);
3487 }
3488 
3489 /// Return true if every element in Mask is undef or if its value
3490 /// falls within the specified range (L, H].
3491 static bool isUndefOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3492   return llvm::all_of(
3493       Mask, [Low, Hi](int M) { return isUndefOrInRange(M, Low, Hi); });
3494 }
3495 
3496 /// Return true if Val is undef, zero or if its value falls within the
3497 /// specified range (L, H].
3498 static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi) {
3499   return isUndefOrZero(Val) || isInRange(Val, Low, Hi);
3500 }
3501 
3502 /// Return true if every element in Mask is undef, zero or if its value
3503 /// falls within the specified range (L, H].
3504 static bool isUndefOrZeroOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3505   return llvm::all_of(
3506       Mask, [Low, Hi](int M) { return isUndefOrZeroOrInRange(M, Low, Hi); });
3507 }
3508 
3509 /// Return true if every element in Mask, beginning
3510 /// from position Pos and ending in Pos + Size, falls within the specified
3511 /// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
3512 static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, unsigned Pos,
3513                                        unsigned Size, int Low, int Step = 1) {
3514   for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3515     if (!isUndefOrEqual(Mask[i], Low))
3516       return false;
3517   return true;
3518 }
3519 
3520 /// Return true if every element in Mask, beginning
3521 /// from position Pos and ending in Pos+Size, falls within the specified
3522 /// sequential range (Low, Low+Size], or is undef or is zero.
3523 static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3524                                              unsigned Size, int Low,
3525                                              int Step = 1) {
3526   for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3527     if (!isUndefOrZero(Mask[i]) && Mask[i] != Low)
3528       return false;
3529   return true;
3530 }
3531 
3532 /// Return true if every element in Mask, beginning
3533 /// from position Pos and ending in Pos+Size is undef or is zero.
3534 static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3535                                  unsigned Size) {
3536   return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
3537 }
3538 
3539 /// Helper function to test whether a shuffle mask could be
3540 /// simplified by widening the elements being shuffled.
3541 ///
3542 /// Appends the mask for wider elements in WidenedMask if valid. Otherwise
3543 /// leaves it in an unspecified state.
3544 ///
3545 /// NOTE: This must handle normal vector shuffle masks and *target* vector
3546 /// shuffle masks. The latter have the special property of a '-2' representing
3547 /// a zero-ed lane of a vector.
3548 static bool canWidenShuffleElements(ArrayRef<int> Mask,
3549                                     SmallVectorImpl<int> &WidenedMask) {
3550   WidenedMask.assign(Mask.size() / 2, 0);
3551   for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
3552     int M0 = Mask[i];
3553     int M1 = Mask[i + 1];
3554 
3555     // If both elements are undef, its trivial.
3556     if (M0 == SM_SentinelUndef && M1 == SM_SentinelUndef) {
3557       WidenedMask[i / 2] = SM_SentinelUndef;
3558       continue;
3559     }
3560 
3561     // Check for an undef mask and a mask value properly aligned to fit with
3562     // a pair of values. If we find such a case, use the non-undef mask's value.
3563     if (M0 == SM_SentinelUndef && M1 >= 0 && (M1 % 2) == 1) {
3564       WidenedMask[i / 2] = M1 / 2;
3565       continue;
3566     }
3567     if (M1 == SM_SentinelUndef && M0 >= 0 && (M0 % 2) == 0) {
3568       WidenedMask[i / 2] = M0 / 2;
3569       continue;
3570     }
3571 
3572     // When zeroing, we need to spread the zeroing across both lanes to widen.
3573     if (M0 == SM_SentinelZero || M1 == SM_SentinelZero) {
3574       if ((M0 == SM_SentinelZero || M0 == SM_SentinelUndef) &&
3575           (M1 == SM_SentinelZero || M1 == SM_SentinelUndef)) {
3576         WidenedMask[i / 2] = SM_SentinelZero;
3577         continue;
3578       }
3579       return false;
3580     }
3581 
3582     // Finally check if the two mask values are adjacent and aligned with
3583     // a pair.
3584     if (M0 != SM_SentinelUndef && (M0 % 2) == 0 && (M0 + 1) == M1) {
3585       WidenedMask[i / 2] = M0 / 2;
3586       continue;
3587     }
3588 
3589     // Otherwise we can't safely widen the elements used in this shuffle.
3590     return false;
3591   }
3592   assert(WidenedMask.size() == Mask.size() / 2 &&
3593          "Incorrect size of mask after widening the elements!");
3594 
3595   return true;
3596 }
3597 
3598 static bool canWidenShuffleElements(ArrayRef<int> Mask,
3599                                     const APInt &Zeroable,
3600                                     bool V2IsZero,
3601                                     SmallVectorImpl<int> &WidenedMask) {
3602   // Create an alternative mask with info about zeroable elements.
3603   // Here we do not set undef elements as zeroable.
3604   SmallVector<int, 64> ZeroableMask(Mask);
3605   if (V2IsZero) {
3606     assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
3607     for (int i = 0, Size = Mask.size(); i != Size; ++i)
3608       if (Mask[i] != SM_SentinelUndef && Zeroable[i])
3609         ZeroableMask[i] = SM_SentinelZero;
3610   }
3611   return canWidenShuffleElements(ZeroableMask, WidenedMask);
3612 }
3613 
3614 static bool canWidenShuffleElements(ArrayRef<int> Mask) {
3615   SmallVector<int, 32> WidenedMask;
3616   return canWidenShuffleElements(Mask, WidenedMask);
3617 }
3618 
3619 // Attempt to narrow/widen shuffle mask until it matches the target number of
3620 // elements.
3621 static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
3622                                  SmallVectorImpl<int> &ScaledMask) {
3623   unsigned NumSrcElts = Mask.size();
3624   assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) &&
3625          "Illegal shuffle scale factor");
3626 
3627   // Narrowing is guaranteed to work.
3628   if (NumDstElts >= NumSrcElts) {
3629     int Scale = NumDstElts / NumSrcElts;
3630     llvm::narrowShuffleMaskElts(Scale, Mask, ScaledMask);
3631     return true;
3632   }
3633 
3634   // We have to repeat the widening until we reach the target size, but we can
3635   // split out the first widening as it sets up ScaledMask for us.
3636   if (canWidenShuffleElements(Mask, ScaledMask)) {
3637     while (ScaledMask.size() > NumDstElts) {
3638       SmallVector<int, 16> WidenedMask;
3639       if (!canWidenShuffleElements(ScaledMask, WidenedMask))
3640         return false;
3641       ScaledMask = std::move(WidenedMask);
3642     }
3643     return true;
3644   }
3645 
3646   return false;
3647 }
3648 
3649 /// Returns true if Elt is a constant zero or a floating point constant +0.0.
3650 bool X86::isZeroNode(SDValue Elt) {
3651   return isNullConstant(Elt) || isNullFPConstant(Elt);
3652 }
3653 
3654 // Build a vector of constants.
3655 // Use an UNDEF node if MaskElt == -1.
3656 // Split 64-bit constants in the 32-bit mode.
3657 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
3658                               const SDLoc &dl, bool IsMask = false) {
3659 
3660   SmallVector<SDValue, 32>  Ops;
3661   bool Split = false;
3662 
3663   MVT ConstVecVT = VT;
3664   unsigned NumElts = VT.getVectorNumElements();
3665   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3666   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3667     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3668     Split = true;
3669   }
3670 
3671   MVT EltVT = ConstVecVT.getVectorElementType();
3672   for (unsigned i = 0; i < NumElts; ++i) {
3673     bool IsUndef = Values[i] < 0 && IsMask;
3674     SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
3675       DAG.getConstant(Values[i], dl, EltVT);
3676     Ops.push_back(OpNode);
3677     if (Split)
3678       Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
3679                     DAG.getConstant(0, dl, EltVT));
3680   }
3681   SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3682   if (Split)
3683     ConstsNode = DAG.getBitcast(VT, ConstsNode);
3684   return ConstsNode;
3685 }
3686 
3687 static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
3688                               MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
3689   assert(Bits.size() == Undefs.getBitWidth() &&
3690          "Unequal constant and undef arrays");
3691   SmallVector<SDValue, 32> Ops;
3692   bool Split = false;
3693 
3694   MVT ConstVecVT = VT;
3695   unsigned NumElts = VT.getVectorNumElements();
3696   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3697   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3698     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3699     Split = true;
3700   }
3701 
3702   MVT EltVT = ConstVecVT.getVectorElementType();
3703   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
3704     if (Undefs[i]) {
3705       Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
3706       continue;
3707     }
3708     const APInt &V = Bits[i];
3709     assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes");
3710     if (Split) {
3711       Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT));
3712       Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT));
3713     } else if (EltVT == MVT::f32) {
3714       APFloat FV(APFloat::IEEEsingle(), V);
3715       Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3716     } else if (EltVT == MVT::f64) {
3717       APFloat FV(APFloat::IEEEdouble(), V);
3718       Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3719     } else {
3720       Ops.push_back(DAG.getConstant(V, dl, EltVT));
3721     }
3722   }
3723 
3724   SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3725   return DAG.getBitcast(VT, ConstsNode);
3726 }
3727 
3728 static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
3729                               SelectionDAG &DAG, const SDLoc &dl) {
3730   APInt Undefs = APInt::getZero(Bits.size());
3731   return getConstVector(Bits, Undefs, VT, DAG, dl);
3732 }
3733 
3734 /// Returns a vector of specified type with all zero elements.
3735 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
3736                              SelectionDAG &DAG, const SDLoc &dl) {
3737   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||
3738           VT.getVectorElementType() == MVT::i1) &&
3739          "Unexpected vector type");
3740 
3741   // Try to build SSE/AVX zero vectors as <N x i32> bitcasted to their dest
3742   // type. This ensures they get CSE'd. But if the integer type is not
3743   // available, use a floating-point +0.0 instead.
3744   SDValue Vec;
3745   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3746   if (!Subtarget.hasSSE2() && VT.is128BitVector()) {
3747     Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
3748   } else if (VT.isFloatingPoint() &&
3749              TLI.isTypeLegal(VT.getVectorElementType())) {
3750     Vec = DAG.getConstantFP(+0.0, dl, VT);
3751   } else if (VT.getVectorElementType() == MVT::i1) {
3752     assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
3753            "Unexpected vector type");
3754     Vec = DAG.getConstant(0, dl, VT);
3755   } else {
3756     unsigned Num32BitElts = VT.getSizeInBits() / 32;
3757     Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
3758   }
3759   return DAG.getBitcast(VT, Vec);
3760 }
3761 
3762 // Helper to determine if the ops are all the extracted subvectors come from a
3763 // single source. If we allow commute they don't have to be in order (Lo/Hi).
3764 static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute) {
3765   if (LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3766       RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3767       LHS.getValueType() != RHS.getValueType() ||
3768       LHS.getOperand(0) != RHS.getOperand(0))
3769     return SDValue();
3770 
3771   SDValue Src = LHS.getOperand(0);
3772   if (Src.getValueSizeInBits() != (LHS.getValueSizeInBits() * 2))
3773     return SDValue();
3774 
3775   unsigned NumElts = LHS.getValueType().getVectorNumElements();
3776   if ((LHS.getConstantOperandAPInt(1) == 0 &&
3777        RHS.getConstantOperandAPInt(1) == NumElts) ||
3778       (AllowCommute && RHS.getConstantOperandAPInt(1) == 0 &&
3779        LHS.getConstantOperandAPInt(1) == NumElts))
3780     return Src;
3781 
3782   return SDValue();
3783 }
3784 
3785 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3786                                 const SDLoc &dl, unsigned vectorWidth) {
3787   EVT VT = Vec.getValueType();
3788   EVT ElVT = VT.getVectorElementType();
3789   unsigned Factor = VT.getSizeInBits() / vectorWidth;
3790   EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
3791                                   VT.getVectorNumElements() / Factor);
3792 
3793   // Extract the relevant vectorWidth bits.  Generate an EXTRACT_SUBVECTOR
3794   unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
3795   assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3796 
3797   // This is the index of the first element of the vectorWidth-bit chunk
3798   // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3799   IdxVal &= ~(ElemsPerChunk - 1);
3800 
3801   // If the input is a buildvector just emit a smaller one.
3802   if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3803     return DAG.getBuildVector(ResultVT, dl,
3804                               Vec->ops().slice(IdxVal, ElemsPerChunk));
3805 
3806   // Check if we're extracting the upper undef of a widening pattern.
3807   if (Vec.getOpcode() == ISD::INSERT_SUBVECTOR && Vec.getOperand(0).isUndef() &&
3808       Vec.getOperand(1).getValueType().getVectorNumElements() <= IdxVal &&
3809       isNullConstant(Vec.getOperand(2)))
3810     return DAG.getUNDEF(ResultVT);
3811 
3812   SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3813   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
3814 }
3815 
3816 /// Generate a DAG to grab 128-bits from a vector > 128 bits.  This
3817 /// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
3818 /// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
3819 /// instructions or a simple subregister reference. Idx is an index in the
3820 /// 128 bits we want.  It need not be aligned to a 128-bit boundary.  That makes
3821 /// lowering EXTRACT_VECTOR_ELT operations easier.
3822 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
3823                                    SelectionDAG &DAG, const SDLoc &dl) {
3824   assert((Vec.getValueType().is256BitVector() ||
3825           Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
3826   return extractSubVector(Vec, IdxVal, DAG, dl, 128);
3827 }
3828 
3829 /// Generate a DAG to grab 256-bits from a 512-bit vector.
3830 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
3831                                    SelectionDAG &DAG, const SDLoc &dl) {
3832   assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
3833   return extractSubVector(Vec, IdxVal, DAG, dl, 256);
3834 }
3835 
3836 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3837                                SelectionDAG &DAG, const SDLoc &dl,
3838                                unsigned vectorWidth) {
3839   assert((vectorWidth == 128 || vectorWidth == 256) &&
3840          "Unsupported vector width");
3841   // Inserting UNDEF is Result
3842   if (Vec.isUndef())
3843     return Result;
3844   EVT VT = Vec.getValueType();
3845   EVT ElVT = VT.getVectorElementType();
3846   EVT ResultVT = Result.getValueType();
3847 
3848   // Insert the relevant vectorWidth bits.
3849   unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
3850   assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3851 
3852   // This is the index of the first element of the vectorWidth-bit chunk
3853   // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3854   IdxVal &= ~(ElemsPerChunk - 1);
3855 
3856   SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3857   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
3858 }
3859 
3860 /// Generate a DAG to put 128-bits into a vector > 128 bits.  This
3861 /// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
3862 /// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
3863 /// simple superregister reference.  Idx is an index in the 128 bits
3864 /// we want.  It need not be aligned to a 128-bit boundary.  That makes
3865 /// lowering INSERT_VECTOR_ELT operations easier.
3866 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3867                                   SelectionDAG &DAG, const SDLoc &dl) {
3868   assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
3869   return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
3870 }
3871 
3872 /// Widen a vector to a larger size with the same scalar type, with the new
3873 /// elements either zero or undef.
3874 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
3875                               const X86Subtarget &Subtarget, SelectionDAG &DAG,
3876                               const SDLoc &dl) {
3877   assert(Vec.getValueSizeInBits().getFixedValue() <= VT.getFixedSizeInBits() &&
3878          Vec.getValueType().getScalarType() == VT.getScalarType() &&
3879          "Unsupported vector widening type");
3880   SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
3881                                 : DAG.getUNDEF(VT);
3882   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
3883                      DAG.getIntPtrConstant(0, dl));
3884 }
3885 
3886 /// Widen a vector to a larger size with the same scalar type, with the new
3887 /// elements either zero or undef.
3888 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
3889                               const X86Subtarget &Subtarget, SelectionDAG &DAG,
3890                               const SDLoc &dl, unsigned WideSizeInBits) {
3891   assert(Vec.getValueSizeInBits() <= WideSizeInBits &&
3892          (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 &&
3893          "Unsupported vector widening type");
3894   unsigned WideNumElts = WideSizeInBits / Vec.getScalarValueSizeInBits();
3895   MVT SVT = Vec.getSimpleValueType().getScalarType();
3896   MVT VT = MVT::getVectorVT(SVT, WideNumElts);
3897   return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
3898 }
3899 
3900 /// Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT
3901 /// and bitcast with integer types.
3902 static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget) {
3903   assert(VT.getVectorElementType() == MVT::i1 && "Expected bool vector");
3904   unsigned NumElts = VT.getVectorNumElements();
3905   if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
3906     return Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
3907   return VT;
3908 }
3909 
3910 /// Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and
3911 /// bitcast with integer types.
3912 static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements,
3913                                const X86Subtarget &Subtarget, SelectionDAG &DAG,
3914                                const SDLoc &dl) {
3915   MVT VT = widenMaskVectorType(Vec.getSimpleValueType(), Subtarget);
3916   return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
3917 }
3918 
3919 // Helper function to collect subvector ops that are concatenated together,
3920 // either by ISD::CONCAT_VECTORS or a ISD::INSERT_SUBVECTOR series.
3921 // The subvectors in Ops are guaranteed to be the same type.
3922 static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
3923                              SelectionDAG &DAG) {
3924   assert(Ops.empty() && "Expected an empty ops vector");
3925 
3926   if (N->getOpcode() == ISD::CONCAT_VECTORS) {
3927     Ops.append(N->op_begin(), N->op_end());
3928     return true;
3929   }
3930 
3931   if (N->getOpcode() == ISD::INSERT_SUBVECTOR) {
3932     SDValue Src = N->getOperand(0);
3933     SDValue Sub = N->getOperand(1);
3934     const APInt &Idx = N->getConstantOperandAPInt(2);
3935     EVT VT = Src.getValueType();
3936     EVT SubVT = Sub.getValueType();
3937 
3938     // TODO - Handle more general insert_subvector chains.
3939     if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) {
3940       // insert_subvector(undef, x, lo)
3941       if (Idx == 0 && Src.isUndef()) {
3942         Ops.push_back(Sub);
3943         Ops.push_back(DAG.getUNDEF(SubVT));
3944         return true;
3945       }
3946       if (Idx == (VT.getVectorNumElements() / 2)) {
3947         // insert_subvector(insert_subvector(undef, x, lo), y, hi)
3948         if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
3949             Src.getOperand(1).getValueType() == SubVT &&
3950             isNullConstant(Src.getOperand(2))) {
3951           Ops.push_back(Src.getOperand(1));
3952           Ops.push_back(Sub);
3953           return true;
3954         }
3955         // insert_subvector(x, extract_subvector(x, lo), hi)
3956         if (Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
3957             Sub.getOperand(0) == Src && isNullConstant(Sub.getOperand(1))) {
3958           Ops.append(2, Sub);
3959           return true;
3960         }
3961         // insert_subvector(undef, x, hi)
3962         if (Src.isUndef()) {
3963           Ops.push_back(DAG.getUNDEF(SubVT));
3964           Ops.push_back(Sub);
3965           return true;
3966         }
3967       }
3968     }
3969   }
3970 
3971   return false;
3972 }
3973 
3974 // Helper to check if \p V can be split into subvectors and the upper subvectors
3975 // are all undef. In which case return the lower subvector.
3976 static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL,
3977                                      SelectionDAG &DAG) {
3978   SmallVector<SDValue> SubOps;
3979   if (!collectConcatOps(V.getNode(), SubOps, DAG))
3980     return SDValue();
3981 
3982   unsigned NumSubOps = SubOps.size();
3983   unsigned HalfNumSubOps = NumSubOps / 2;
3984   assert((NumSubOps % 2) == 0 && "Unexpected number of subvectors");
3985 
3986   ArrayRef<SDValue> UpperOps(SubOps.begin() + HalfNumSubOps, SubOps.end());
3987   if (any_of(UpperOps, [](SDValue Op) { return !Op.isUndef(); }))
3988     return SDValue();
3989 
3990   EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
3991   ArrayRef<SDValue> LowerOps(SubOps.begin(), SubOps.begin() + HalfNumSubOps);
3992   return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps);
3993 }
3994 
3995 // Helper to check if we can access all the constituent subvectors without any
3996 // extract ops.
3997 static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG) {
3998   SmallVector<SDValue> Ops;
3999   return collectConcatOps(N, Ops, DAG);
4000 }
4001 
4002 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
4003                                                const SDLoc &dl) {
4004   EVT VT = Op.getValueType();
4005   unsigned NumElems = VT.getVectorNumElements();
4006   unsigned SizeInBits = VT.getSizeInBits();
4007   assert((NumElems % 2) == 0 && (SizeInBits % 2) == 0 &&
4008          "Can't split odd sized vector");
4009 
4010   // If this is a splat value (with no-undefs) then use the lower subvector,
4011   // which should be a free extraction.
4012   SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
4013   if (DAG.isSplatValue(Op, /*AllowUndefs*/ false))
4014     return std::make_pair(Lo, Lo);
4015 
4016   SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
4017   return std::make_pair(Lo, Hi);
4018 }
4019 
4020 /// Break an operation into 2 half sized ops and then concatenate the results.
4021 static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG) {
4022   unsigned NumOps = Op.getNumOperands();
4023   EVT VT = Op.getValueType();
4024   SDLoc dl(Op);
4025 
4026   // Extract the LHS Lo/Hi vectors
4027   SmallVector<SDValue> LoOps(NumOps, SDValue());
4028   SmallVector<SDValue> HiOps(NumOps, SDValue());
4029   for (unsigned I = 0; I != NumOps; ++I) {
4030     SDValue SrcOp = Op.getOperand(I);
4031     if (!SrcOp.getValueType().isVector()) {
4032       LoOps[I] = HiOps[I] = SrcOp;
4033       continue;
4034     }
4035     std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl);
4036   }
4037 
4038   EVT LoVT, HiVT;
4039   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
4040   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
4041                      DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps),
4042                      DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps));
4043 }
4044 
4045 /// Break an unary integer operation into 2 half sized ops and then
4046 /// concatenate the result back.
4047 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
4048   // Make sure we only try to split 256/512-bit types to avoid creating
4049   // narrow vectors.
4050   EVT VT = Op.getValueType();
4051   (void)VT;
4052   assert((Op.getOperand(0).getValueType().is256BitVector() ||
4053           Op.getOperand(0).getValueType().is512BitVector()) &&
4054          (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4055   assert(Op.getOperand(0).getValueType().getVectorNumElements() ==
4056              VT.getVectorNumElements() &&
4057          "Unexpected VTs!");
4058   return splitVectorOp(Op, DAG);
4059 }
4060 
4061 /// Break a binary integer operation into 2 half sized ops and then
4062 /// concatenate the result back.
4063 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG) {
4064   // Assert that all the types match.
4065   EVT VT = Op.getValueType();
4066   (void)VT;
4067   assert(Op.getOperand(0).getValueType() == VT &&
4068          Op.getOperand(1).getValueType() == VT && "Unexpected VTs!");
4069   assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4070   return splitVectorOp(Op, DAG);
4071 }
4072 
4073 // Helper for splitting operands of an operation to legal target size and
4074 // apply a function on each part.
4075 // Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
4076 // 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
4077 // deciding if/how to split Ops. Ops elements do *not* have to be of type VT.
4078 // The argument Builder is a function that will be applied on each split part:
4079 // SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
4080 template <typename F>
4081 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4082                          const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
4083                          F Builder, bool CheckBWI = true) {
4084   assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2");
4085   unsigned NumSubs = 1;
4086   if ((CheckBWI && Subtarget.useBWIRegs()) ||
4087       (!CheckBWI && Subtarget.useAVX512Regs())) {
4088     if (VT.getSizeInBits() > 512) {
4089       NumSubs = VT.getSizeInBits() / 512;
4090       assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");
4091     }
4092   } else if (Subtarget.hasAVX2()) {
4093     if (VT.getSizeInBits() > 256) {
4094       NumSubs = VT.getSizeInBits() / 256;
4095       assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size");
4096     }
4097   } else {
4098     if (VT.getSizeInBits() > 128) {
4099       NumSubs = VT.getSizeInBits() / 128;
4100       assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size");
4101     }
4102   }
4103 
4104   if (NumSubs == 1)
4105     return Builder(DAG, DL, Ops);
4106 
4107   SmallVector<SDValue, 4> Subs;
4108   for (unsigned i = 0; i != NumSubs; ++i) {
4109     SmallVector<SDValue, 2> SubOps;
4110     for (SDValue Op : Ops) {
4111       EVT OpVT = Op.getValueType();
4112       unsigned NumSubElts = OpVT.getVectorNumElements() / NumSubs;
4113       unsigned SizeSub = OpVT.getSizeInBits() / NumSubs;
4114       SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
4115     }
4116     Subs.push_back(Builder(DAG, DL, SubOps));
4117   }
4118   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
4119 }
4120 
4121 // Helper function that extends a non-512-bit vector op to 512-bits on non-VLX
4122 // targets.
4123 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,
4124                              ArrayRef<SDValue> Ops, SelectionDAG &DAG,
4125                              const X86Subtarget &Subtarget) {
4126   assert(Subtarget.hasAVX512() && "AVX512 target expected");
4127   MVT SVT = VT.getScalarType();
4128 
4129   // If we have a 32/64 splatted constant, splat it to DstTy to
4130   // encourage a foldable broadcast'd operand.
4131   auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {
4132     unsigned OpEltSizeInBits = OpVT.getScalarSizeInBits();
4133     // AVX512 broadcasts 32/64-bit operands.
4134     // TODO: Support float once getAVX512Node is used by fp-ops.
4135     if (!OpVT.isInteger() || OpEltSizeInBits < 32 ||
4136         !DAG.getTargetLoweringInfo().isTypeLegal(SVT))
4137       return SDValue();
4138     // If we're not widening, don't bother if we're not bitcasting.
4139     if (OpVT == DstVT && Op.getOpcode() != ISD::BITCAST)
4140       return SDValue();
4141     if (auto *BV = dyn_cast<BuildVectorSDNode>(peekThroughBitcasts(Op))) {
4142       APInt SplatValue, SplatUndef;
4143       unsigned SplatBitSize;
4144       bool HasAnyUndefs;
4145       if (BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
4146                               HasAnyUndefs, OpEltSizeInBits) &&
4147           !HasAnyUndefs && SplatValue.getBitWidth() == OpEltSizeInBits)
4148         return DAG.getConstant(SplatValue, DL, DstVT);
4149     }
4150     return SDValue();
4151   };
4152 
4153   bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector());
4154 
4155   MVT DstVT = VT;
4156   if (Widen)
4157     DstVT = MVT::getVectorVT(SVT, 512 / SVT.getSizeInBits());
4158 
4159   // Canonicalize src operands.
4160   SmallVector<SDValue> SrcOps(Ops.begin(), Ops.end());
4161   for (SDValue &Op : SrcOps) {
4162     MVT OpVT = Op.getSimpleValueType();
4163     // Just pass through scalar operands.
4164     if (!OpVT.isVector())
4165       continue;
4166     assert(OpVT == VT && "Vector type mismatch");
4167 
4168     if (SDValue BroadcastOp = MakeBroadcastOp(Op, OpVT, DstVT)) {
4169       Op = BroadcastOp;
4170       continue;
4171     }
4172 
4173     // Just widen the subvector by inserting into an undef wide vector.
4174     if (Widen)
4175       Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512);
4176   }
4177 
4178   SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
4179 
4180   // Perform the 512-bit op then extract the bottom subvector.
4181   if (Widen)
4182     Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
4183   return Res;
4184 }
4185 
4186 /// Insert i1-subvector to i1-vector.
4187 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4188                                 const X86Subtarget &Subtarget) {
4189 
4190   SDLoc dl(Op);
4191   SDValue Vec = Op.getOperand(0);
4192   SDValue SubVec = Op.getOperand(1);
4193   SDValue Idx = Op.getOperand(2);
4194   unsigned IdxVal = Op.getConstantOperandVal(2);
4195 
4196   // Inserting undef is a nop. We can just return the original vector.
4197   if (SubVec.isUndef())
4198     return Vec;
4199 
4200   if (IdxVal == 0 && Vec.isUndef()) // the operation is legal
4201     return Op;
4202 
4203   MVT OpVT = Op.getSimpleValueType();
4204   unsigned NumElems = OpVT.getVectorNumElements();
4205   SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
4206 
4207   // Extend to natively supported kshift.
4208   MVT WideOpVT = widenMaskVectorType(OpVT, Subtarget);
4209 
4210   // Inserting into the lsbs of a zero vector is legal. ISel will insert shifts
4211   // if necessary.
4212   if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
4213     // May need to promote to a legal type.
4214     Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4215                      DAG.getConstant(0, dl, WideOpVT),
4216                      SubVec, Idx);
4217     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4218   }
4219 
4220   MVT SubVecVT = SubVec.getSimpleValueType();
4221   unsigned SubVecNumElems = SubVecVT.getVectorNumElements();
4222   assert(IdxVal + SubVecNumElems <= NumElems &&
4223          IdxVal % SubVecVT.getSizeInBits() == 0 &&
4224          "Unexpected index value in INSERT_SUBVECTOR");
4225 
4226   SDValue Undef = DAG.getUNDEF(WideOpVT);
4227 
4228   if (IdxVal == 0) {
4229     // Zero lower bits of the Vec
4230     SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
4231     Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
4232                       ZeroIdx);
4233     Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4234     Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4235     // Merge them together, SubVec should be zero extended.
4236     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4237                          DAG.getConstant(0, dl, WideOpVT),
4238                          SubVec, ZeroIdx);
4239     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4240     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4241   }
4242 
4243   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4244                        Undef, SubVec, ZeroIdx);
4245 
4246   if (Vec.isUndef()) {
4247     assert(IdxVal != 0 && "Unexpected index");
4248     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4249                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4250     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4251   }
4252 
4253   if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
4254     assert(IdxVal != 0 && "Unexpected index");
4255     // If upper elements of Vec are known undef, then just shift into place.
4256     if (llvm::all_of(Vec->ops().slice(IdxVal + SubVecNumElems),
4257                      [](SDValue V) { return V.isUndef(); })) {
4258       SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4259                            DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4260     } else {
4261       NumElems = WideOpVT.getVectorNumElements();
4262       unsigned ShiftLeft = NumElems - SubVecNumElems;
4263       unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4264       SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4265                            DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4266       if (ShiftRight != 0)
4267         SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4268                              DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4269     }
4270     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4271   }
4272 
4273   // Simple case when we put subvector in the upper part
4274   if (IdxVal + SubVecNumElems == NumElems) {
4275     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4276                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4277     if (SubVecNumElems * 2 == NumElems) {
4278       // Special case, use legal zero extending insert_subvector. This allows
4279       // isel to optimize when bits are known zero.
4280       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
4281       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4282                         DAG.getConstant(0, dl, WideOpVT),
4283                         Vec, ZeroIdx);
4284     } else {
4285       // Otherwise use explicit shifts to zero the bits.
4286       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4287                         Undef, Vec, ZeroIdx);
4288       NumElems = WideOpVT.getVectorNumElements();
4289       SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
4290       Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4291       Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4292     }
4293     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4294     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4295   }
4296 
4297   // Inserting into the middle is more complicated.
4298 
4299   NumElems = WideOpVT.getVectorNumElements();
4300 
4301   // Widen the vector if needed.
4302   Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
4303 
4304   unsigned ShiftLeft = NumElems - SubVecNumElems;
4305   unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4306 
4307   // Do an optimization for the most frequently used types.
4308   if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {
4309     APInt Mask0 = APInt::getBitsSet(NumElems, IdxVal, IdxVal + SubVecNumElems);
4310     Mask0.flipAllBits();
4311     SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
4312     SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
4313     Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
4314     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4315                          DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4316     SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4317                          DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4318     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4319 
4320     // Reduce to original width if needed.
4321     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4322   }
4323 
4324   // Clear the upper bits of the subvector and move it to its insert position.
4325   SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4326                        DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4327   SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4328                        DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4329 
4330   // Isolate the bits below the insertion point.
4331   unsigned LowShift = NumElems - IdxVal;
4332   SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
4333                             DAG.getTargetConstant(LowShift, dl, MVT::i8));
4334   Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
4335                     DAG.getTargetConstant(LowShift, dl, MVT::i8));
4336 
4337   // Isolate the bits after the last inserted bit.
4338   unsigned HighShift = IdxVal + SubVecNumElems;
4339   SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
4340                             DAG.getTargetConstant(HighShift, dl, MVT::i8));
4341   High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
4342                     DAG.getTargetConstant(HighShift, dl, MVT::i8));
4343 
4344   // Now OR all 3 pieces together.
4345   Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
4346   SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
4347 
4348   // Reduce to original width if needed.
4349   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4350 }
4351 
4352 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
4353                                 const SDLoc &dl) {
4354   assert(V1.getValueType() == V2.getValueType() && "subvector type mismatch");
4355   EVT SubVT = V1.getValueType();
4356   EVT SubSVT = SubVT.getScalarType();
4357   unsigned SubNumElts = SubVT.getVectorNumElements();
4358   unsigned SubVectorWidth = SubVT.getSizeInBits();
4359   EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
4360   SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
4361   return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
4362 }
4363 
4364 /// Returns a vector of specified type with all bits set.
4365 /// Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>.
4366 /// Then bitcast to their original type, ensuring they get CSE'd.
4367 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
4368   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
4369          "Expected a 128/256/512-bit vector type");
4370 
4371   APInt Ones = APInt::getAllOnes(32);
4372   unsigned NumElts = VT.getSizeInBits() / 32;
4373   SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));
4374   return DAG.getBitcast(VT, Vec);
4375 }
4376 
4377 static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
4378                                       SDValue In, SelectionDAG &DAG) {
4379   EVT InVT = In.getValueType();
4380   assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.");
4381   assert((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
4382           ISD::ZERO_EXTEND == Opcode) &&
4383          "Unknown extension opcode");
4384 
4385   // For 256-bit vectors, we only need the lower (128-bit) input half.
4386   // For 512-bit vectors, we only need the lower input half or quarter.
4387   if (InVT.getSizeInBits() > 128) {
4388     assert(VT.getSizeInBits() == InVT.getSizeInBits() &&
4389            "Expected VTs to be the same size!");
4390     unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();
4391     In = extractSubVector(In, 0, DAG, DL,
4392                           std::max(128U, (unsigned)VT.getSizeInBits() / Scale));
4393     InVT = In.getValueType();
4394   }
4395 
4396   if (VT.getVectorNumElements() != InVT.getVectorNumElements())
4397     Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode);
4398 
4399   return DAG.getNode(Opcode, DL, VT, In);
4400 }
4401 
4402 // Create OR(AND(LHS,MASK),AND(RHS,~MASK)) bit select pattern
4403 static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
4404                             SDValue Mask, SelectionDAG &DAG) {
4405   LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask);
4406   RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS);
4407   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
4408 }
4409 
4410 void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
4411                                    bool Lo, bool Unary) {
4412   assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
4413          "Illegal vector type to unpack");
4414   assert(Mask.empty() && "Expected an empty shuffle mask vector");
4415   int NumElts = VT.getVectorNumElements();
4416   int NumEltsInLane = 128 / VT.getScalarSizeInBits();
4417   for (int i = 0; i < NumElts; ++i) {
4418     unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
4419     int Pos = (i % NumEltsInLane) / 2 + LaneStart;
4420     Pos += (Unary ? 0 : NumElts * (i % 2));
4421     Pos += (Lo ? 0 : NumEltsInLane / 2);
4422     Mask.push_back(Pos);
4423   }
4424 }
4425 
4426 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
4427 /// imposed by AVX and specific to the unary pattern. Example:
4428 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
4429 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
4430 void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
4431                                    bool Lo) {
4432   assert(Mask.empty() && "Expected an empty shuffle mask vector");
4433   int NumElts = VT.getVectorNumElements();
4434   for (int i = 0; i < NumElts; ++i) {
4435     int Pos = i / 2;
4436     Pos += (Lo ? 0 : NumElts / 2);
4437     Mask.push_back(Pos);
4438   }
4439 }
4440 
4441 // Attempt to constant fold, else just create a VECTOR_SHUFFLE.
4442 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
4443                                 SDValue V1, SDValue V2, ArrayRef<int> Mask) {
4444   if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) &&
4445       (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) {
4446     SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
4447     for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) {
4448       int M = Mask[I];
4449       if (M < 0)
4450         continue;
4451       SDValue V = (M < NumElts) ? V1 : V2;
4452       if (V.isUndef())
4453         continue;
4454       Ops[I] = V.getOperand(M % NumElts);
4455     }
4456     return DAG.getBuildVector(VT, dl, Ops);
4457   }
4458 
4459   return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4460 }
4461 
4462 /// Returns a vector_shuffle node for an unpackl operation.
4463 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4464                           SDValue V1, SDValue V2) {
4465   SmallVector<int, 8> Mask;
4466   createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
4467   return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4468 }
4469 
4470 /// Returns a vector_shuffle node for an unpackh operation.
4471 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4472                           SDValue V1, SDValue V2) {
4473   SmallVector<int, 8> Mask;
4474   createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
4475   return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4476 }
4477 
4478 /// Returns a node that packs the LHS + RHS nodes together at half width.
4479 /// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half.
4480 /// TODO: Add subvector splitting if/when we have a need for it.
4481 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4482                        const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
4483                        bool PackHiHalf = false) {
4484   MVT OpVT = LHS.getSimpleValueType();
4485   unsigned EltSizeInBits = VT.getScalarSizeInBits();
4486   bool UsePackUS = Subtarget.hasSSE41() || EltSizeInBits == 8;
4487   assert(OpVT == RHS.getSimpleValueType() &&
4488          VT.getSizeInBits() == OpVT.getSizeInBits() &&
4489          (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() &&
4490          "Unexpected PACK operand types");
4491   assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) &&
4492          "Unexpected PACK result type");
4493 
4494   // Rely on vector shuffles for vXi64 -> vXi32 packing.
4495   if (EltSizeInBits == 32) {
4496     SmallVector<int> PackMask;
4497     int Offset = PackHiHalf ? 1 : 0;
4498     int NumElts = VT.getVectorNumElements();
4499     for (int I = 0; I != NumElts; I += 4) {
4500       PackMask.push_back(I + Offset);
4501       PackMask.push_back(I + Offset + 2);
4502       PackMask.push_back(I + Offset + NumElts);
4503       PackMask.push_back(I + Offset + NumElts + 2);
4504     }
4505     return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
4506                                 DAG.getBitcast(VT, RHS), PackMask);
4507   }
4508 
4509   // See if we already have sufficient leading bits for PACKSS/PACKUS.
4510   if (!PackHiHalf) {
4511     if (UsePackUS &&
4512         DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits &&
4513         DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits)
4514       return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4515 
4516     if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits &&
4517         DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits)
4518       return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4519   }
4520 
4521   // Fallback to sign/zero extending the requested half and pack.
4522   SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
4523   if (UsePackUS) {
4524     if (PackHiHalf) {
4525       LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt);
4526       RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt);
4527     } else {
4528       SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
4529       LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask);
4530       RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask);
4531     };
4532     return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4533   };
4534 
4535   if (!PackHiHalf) {
4536     LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt);
4537     RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt);
4538   }
4539   LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt);
4540   RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt);
4541   return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4542 }
4543 
4544 /// Return a vector_shuffle of the specified vector of zero or undef vector.
4545 /// This produces a shuffle where the low element of V2 is swizzled into the
4546 /// zero/undef vector, landing at element Idx.
4547 /// This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
4548 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
4549                                            bool IsZero,
4550                                            const X86Subtarget &Subtarget,
4551                                            SelectionDAG &DAG) {
4552   MVT VT = V2.getSimpleValueType();
4553   SDValue V1 = IsZero
4554     ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
4555   int NumElems = VT.getVectorNumElements();
4556   SmallVector<int, 16> MaskVec(NumElems);
4557   for (int i = 0; i != NumElems; ++i)
4558     // If this is the insertion idx, put the low elt of V2 here.
4559     MaskVec[i] = (i == Idx) ? NumElems : i;
4560   return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
4561 }
4562 
4563 static ConstantPoolSDNode *getTargetConstantPoolFromBasePtr(SDValue Ptr) {
4564   if (Ptr.getOpcode() == X86ISD::Wrapper ||
4565       Ptr.getOpcode() == X86ISD::WrapperRIP)
4566     Ptr = Ptr.getOperand(0);
4567   return dyn_cast<ConstantPoolSDNode>(Ptr);
4568 }
4569 
4570 static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
4571   ConstantPoolSDNode *CNode = getTargetConstantPoolFromBasePtr(Ptr);
4572   if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0)
4573     return nullptr;
4574   return CNode->getConstVal();
4575 }
4576 
4577 static const Constant *getTargetConstantFromNode(LoadSDNode *Load) {
4578   if (!Load || !ISD::isNormalLoad(Load))
4579     return nullptr;
4580   return getTargetConstantFromBasePtr(Load->getBasePtr());
4581 }
4582 
4583 static const Constant *getTargetConstantFromNode(SDValue Op) {
4584   Op = peekThroughBitcasts(Op);
4585   return getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op));
4586 }
4587 
4588 const Constant *
4589 X86TargetLowering::getTargetConstantFromLoad(LoadSDNode *LD) const {
4590   assert(LD && "Unexpected null LoadSDNode");
4591   return getTargetConstantFromNode(LD);
4592 }
4593 
4594 // Extract raw constant bits from constant pools.
4595 static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
4596                                           APInt &UndefElts,
4597                                           SmallVectorImpl<APInt> &EltBits,
4598                                           bool AllowWholeUndefs = true,
4599                                           bool AllowPartialUndefs = true) {
4600   assert(EltBits.empty() && "Expected an empty EltBits vector");
4601 
4602   Op = peekThroughBitcasts(Op);
4603 
4604   EVT VT = Op.getValueType();
4605   unsigned SizeInBits = VT.getSizeInBits();
4606   assert((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!");
4607   unsigned NumElts = SizeInBits / EltSizeInBits;
4608 
4609   // Bitcast a source array of element bits to the target size.
4610   auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
4611     unsigned NumSrcElts = UndefSrcElts.getBitWidth();
4612     unsigned SrcEltSizeInBits = SrcEltBits[0].getBitWidth();
4613     assert((NumSrcElts * SrcEltSizeInBits) == SizeInBits &&
4614            "Constant bit sizes don't match");
4615 
4616     // Don't split if we don't allow undef bits.
4617     bool AllowUndefs = AllowWholeUndefs || AllowPartialUndefs;
4618     if (UndefSrcElts.getBoolValue() && !AllowUndefs)
4619       return false;
4620 
4621     // If we're already the right size, don't bother bitcasting.
4622     if (NumSrcElts == NumElts) {
4623       UndefElts = UndefSrcElts;
4624       EltBits.assign(SrcEltBits.begin(), SrcEltBits.end());
4625       return true;
4626     }
4627 
4628     // Extract all the undef/constant element data and pack into single bitsets.
4629     APInt UndefBits(SizeInBits, 0);
4630     APInt MaskBits(SizeInBits, 0);
4631 
4632     for (unsigned i = 0; i != NumSrcElts; ++i) {
4633       unsigned BitOffset = i * SrcEltSizeInBits;
4634       if (UndefSrcElts[i])
4635         UndefBits.setBits(BitOffset, BitOffset + SrcEltSizeInBits);
4636       MaskBits.insertBits(SrcEltBits[i], BitOffset);
4637     }
4638 
4639     // Split the undef/constant single bitset data into the target elements.
4640     UndefElts = APInt(NumElts, 0);
4641     EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
4642 
4643     for (unsigned i = 0; i != NumElts; ++i) {
4644       unsigned BitOffset = i * EltSizeInBits;
4645       APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
4646 
4647       // Only treat an element as UNDEF if all bits are UNDEF.
4648       if (UndefEltBits.isAllOnes()) {
4649         if (!AllowWholeUndefs)
4650           return false;
4651         UndefElts.setBit(i);
4652         continue;
4653       }
4654 
4655       // If only some bits are UNDEF then treat them as zero (or bail if not
4656       // supported).
4657       if (UndefEltBits.getBoolValue() && !AllowPartialUndefs)
4658         return false;
4659 
4660       EltBits[i] = MaskBits.extractBits(EltSizeInBits, BitOffset);
4661     }
4662     return true;
4663   };
4664 
4665   // Collect constant bits and insert into mask/undef bit masks.
4666   auto CollectConstantBits = [](const Constant *Cst, APInt &Mask, APInt &Undefs,
4667                                 unsigned UndefBitIndex) {
4668     if (!Cst)
4669       return false;
4670     if (isa<UndefValue>(Cst)) {
4671       Undefs.setBit(UndefBitIndex);
4672       return true;
4673     }
4674     if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4675       Mask = CInt->getValue();
4676       return true;
4677     }
4678     if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4679       Mask = CFP->getValueAPF().bitcastToAPInt();
4680       return true;
4681     }
4682     if (auto *CDS = dyn_cast<ConstantDataSequential>(Cst)) {
4683       Type *Ty = CDS->getType();
4684       Mask = APInt::getZero(Ty->getPrimitiveSizeInBits());
4685       Type *EltTy = CDS->getElementType();
4686       bool IsInteger = EltTy->isIntegerTy();
4687       bool IsFP =
4688           EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
4689       if (!IsInteger && !IsFP)
4690         return false;
4691       unsigned EltBits = EltTy->getPrimitiveSizeInBits();
4692       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I)
4693         if (IsInteger)
4694           Mask.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
4695         else
4696           Mask.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
4697                           I * EltBits);
4698       return true;
4699     }
4700     return false;
4701   };
4702 
4703   // Handle UNDEFs.
4704   if (Op.isUndef()) {
4705     APInt UndefSrcElts = APInt::getAllOnes(NumElts);
4706     SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0));
4707     return CastBitData(UndefSrcElts, SrcEltBits);
4708   }
4709 
4710   // Extract scalar constant bits.
4711   if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) {
4712     APInt UndefSrcElts = APInt::getZero(1);
4713     SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue());
4714     return CastBitData(UndefSrcElts, SrcEltBits);
4715   }
4716   if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
4717     APInt UndefSrcElts = APInt::getZero(1);
4718     APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
4719     SmallVector<APInt, 64> SrcEltBits(1, RawBits);
4720     return CastBitData(UndefSrcElts, SrcEltBits);
4721   }
4722 
4723   // Extract constant bits from build vector.
4724   if (auto *BV = dyn_cast<BuildVectorSDNode>(Op)) {
4725     BitVector Undefs;
4726     SmallVector<APInt> SrcEltBits;
4727     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4728     if (BV->getConstantRawBits(true, SrcEltSizeInBits, SrcEltBits, Undefs)) {
4729       APInt UndefSrcElts = APInt::getZero(SrcEltBits.size());
4730       for (unsigned I = 0, E = SrcEltBits.size(); I != E; ++I)
4731         if (Undefs[I])
4732           UndefSrcElts.setBit(I);
4733       return CastBitData(UndefSrcElts, SrcEltBits);
4734     }
4735   }
4736 
4737   // Extract constant bits from constant pool vector.
4738   if (auto *Cst = getTargetConstantFromNode(Op)) {
4739     Type *CstTy = Cst->getType();
4740     unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4741     if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0)
4742       return false;
4743 
4744     unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits();
4745     unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4746     if ((SizeInBits % SrcEltSizeInBits) != 0)
4747       return false;
4748 
4749     APInt UndefSrcElts(NumSrcElts, 0);
4750     SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
4751     for (unsigned i = 0; i != NumSrcElts; ++i)
4752       if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i],
4753                                UndefSrcElts, i))
4754         return false;
4755 
4756     return CastBitData(UndefSrcElts, SrcEltBits);
4757   }
4758 
4759   // Extract constant bits from a broadcasted constant pool scalar.
4760   if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
4761       EltSizeInBits <= VT.getScalarSizeInBits()) {
4762     auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4763     if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits())
4764       return false;
4765 
4766     SDValue Ptr = MemIntr->getBasePtr();
4767     if (const Constant *C = getTargetConstantFromBasePtr(Ptr)) {
4768       unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4769       unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4770 
4771       APInt UndefSrcElts(NumSrcElts, 0);
4772       SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
4773       if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
4774         if (UndefSrcElts[0])
4775           UndefSrcElts.setBits(0, NumSrcElts);
4776         if (SrcEltBits[0].getBitWidth() != SrcEltSizeInBits)
4777           SrcEltBits[0] = SrcEltBits[0].trunc(SrcEltSizeInBits);
4778         SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
4779         return CastBitData(UndefSrcElts, SrcEltBits);
4780       }
4781     }
4782   }
4783 
4784   // Extract constant bits from a subvector broadcast.
4785   if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
4786     auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4787     SDValue Ptr = MemIntr->getBasePtr();
4788     // The source constant may be larger than the subvector broadcast,
4789     // ensure we extract the correct subvector constants.
4790     if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) {
4791       Type *CstTy = Cst->getType();
4792       unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4793       unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits();
4794       if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 ||
4795           (SizeInBits % SubVecSizeInBits) != 0)
4796         return false;
4797       unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
4798       unsigned NumSubElts = SubVecSizeInBits / CstEltSizeInBits;
4799       unsigned NumSubVecs = SizeInBits / SubVecSizeInBits;
4800       APInt UndefSubElts(NumSubElts, 0);
4801       SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs,
4802                                         APInt(CstEltSizeInBits, 0));
4803       for (unsigned i = 0; i != NumSubElts; ++i) {
4804         if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i],
4805                                  UndefSubElts, i))
4806           return false;
4807         for (unsigned j = 1; j != NumSubVecs; ++j)
4808           SubEltBits[i + (j * NumSubElts)] = SubEltBits[i];
4809       }
4810       UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(),
4811                                      UndefSubElts);
4812       return CastBitData(UndefSubElts, SubEltBits);
4813     }
4814   }
4815 
4816   // Extract a rematerialized scalar constant insertion.
4817   if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
4818       Op.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
4819       isa<ConstantSDNode>(Op.getOperand(0).getOperand(0))) {
4820     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4821     unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4822 
4823     APInt UndefSrcElts(NumSrcElts, 0);
4824     SmallVector<APInt, 64> SrcEltBits;
4825     const APInt &C = Op.getOperand(0).getConstantOperandAPInt(0);
4826     SrcEltBits.push_back(C.zextOrTrunc(SrcEltSizeInBits));
4827     SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0));
4828     return CastBitData(UndefSrcElts, SrcEltBits);
4829   }
4830 
4831   // Insert constant bits from a base and sub vector sources.
4832   if (Op.getOpcode() == ISD::INSERT_SUBVECTOR) {
4833     // If bitcasts to larger elements we might lose track of undefs - don't
4834     // allow any to be safe.
4835     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4836     bool AllowUndefs = EltSizeInBits >= SrcEltSizeInBits;
4837 
4838     APInt UndefSrcElts, UndefSubElts;
4839     SmallVector<APInt, 32> EltSrcBits, EltSubBits;
4840     if (getTargetConstantBitsFromNode(Op.getOperand(1), SrcEltSizeInBits,
4841                                       UndefSubElts, EltSubBits,
4842                                       AllowWholeUndefs && AllowUndefs,
4843                                       AllowPartialUndefs && AllowUndefs) &&
4844         getTargetConstantBitsFromNode(Op.getOperand(0), SrcEltSizeInBits,
4845                                       UndefSrcElts, EltSrcBits,
4846                                       AllowWholeUndefs && AllowUndefs,
4847                                       AllowPartialUndefs && AllowUndefs)) {
4848       unsigned BaseIdx = Op.getConstantOperandVal(2);
4849       UndefSrcElts.insertBits(UndefSubElts, BaseIdx);
4850       for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i)
4851         EltSrcBits[BaseIdx + i] = EltSubBits[i];
4852       return CastBitData(UndefSrcElts, EltSrcBits);
4853     }
4854   }
4855 
4856   // Extract constant bits from a subvector's source.
4857   if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
4858     // TODO - support extract_subvector through bitcasts.
4859     if (EltSizeInBits != VT.getScalarSizeInBits())
4860       return false;
4861 
4862     if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
4863                                       UndefElts, EltBits, AllowWholeUndefs,
4864                                       AllowPartialUndefs)) {
4865       EVT SrcVT = Op.getOperand(0).getValueType();
4866       unsigned NumSrcElts = SrcVT.getVectorNumElements();
4867       unsigned NumSubElts = VT.getVectorNumElements();
4868       unsigned BaseIdx = Op.getConstantOperandVal(1);
4869       UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
4870       if ((BaseIdx + NumSubElts) != NumSrcElts)
4871         EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
4872       if (BaseIdx != 0)
4873         EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
4874       return true;
4875     }
4876   }
4877 
4878   // Extract constant bits from shuffle node sources.
4879   if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Op)) {
4880     // TODO - support shuffle through bitcasts.
4881     if (EltSizeInBits != VT.getScalarSizeInBits())
4882       return false;
4883 
4884     ArrayRef<int> Mask = SVN->getMask();
4885     if ((!AllowWholeUndefs || !AllowPartialUndefs) &&
4886         llvm::any_of(Mask, [](int M) { return M < 0; }))
4887       return false;
4888 
4889     APInt UndefElts0, UndefElts1;
4890     SmallVector<APInt, 32> EltBits0, EltBits1;
4891     if (isAnyInRange(Mask, 0, NumElts) &&
4892         !getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
4893                                        UndefElts0, EltBits0, AllowWholeUndefs,
4894                                        AllowPartialUndefs))
4895       return false;
4896     if (isAnyInRange(Mask, NumElts, 2 * NumElts) &&
4897         !getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
4898                                        UndefElts1, EltBits1, AllowWholeUndefs,
4899                                        AllowPartialUndefs))
4900       return false;
4901 
4902     UndefElts = APInt::getZero(NumElts);
4903     for (int i = 0; i != (int)NumElts; ++i) {
4904       int M = Mask[i];
4905       if (M < 0) {
4906         UndefElts.setBit(i);
4907         EltBits.push_back(APInt::getZero(EltSizeInBits));
4908       } else if (M < (int)NumElts) {
4909         if (UndefElts0[M])
4910           UndefElts.setBit(i);
4911         EltBits.push_back(EltBits0[M]);
4912       } else {
4913         if (UndefElts1[M - NumElts])
4914           UndefElts.setBit(i);
4915         EltBits.push_back(EltBits1[M - NumElts]);
4916       }
4917     }
4918     return true;
4919   }
4920 
4921   return false;
4922 }
4923 
4924 namespace llvm {
4925 namespace X86 {
4926 bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
4927   APInt UndefElts;
4928   SmallVector<APInt, 16> EltBits;
4929   if (getTargetConstantBitsFromNode(Op, Op.getScalarValueSizeInBits(),
4930                                     UndefElts, EltBits, true,
4931                                     AllowPartialUndefs)) {
4932     int SplatIndex = -1;
4933     for (int i = 0, e = EltBits.size(); i != e; ++i) {
4934       if (UndefElts[i])
4935         continue;
4936       if (0 <= SplatIndex && EltBits[i] != EltBits[SplatIndex]) {
4937         SplatIndex = -1;
4938         break;
4939       }
4940       SplatIndex = i;
4941     }
4942     if (0 <= SplatIndex) {
4943       SplatVal = EltBits[SplatIndex];
4944       return true;
4945     }
4946   }
4947 
4948   return false;
4949 }
4950 } // namespace X86
4951 } // namespace llvm
4952 
4953 static bool getTargetShuffleMaskIndices(SDValue MaskNode,
4954                                         unsigned MaskEltSizeInBits,
4955                                         SmallVectorImpl<uint64_t> &RawMask,
4956                                         APInt &UndefElts) {
4957   // Extract the raw target constant bits.
4958   SmallVector<APInt, 64> EltBits;
4959   if (!getTargetConstantBitsFromNode(MaskNode, MaskEltSizeInBits, UndefElts,
4960                                      EltBits, /* AllowWholeUndefs */ true,
4961                                      /* AllowPartialUndefs */ false))
4962     return false;
4963 
4964   // Insert the extracted elements into the mask.
4965   for (const APInt &Elt : EltBits)
4966     RawMask.push_back(Elt.getZExtValue());
4967 
4968   return true;
4969 }
4970 
4971 // Match not(xor X, -1) -> X.
4972 // Match not(pcmpgt(C, X)) -> pcmpgt(X, C - 1).
4973 // Match not(extract_subvector(xor X, -1)) -> extract_subvector(X).
4974 // Match not(concat_vectors(xor X, -1, xor Y, -1)) -> concat_vectors(X, Y).
4975 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
4976   V = peekThroughBitcasts(V);
4977   if (V.getOpcode() == ISD::XOR &&
4978       (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) ||
4979        isAllOnesConstant(V.getOperand(1))))
4980     return V.getOperand(0);
4981   if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4982       (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
4983     if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
4984       Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
4985       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(),
4986                          Not, V.getOperand(1));
4987     }
4988   }
4989   if (V.getOpcode() == X86ISD::PCMPGT &&
4990       !ISD::isBuildVectorAllZeros(V.getOperand(0).getNode()) &&
4991       !ISD::isBuildVectorAllOnes(V.getOperand(0).getNode()) &&
4992       V.getOperand(0).hasOneUse()) {
4993     APInt UndefElts;
4994     SmallVector<APInt> EltBits;
4995     if (getTargetConstantBitsFromNode(V.getOperand(0),
4996                                       V.getScalarValueSizeInBits(), UndefElts,
4997                                       EltBits)) {
4998       // Don't fold min_signed_value -> (min_signed_value - 1)
4999       bool MinSigned = false;
5000       for (APInt &Elt : EltBits) {
5001         MinSigned |= Elt.isMinSignedValue();
5002         Elt -= 1;
5003       }
5004       if (!MinSigned) {
5005         SDLoc DL(V);
5006         MVT VT = V.getSimpleValueType();
5007         return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1),
5008                            getConstVector(EltBits, UndefElts, VT, DAG, DL));
5009       }
5010     }
5011   }
5012   SmallVector<SDValue, 2> CatOps;
5013   if (collectConcatOps(V.getNode(), CatOps, DAG)) {
5014     for (SDValue &CatOp : CatOps) {
5015       SDValue NotCat = IsNOT(CatOp, DAG);
5016       if (!NotCat) return SDValue();
5017       CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
5018     }
5019     return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps);
5020   }
5021   return SDValue();
5022 }
5023 
5024 /// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
5025 /// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
5026 /// Note: This ignores saturation, so inputs must be checked first.
5027 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
5028                                   bool Unary, unsigned NumStages = 1) {
5029   assert(Mask.empty() && "Expected an empty shuffle mask vector");
5030   unsigned NumElts = VT.getVectorNumElements();
5031   unsigned NumLanes = VT.getSizeInBits() / 128;
5032   unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
5033   unsigned Offset = Unary ? 0 : NumElts;
5034   unsigned Repetitions = 1u << (NumStages - 1);
5035   unsigned Increment = 1u << NumStages;
5036   assert((NumEltsPerLane >> NumStages) > 0 && "Illegal packing compaction");
5037 
5038   for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
5039     for (unsigned Stage = 0; Stage != Repetitions; ++Stage) {
5040       for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5041         Mask.push_back(Elt + (Lane * NumEltsPerLane));
5042       for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5043         Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
5044     }
5045   }
5046 }
5047 
5048 // Split the demanded elts of a PACKSS/PACKUS node between its operands.
5049 static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
5050                                 APInt &DemandedLHS, APInt &DemandedRHS) {
5051   int NumLanes = VT.getSizeInBits() / 128;
5052   int NumElts = DemandedElts.getBitWidth();
5053   int NumInnerElts = NumElts / 2;
5054   int NumEltsPerLane = NumElts / NumLanes;
5055   int NumInnerEltsPerLane = NumInnerElts / NumLanes;
5056 
5057   DemandedLHS = APInt::getZero(NumInnerElts);
5058   DemandedRHS = APInt::getZero(NumInnerElts);
5059 
5060   // Map DemandedElts to the packed operands.
5061   for (int Lane = 0; Lane != NumLanes; ++Lane) {
5062     for (int Elt = 0; Elt != NumInnerEltsPerLane; ++Elt) {
5063       int OuterIdx = (Lane * NumEltsPerLane) + Elt;
5064       int InnerIdx = (Lane * NumInnerEltsPerLane) + Elt;
5065       if (DemandedElts[OuterIdx])
5066         DemandedLHS.setBit(InnerIdx);
5067       if (DemandedElts[OuterIdx + NumInnerEltsPerLane])
5068         DemandedRHS.setBit(InnerIdx);
5069     }
5070   }
5071 }
5072 
5073 // Split the demanded elts of a HADD/HSUB node between its operands.
5074 static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
5075                                  APInt &DemandedLHS, APInt &DemandedRHS) {
5076   int NumLanes = VT.getSizeInBits() / 128;
5077   int NumElts = DemandedElts.getBitWidth();
5078   int NumEltsPerLane = NumElts / NumLanes;
5079   int HalfEltsPerLane = NumEltsPerLane / 2;
5080 
5081   DemandedLHS = APInt::getZero(NumElts);
5082   DemandedRHS = APInt::getZero(NumElts);
5083 
5084   // Map DemandedElts to the horizontal operands.
5085   for (int Idx = 0; Idx != NumElts; ++Idx) {
5086     if (!DemandedElts[Idx])
5087       continue;
5088     int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
5089     int LocalIdx = Idx % NumEltsPerLane;
5090     if (LocalIdx < HalfEltsPerLane) {
5091       DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5092       DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5093     } else {
5094       LocalIdx -= HalfEltsPerLane;
5095       DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5096       DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5097     }
5098   }
5099 }
5100 
5101 /// Calculates the shuffle mask corresponding to the target-specific opcode.
5102 /// If the mask could be calculated, returns it in \p Mask, returns the shuffle
5103 /// operands in \p Ops, and returns true.
5104 /// Sets \p IsUnary to true if only one source is used. Note that this will set
5105 /// IsUnary for shuffles which use a single input multiple times, and in those
5106 /// cases it will adjust the mask to only have indices within that single input.
5107 /// It is an error to call this with non-empty Mask/Ops vectors.
5108 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
5109                                  SmallVectorImpl<SDValue> &Ops,
5110                                  SmallVectorImpl<int> &Mask, bool &IsUnary) {
5111   unsigned NumElems = VT.getVectorNumElements();
5112   unsigned MaskEltSize = VT.getScalarSizeInBits();
5113   SmallVector<uint64_t, 32> RawMask;
5114   APInt RawUndefs;
5115   uint64_t ImmN;
5116 
5117   assert(Mask.empty() && "getTargetShuffleMask expects an empty Mask vector");
5118   assert(Ops.empty() && "getTargetShuffleMask expects an empty Ops vector");
5119 
5120   IsUnary = false;
5121   bool IsFakeUnary = false;
5122   switch (N->getOpcode()) {
5123   case X86ISD::BLENDI:
5124     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5125     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5126     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5127     DecodeBLENDMask(NumElems, ImmN, Mask);
5128     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5129     break;
5130   case X86ISD::SHUFP:
5131     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5132     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5133     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5134     DecodeSHUFPMask(NumElems, MaskEltSize, ImmN, Mask);
5135     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5136     break;
5137   case X86ISD::INSERTPS:
5138     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5139     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5140     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5141     DecodeINSERTPSMask(ImmN, Mask);
5142     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5143     break;
5144   case X86ISD::EXTRQI:
5145     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5146     if (isa<ConstantSDNode>(N->getOperand(1)) &&
5147         isa<ConstantSDNode>(N->getOperand(2))) {
5148       int BitLen = N->getConstantOperandVal(1);
5149       int BitIdx = N->getConstantOperandVal(2);
5150       DecodeEXTRQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5151       IsUnary = true;
5152     }
5153     break;
5154   case X86ISD::INSERTQI:
5155     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5156     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5157     if (isa<ConstantSDNode>(N->getOperand(2)) &&
5158         isa<ConstantSDNode>(N->getOperand(3))) {
5159       int BitLen = N->getConstantOperandVal(2);
5160       int BitIdx = N->getConstantOperandVal(3);
5161       DecodeINSERTQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5162       IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5163     }
5164     break;
5165   case X86ISD::UNPCKH:
5166     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5167     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5168     DecodeUNPCKHMask(NumElems, MaskEltSize, Mask);
5169     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5170     break;
5171   case X86ISD::UNPCKL:
5172     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5173     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5174     DecodeUNPCKLMask(NumElems, MaskEltSize, Mask);
5175     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5176     break;
5177   case X86ISD::MOVHLPS:
5178     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5179     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5180     DecodeMOVHLPSMask(NumElems, Mask);
5181     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5182     break;
5183   case X86ISD::MOVLHPS:
5184     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5185     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5186     DecodeMOVLHPSMask(NumElems, Mask);
5187     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5188     break;
5189   case X86ISD::VALIGN:
5190     assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
5191            "Only 32-bit and 64-bit elements are supported!");
5192     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5193     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5194     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5195     DecodeVALIGNMask(NumElems, ImmN, Mask);
5196     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5197     Ops.push_back(N->getOperand(1));
5198     Ops.push_back(N->getOperand(0));
5199     break;
5200   case X86ISD::PALIGNR:
5201     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5202     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5203     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5204     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5205     DecodePALIGNRMask(NumElems, ImmN, Mask);
5206     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5207     Ops.push_back(N->getOperand(1));
5208     Ops.push_back(N->getOperand(0));
5209     break;
5210   case X86ISD::VSHLDQ:
5211     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5212     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5213     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5214     DecodePSLLDQMask(NumElems, ImmN, Mask);
5215     IsUnary = true;
5216     break;
5217   case X86ISD::VSRLDQ:
5218     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5219     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5220     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5221     DecodePSRLDQMask(NumElems, ImmN, Mask);
5222     IsUnary = true;
5223     break;
5224   case X86ISD::PSHUFD:
5225   case X86ISD::VPERMILPI:
5226     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5227     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5228     DecodePSHUFMask(NumElems, MaskEltSize, ImmN, Mask);
5229     IsUnary = true;
5230     break;
5231   case X86ISD::PSHUFHW:
5232     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5233     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5234     DecodePSHUFHWMask(NumElems, ImmN, Mask);
5235     IsUnary = true;
5236     break;
5237   case X86ISD::PSHUFLW:
5238     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5239     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5240     DecodePSHUFLWMask(NumElems, ImmN, Mask);
5241     IsUnary = true;
5242     break;
5243   case X86ISD::VZEXT_MOVL:
5244     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5245     DecodeZeroMoveLowMask(NumElems, Mask);
5246     IsUnary = true;
5247     break;
5248   case X86ISD::VBROADCAST:
5249     // We only decode broadcasts of same-sized vectors, peeking through to
5250     // extracted subvectors is likely to cause hasOneUse issues with
5251     // SimplifyDemandedBits etc.
5252     if (N->getOperand(0).getValueType() == VT) {
5253       DecodeVectorBroadcast(NumElems, Mask);
5254       IsUnary = true;
5255       break;
5256     }
5257     return false;
5258   case X86ISD::VPERMILPV: {
5259     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5260     IsUnary = true;
5261     SDValue MaskNode = N->getOperand(1);
5262     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5263                                     RawUndefs)) {
5264       DecodeVPERMILPMask(NumElems, MaskEltSize, RawMask, RawUndefs, Mask);
5265       break;
5266     }
5267     return false;
5268   }
5269   case X86ISD::PSHUFB: {
5270     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5271     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5272     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5273     IsUnary = true;
5274     SDValue MaskNode = N->getOperand(1);
5275     if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5276       DecodePSHUFBMask(RawMask, RawUndefs, Mask);
5277       break;
5278     }
5279     return false;
5280   }
5281   case X86ISD::VPERMI:
5282     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5283     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5284     DecodeVPERMMask(NumElems, ImmN, Mask);
5285     IsUnary = true;
5286     break;
5287   case X86ISD::MOVSS:
5288   case X86ISD::MOVSD:
5289   case X86ISD::MOVSH:
5290     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5291     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5292     DecodeScalarMoveMask(NumElems, /* IsLoad */ false, Mask);
5293     break;
5294   case X86ISD::VPERM2X128:
5295     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5296     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5297     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5298     DecodeVPERM2X128Mask(NumElems, ImmN, Mask);
5299     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5300     break;
5301   case X86ISD::SHUF128:
5302     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5303     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5304     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5305     decodeVSHUF64x2FamilyMask(NumElems, MaskEltSize, ImmN, Mask);
5306     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5307     break;
5308   case X86ISD::MOVSLDUP:
5309     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5310     DecodeMOVSLDUPMask(NumElems, Mask);
5311     IsUnary = true;
5312     break;
5313   case X86ISD::MOVSHDUP:
5314     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5315     DecodeMOVSHDUPMask(NumElems, Mask);
5316     IsUnary = true;
5317     break;
5318   case X86ISD::MOVDDUP:
5319     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5320     DecodeMOVDDUPMask(NumElems, Mask);
5321     IsUnary = true;
5322     break;
5323   case X86ISD::VPERMIL2: {
5324     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5325     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5326     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5327     SDValue MaskNode = N->getOperand(2);
5328     SDValue CtrlNode = N->getOperand(3);
5329     if (ConstantSDNode *CtrlOp = dyn_cast<ConstantSDNode>(CtrlNode)) {
5330       unsigned CtrlImm = CtrlOp->getZExtValue();
5331       if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5332                                       RawUndefs)) {
5333         DecodeVPERMIL2PMask(NumElems, MaskEltSize, CtrlImm, RawMask, RawUndefs,
5334                             Mask);
5335         break;
5336       }
5337     }
5338     return false;
5339   }
5340   case X86ISD::VPPERM: {
5341     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5342     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5343     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5344     SDValue MaskNode = N->getOperand(2);
5345     if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5346       DecodeVPPERMMask(RawMask, RawUndefs, Mask);
5347       break;
5348     }
5349     return false;
5350   }
5351   case X86ISD::VPERMV: {
5352     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5353     IsUnary = true;
5354     // Unlike most shuffle nodes, VPERMV's mask operand is operand 0.
5355     Ops.push_back(N->getOperand(1));
5356     SDValue MaskNode = N->getOperand(0);
5357     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5358                                     RawUndefs)) {
5359       DecodeVPERMVMask(RawMask, RawUndefs, Mask);
5360       break;
5361     }
5362     return false;
5363   }
5364   case X86ISD::VPERMV3: {
5365     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5366     assert(N->getOperand(2).getValueType() == VT && "Unexpected value type");
5367     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(2);
5368     // Unlike most shuffle nodes, VPERMV3's mask operand is the middle one.
5369     Ops.push_back(N->getOperand(0));
5370     Ops.push_back(N->getOperand(2));
5371     SDValue MaskNode = N->getOperand(1);
5372     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5373                                     RawUndefs)) {
5374       DecodeVPERMV3Mask(RawMask, RawUndefs, Mask);
5375       break;
5376     }
5377     return false;
5378   }
5379   default: llvm_unreachable("unknown target shuffle node");
5380   }
5381 
5382   // Empty mask indicates the decode failed.
5383   if (Mask.empty())
5384     return false;
5385 
5386   // Check if we're getting a shuffle mask with zero'd elements.
5387   if (!AllowSentinelZero && isAnyZero(Mask))
5388     return false;
5389 
5390   // If we have a fake unary shuffle, the shuffle mask is spread across two
5391   // inputs that are actually the same node. Re-map the mask to always point
5392   // into the first input.
5393   if (IsFakeUnary)
5394     for (int &M : Mask)
5395       if (M >= (int)Mask.size())
5396         M -= Mask.size();
5397 
5398   // If we didn't already add operands in the opcode-specific code, default to
5399   // adding 1 or 2 operands starting at 0.
5400   if (Ops.empty()) {
5401     Ops.push_back(N->getOperand(0));
5402     if (!IsUnary || IsFakeUnary)
5403       Ops.push_back(N->getOperand(1));
5404   }
5405 
5406   return true;
5407 }
5408 
5409 // Wrapper for getTargetShuffleMask with InUnary;
5410 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
5411                                  SmallVectorImpl<SDValue> &Ops,
5412                                  SmallVectorImpl<int> &Mask) {
5413   bool IsUnary;
5414   return getTargetShuffleMask(N, VT, AllowSentinelZero, Ops, Mask, IsUnary);
5415 }
5416 
5417 /// Compute whether each element of a shuffle is zeroable.
5418 ///
5419 /// A "zeroable" vector shuffle element is one which can be lowered to zero.
5420 /// Either it is an undef element in the shuffle mask, the element of the input
5421 /// referenced is undef, or the element of the input referenced is known to be
5422 /// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
5423 /// as many lanes with this technique as possible to simplify the remaining
5424 /// shuffle.
5425 static void computeZeroableShuffleElements(ArrayRef<int> Mask,
5426                                            SDValue V1, SDValue V2,
5427                                            APInt &KnownUndef, APInt &KnownZero) {
5428   int Size = Mask.size();
5429   KnownUndef = KnownZero = APInt::getZero(Size);
5430 
5431   V1 = peekThroughBitcasts(V1);
5432   V2 = peekThroughBitcasts(V2);
5433 
5434   bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
5435   bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
5436 
5437   int VectorSizeInBits = V1.getValueSizeInBits();
5438   int ScalarSizeInBits = VectorSizeInBits / Size;
5439   assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
5440 
5441   for (int i = 0; i < Size; ++i) {
5442     int M = Mask[i];
5443     // Handle the easy cases.
5444     if (M < 0) {
5445       KnownUndef.setBit(i);
5446       continue;
5447     }
5448     if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
5449       KnownZero.setBit(i);
5450       continue;
5451     }
5452 
5453     // Determine shuffle input and normalize the mask.
5454     SDValue V = M < Size ? V1 : V2;
5455     M %= Size;
5456 
5457     // Currently we can only search BUILD_VECTOR for UNDEF/ZERO elements.
5458     if (V.getOpcode() != ISD::BUILD_VECTOR)
5459       continue;
5460 
5461     // If the BUILD_VECTOR has fewer elements then the bitcasted portion of
5462     // the (larger) source element must be UNDEF/ZERO.
5463     if ((Size % V.getNumOperands()) == 0) {
5464       int Scale = Size / V->getNumOperands();
5465       SDValue Op = V.getOperand(M / Scale);
5466       if (Op.isUndef())
5467         KnownUndef.setBit(i);
5468       if (X86::isZeroNode(Op))
5469         KnownZero.setBit(i);
5470       else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
5471         APInt Val = Cst->getAPIntValue();
5472         Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5473         if (Val == 0)
5474           KnownZero.setBit(i);
5475       } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
5476         APInt Val = Cst->getValueAPF().bitcastToAPInt();
5477         Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5478         if (Val == 0)
5479           KnownZero.setBit(i);
5480       }
5481       continue;
5482     }
5483 
5484     // If the BUILD_VECTOR has more elements then all the (smaller) source
5485     // elements must be UNDEF or ZERO.
5486     if ((V.getNumOperands() % Size) == 0) {
5487       int Scale = V->getNumOperands() / Size;
5488       bool AllUndef = true;
5489       bool AllZero = true;
5490       for (int j = 0; j < Scale; ++j) {
5491         SDValue Op = V.getOperand((M * Scale) + j);
5492         AllUndef &= Op.isUndef();
5493         AllZero &= X86::isZeroNode(Op);
5494       }
5495       if (AllUndef)
5496         KnownUndef.setBit(i);
5497       if (AllZero)
5498         KnownZero.setBit(i);
5499       continue;
5500     }
5501   }
5502 }
5503 
5504 /// Decode a target shuffle mask and inputs and see if any values are
5505 /// known to be undef or zero from their inputs.
5506 /// Returns true if the target shuffle mask was decoded.
5507 /// FIXME: Merge this with computeZeroableShuffleElements?
5508 static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
5509                                          SmallVectorImpl<SDValue> &Ops,
5510                                          APInt &KnownUndef, APInt &KnownZero) {
5511   bool IsUnary;
5512   if (!isTargetShuffle(N.getOpcode()))
5513     return false;
5514 
5515   MVT VT = N.getSimpleValueType();
5516   if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary))
5517     return false;
5518 
5519   int Size = Mask.size();
5520   SDValue V1 = Ops[0];
5521   SDValue V2 = IsUnary ? V1 : Ops[1];
5522   KnownUndef = KnownZero = APInt::getZero(Size);
5523 
5524   V1 = peekThroughBitcasts(V1);
5525   V2 = peekThroughBitcasts(V2);
5526 
5527   assert((VT.getSizeInBits() % Size) == 0 &&
5528          "Illegal split of shuffle value type");
5529   unsigned EltSizeInBits = VT.getSizeInBits() / Size;
5530 
5531   // Extract known constant input data.
5532   APInt UndefSrcElts[2];
5533   SmallVector<APInt, 32> SrcEltBits[2];
5534   bool IsSrcConstant[2] = {
5535       getTargetConstantBitsFromNode(V1, EltSizeInBits, UndefSrcElts[0],
5536                                     SrcEltBits[0], true, false),
5537       getTargetConstantBitsFromNode(V2, EltSizeInBits, UndefSrcElts[1],
5538                                     SrcEltBits[1], true, false)};
5539 
5540   for (int i = 0; i < Size; ++i) {
5541     int M = Mask[i];
5542 
5543     // Already decoded as SM_SentinelZero / SM_SentinelUndef.
5544     if (M < 0) {
5545       assert(isUndefOrZero(M) && "Unknown shuffle sentinel value!");
5546       if (SM_SentinelUndef == M)
5547         KnownUndef.setBit(i);
5548       if (SM_SentinelZero == M)
5549         KnownZero.setBit(i);
5550       continue;
5551     }
5552 
5553     // Determine shuffle input and normalize the mask.
5554     unsigned SrcIdx = M / Size;
5555     SDValue V = M < Size ? V1 : V2;
5556     M %= Size;
5557 
5558     // We are referencing an UNDEF input.
5559     if (V.isUndef()) {
5560       KnownUndef.setBit(i);
5561       continue;
5562     }
5563 
5564     // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF.
5565     // TODO: We currently only set UNDEF for integer types - floats use the same
5566     // registers as vectors and many of the scalar folded loads rely on the
5567     // SCALAR_TO_VECTOR pattern.
5568     if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
5569         (Size % V.getValueType().getVectorNumElements()) == 0) {
5570       int Scale = Size / V.getValueType().getVectorNumElements();
5571       int Idx = M / Scale;
5572       if (Idx != 0 && !VT.isFloatingPoint())
5573         KnownUndef.setBit(i);
5574       else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
5575         KnownZero.setBit(i);
5576       continue;
5577     }
5578 
5579     // INSERT_SUBVECTOR - to widen vectors we often insert them into UNDEF
5580     // base vectors.
5581     if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
5582       SDValue Vec = V.getOperand(0);
5583       int NumVecElts = Vec.getValueType().getVectorNumElements();
5584       if (Vec.isUndef() && Size == NumVecElts) {
5585         int Idx = V.getConstantOperandVal(2);
5586         int NumSubElts = V.getOperand(1).getValueType().getVectorNumElements();
5587         if (M < Idx || (Idx + NumSubElts) <= M)
5588           KnownUndef.setBit(i);
5589       }
5590       continue;
5591     }
5592 
5593     // Attempt to extract from the source's constant bits.
5594     if (IsSrcConstant[SrcIdx]) {
5595       if (UndefSrcElts[SrcIdx][M])
5596         KnownUndef.setBit(i);
5597       else if (SrcEltBits[SrcIdx][M] == 0)
5598         KnownZero.setBit(i);
5599     }
5600   }
5601 
5602   assert(VT.getVectorNumElements() == (unsigned)Size &&
5603          "Different mask size from vector size!");
5604   return true;
5605 }
5606 
5607 // Replace target shuffle mask elements with known undef/zero sentinels.
5608 static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
5609                                               const APInt &KnownUndef,
5610                                               const APInt &KnownZero,
5611                                               bool ResolveKnownZeros= true) {
5612   unsigned NumElts = Mask.size();
5613   assert(KnownUndef.getBitWidth() == NumElts &&
5614          KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
5615 
5616   for (unsigned i = 0; i != NumElts; ++i) {
5617     if (KnownUndef[i])
5618       Mask[i] = SM_SentinelUndef;
5619     else if (ResolveKnownZeros && KnownZero[i])
5620       Mask[i] = SM_SentinelZero;
5621   }
5622 }
5623 
5624 // Extract target shuffle mask sentinel elements to known undef/zero bitmasks.
5625 static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl<int> &Mask,
5626                                               APInt &KnownUndef,
5627                                               APInt &KnownZero) {
5628   unsigned NumElts = Mask.size();
5629   KnownUndef = KnownZero = APInt::getZero(NumElts);
5630 
5631   for (unsigned i = 0; i != NumElts; ++i) {
5632     int M = Mask[i];
5633     if (SM_SentinelUndef == M)
5634       KnownUndef.setBit(i);
5635     if (SM_SentinelZero == M)
5636       KnownZero.setBit(i);
5637   }
5638 }
5639 
5640 // Attempt to create a shuffle mask from a VSELECT/BLENDV condition mask.
5641 static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
5642                                          SDValue Cond, bool IsBLENDV = false) {
5643   EVT CondVT = Cond.getValueType();
5644   unsigned EltSizeInBits = CondVT.getScalarSizeInBits();
5645   unsigned NumElts = CondVT.getVectorNumElements();
5646 
5647   APInt UndefElts;
5648   SmallVector<APInt, 32> EltBits;
5649   if (!getTargetConstantBitsFromNode(Cond, EltSizeInBits, UndefElts, EltBits,
5650                                      true, false))
5651     return false;
5652 
5653   Mask.resize(NumElts, SM_SentinelUndef);
5654 
5655   for (int i = 0; i != (int)NumElts; ++i) {
5656     Mask[i] = i;
5657     // Arbitrarily choose from the 2nd operand if the select condition element
5658     // is undef.
5659     // TODO: Can we do better by matching patterns such as even/odd?
5660     if (UndefElts[i] || (!IsBLENDV && EltBits[i].isZero()) ||
5661         (IsBLENDV && EltBits[i].isNonNegative()))
5662       Mask[i] += NumElts;
5663   }
5664 
5665   return true;
5666 }
5667 
5668 // Forward declaration (for getFauxShuffleMask recursive check).
5669 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
5670                                    SmallVectorImpl<SDValue> &Inputs,
5671                                    SmallVectorImpl<int> &Mask,
5672                                    const SelectionDAG &DAG, unsigned Depth,
5673                                    bool ResolveKnownElts);
5674 
5675 // Attempt to decode ops that could be represented as a shuffle mask.
5676 // The decoded shuffle mask may contain a different number of elements to the
5677 // destination value type.
5678 // TODO: Merge into getTargetShuffleInputs()
5679 static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
5680                                SmallVectorImpl<int> &Mask,
5681                                SmallVectorImpl<SDValue> &Ops,
5682                                const SelectionDAG &DAG, unsigned Depth,
5683                                bool ResolveKnownElts) {
5684   Mask.clear();
5685   Ops.clear();
5686 
5687   MVT VT = N.getSimpleValueType();
5688   unsigned NumElts = VT.getVectorNumElements();
5689   unsigned NumSizeInBits = VT.getSizeInBits();
5690   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
5691   if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
5692     return false;
5693   assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size");
5694   unsigned NumSizeInBytes = NumSizeInBits / 8;
5695   unsigned NumBytesPerElt = NumBitsPerElt / 8;
5696 
5697   unsigned Opcode = N.getOpcode();
5698   switch (Opcode) {
5699   case ISD::VECTOR_SHUFFLE: {
5700     // Don't treat ISD::VECTOR_SHUFFLE as a target shuffle so decode it here.
5701     ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask();
5702     if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) {
5703       Mask.append(ShuffleMask.begin(), ShuffleMask.end());
5704       Ops.push_back(N.getOperand(0));
5705       Ops.push_back(N.getOperand(1));
5706       return true;
5707     }
5708     return false;
5709   }
5710   case ISD::AND:
5711   case X86ISD::ANDNP: {
5712     // Attempt to decode as a per-byte mask.
5713     APInt UndefElts;
5714     SmallVector<APInt, 32> EltBits;
5715     SDValue N0 = N.getOperand(0);
5716     SDValue N1 = N.getOperand(1);
5717     bool IsAndN = (X86ISD::ANDNP == Opcode);
5718     uint64_t ZeroMask = IsAndN ? 255 : 0;
5719     if (!getTargetConstantBitsFromNode(IsAndN ? N0 : N1, 8, UndefElts, EltBits))
5720       return false;
5721     // We can't assume an undef src element gives an undef dst - the other src
5722     // might be zero.
5723     if (!UndefElts.isZero())
5724       return false;
5725     for (int i = 0, e = (int)EltBits.size(); i != e; ++i) {
5726       const APInt &ByteBits = EltBits[i];
5727       if (ByteBits != 0 && ByteBits != 255)
5728         return false;
5729       Mask.push_back(ByteBits == ZeroMask ? SM_SentinelZero : i);
5730     }
5731     Ops.push_back(IsAndN ? N1 : N0);
5732     return true;
5733   }
5734   case ISD::OR: {
5735     // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
5736     // is a valid shuffle index.
5737     SDValue N0 = peekThroughBitcasts(N.getOperand(0));
5738     SDValue N1 = peekThroughBitcasts(N.getOperand(1));
5739     if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
5740       return false;
5741 
5742     SmallVector<int, 64> SrcMask0, SrcMask1;
5743     SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
5744     APInt Demand0 = APInt::getAllOnes(N0.getValueType().getVectorNumElements());
5745     APInt Demand1 = APInt::getAllOnes(N1.getValueType().getVectorNumElements());
5746     if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG,
5747                                 Depth + 1, true) ||
5748         !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG,
5749                                 Depth + 1, true))
5750       return false;
5751 
5752     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
5753     SmallVector<int, 64> Mask0, Mask1;
5754     narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
5755     narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
5756     for (int i = 0; i != (int)MaskSize; ++i) {
5757       // NOTE: Don't handle SM_SentinelUndef, as we can end up in infinite
5758       // loops converting between OR and BLEND shuffles due to
5759       // canWidenShuffleElements merging away undef elements, meaning we
5760       // fail to recognise the OR as the undef element isn't known zero.
5761       if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
5762         Mask.push_back(SM_SentinelZero);
5763       else if (Mask1[i] == SM_SentinelZero)
5764         Mask.push_back(i);
5765       else if (Mask0[i] == SM_SentinelZero)
5766         Mask.push_back(i + MaskSize);
5767       else
5768         return false;
5769     }
5770     Ops.push_back(N0);
5771     Ops.push_back(N1);
5772     return true;
5773   }
5774   case ISD::INSERT_SUBVECTOR: {
5775     SDValue Src = N.getOperand(0);
5776     SDValue Sub = N.getOperand(1);
5777     EVT SubVT = Sub.getValueType();
5778     unsigned NumSubElts = SubVT.getVectorNumElements();
5779     if (!N->isOnlyUserOf(Sub.getNode()))
5780       return false;
5781     SDValue SubBC = peekThroughBitcasts(Sub);
5782     uint64_t InsertIdx = N.getConstantOperandVal(2);
5783     // Handle INSERT_SUBVECTOR(SRC0, EXTRACT_SUBVECTOR(SRC1)).
5784     if (SubBC.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5785         SubBC.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
5786       uint64_t ExtractIdx = SubBC.getConstantOperandVal(1);
5787       SDValue SubBCSrc = SubBC.getOperand(0);
5788       unsigned NumSubSrcBCElts = SubBCSrc.getValueType().getVectorNumElements();
5789       unsigned MaxElts = std::max(NumElts, NumSubSrcBCElts);
5790       assert((MaxElts % NumElts) == 0 && (MaxElts % NumSubSrcBCElts) == 0 &&
5791              "Subvector valuetype mismatch");
5792       InsertIdx *= (MaxElts / NumElts);
5793       ExtractIdx *= (MaxElts / NumSubSrcBCElts);
5794       NumSubElts *= (MaxElts / NumElts);
5795       bool SrcIsUndef = Src.isUndef();
5796       for (int i = 0; i != (int)MaxElts; ++i)
5797         Mask.push_back(SrcIsUndef ? SM_SentinelUndef : i);
5798       for (int i = 0; i != (int)NumSubElts; ++i)
5799         Mask[InsertIdx + i] = (SrcIsUndef ? 0 : MaxElts) + ExtractIdx + i;
5800       if (!SrcIsUndef)
5801         Ops.push_back(Src);
5802       Ops.push_back(SubBCSrc);
5803       return true;
5804     }
5805     // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
5806     SmallVector<int, 64> SubMask;
5807     SmallVector<SDValue, 2> SubInputs;
5808     SDValue SubSrc = peekThroughOneUseBitcasts(Sub);
5809     EVT SubSrcVT = SubSrc.getValueType();
5810     if (!SubSrcVT.isVector())
5811       return false;
5812 
5813     APInt SubDemand = APInt::getAllOnes(SubSrcVT.getVectorNumElements());
5814     if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG,
5815                                 Depth + 1, ResolveKnownElts))
5816       return false;
5817 
5818     // Subvector shuffle inputs must not be larger than the subvector.
5819     if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
5820           return SubVT.getFixedSizeInBits() <
5821                  SubInput.getValueSizeInBits().getFixedValue();
5822         }))
5823       return false;
5824 
5825     if (SubMask.size() != NumSubElts) {
5826       assert(((SubMask.size() % NumSubElts) == 0 ||
5827               (NumSubElts % SubMask.size()) == 0) && "Illegal submask scale");
5828       if ((NumSubElts % SubMask.size()) == 0) {
5829         int Scale = NumSubElts / SubMask.size();
5830         SmallVector<int,64> ScaledSubMask;
5831         narrowShuffleMaskElts(Scale, SubMask, ScaledSubMask);
5832         SubMask = ScaledSubMask;
5833       } else {
5834         int Scale = SubMask.size() / NumSubElts;
5835         NumSubElts = SubMask.size();
5836         NumElts *= Scale;
5837         InsertIdx *= Scale;
5838       }
5839     }
5840     Ops.push_back(Src);
5841     Ops.append(SubInputs.begin(), SubInputs.end());
5842     if (ISD::isBuildVectorAllZeros(Src.getNode()))
5843       Mask.append(NumElts, SM_SentinelZero);
5844     else
5845       for (int i = 0; i != (int)NumElts; ++i)
5846         Mask.push_back(i);
5847     for (int i = 0; i != (int)NumSubElts; ++i) {
5848       int M = SubMask[i];
5849       if (0 <= M) {
5850         int InputIdx = M / NumSubElts;
5851         M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
5852       }
5853       Mask[i + InsertIdx] = M;
5854     }
5855     return true;
5856   }
5857   case X86ISD::PINSRB:
5858   case X86ISD::PINSRW:
5859   case ISD::SCALAR_TO_VECTOR:
5860   case ISD::INSERT_VECTOR_ELT: {
5861     // Match against a insert_vector_elt/scalar_to_vector of an extract from a
5862     // vector, for matching src/dst vector types.
5863     SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
5864 
5865     unsigned DstIdx = 0;
5866     if (Opcode != ISD::SCALAR_TO_VECTOR) {
5867       // Check we have an in-range constant insertion index.
5868       if (!isa<ConstantSDNode>(N.getOperand(2)) ||
5869           N.getConstantOperandAPInt(2).uge(NumElts))
5870         return false;
5871       DstIdx = N.getConstantOperandVal(2);
5872 
5873       // Attempt to recognise an INSERT*(VEC, 0, DstIdx) shuffle pattern.
5874       if (X86::isZeroNode(Scl)) {
5875         Ops.push_back(N.getOperand(0));
5876         for (unsigned i = 0; i != NumElts; ++i)
5877           Mask.push_back(i == DstIdx ? SM_SentinelZero : (int)i);
5878         return true;
5879       }
5880     }
5881 
5882     // Peek through trunc/aext/zext.
5883     // TODO: aext shouldn't require SM_SentinelZero padding.
5884     // TODO: handle shift of scalars.
5885     unsigned MinBitsPerElt = Scl.getScalarValueSizeInBits();
5886     while (Scl.getOpcode() == ISD::TRUNCATE ||
5887            Scl.getOpcode() == ISD::ANY_EXTEND ||
5888            Scl.getOpcode() == ISD::ZERO_EXTEND) {
5889       Scl = Scl.getOperand(0);
5890       MinBitsPerElt =
5891           std::min<unsigned>(MinBitsPerElt, Scl.getScalarValueSizeInBits());
5892     }
5893     if ((MinBitsPerElt % 8) != 0)
5894       return false;
5895 
5896     // Attempt to find the source vector the scalar was extracted from.
5897     SDValue SrcExtract;
5898     if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5899          Scl.getOpcode() == X86ISD::PEXTRW ||
5900          Scl.getOpcode() == X86ISD::PEXTRB) &&
5901         Scl.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
5902       SrcExtract = Scl;
5903     }
5904     if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
5905       return false;
5906 
5907     SDValue SrcVec = SrcExtract.getOperand(0);
5908     EVT SrcVT = SrcVec.getValueType();
5909     if (!SrcVT.getScalarType().isByteSized())
5910       return false;
5911     unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
5912     unsigned SrcByte = SrcIdx * (SrcVT.getScalarSizeInBits() / 8);
5913     unsigned DstByte = DstIdx * NumBytesPerElt;
5914     MinBitsPerElt =
5915         std::min<unsigned>(MinBitsPerElt, SrcVT.getScalarSizeInBits());
5916 
5917     // Create 'identity' byte level shuffle mask and then add inserted bytes.
5918     if (Opcode == ISD::SCALAR_TO_VECTOR) {
5919       Ops.push_back(SrcVec);
5920       Mask.append(NumSizeInBytes, SM_SentinelUndef);
5921     } else {
5922       Ops.push_back(SrcVec);
5923       Ops.push_back(N.getOperand(0));
5924       for (int i = 0; i != (int)NumSizeInBytes; ++i)
5925         Mask.push_back(NumSizeInBytes + i);
5926     }
5927 
5928     unsigned MinBytesPerElts = MinBitsPerElt / 8;
5929     MinBytesPerElts = std::min(MinBytesPerElts, NumBytesPerElt);
5930     for (unsigned i = 0; i != MinBytesPerElts; ++i)
5931       Mask[DstByte + i] = SrcByte + i;
5932     for (unsigned i = MinBytesPerElts; i < NumBytesPerElt; ++i)
5933       Mask[DstByte + i] = SM_SentinelZero;
5934     return true;
5935   }
5936   case X86ISD::PACKSS:
5937   case X86ISD::PACKUS: {
5938     SDValue N0 = N.getOperand(0);
5939     SDValue N1 = N.getOperand(1);
5940     assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
5941            N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
5942            "Unexpected input value type");
5943 
5944     APInt EltsLHS, EltsRHS;
5945     getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
5946 
5947     // If we know input saturation won't happen (or we don't care for particular
5948     // lanes), we can treat this as a truncation shuffle.
5949     bool Offset0 = false, Offset1 = false;
5950     if (Opcode == X86ISD::PACKSS) {
5951       if ((!(N0.isUndef() || EltsLHS.isZero()) &&
5952            DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
5953           (!(N1.isUndef() || EltsRHS.isZero()) &&
5954            DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
5955         return false;
5956       // We can't easily fold ASHR into a shuffle, but if it was feeding a
5957       // PACKSS then it was likely being used for sign-extension for a
5958       // truncation, so just peek through and adjust the mask accordingly.
5959       if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) &&
5960           N0.getConstantOperandAPInt(1) == NumBitsPerElt) {
5961         Offset0 = true;
5962         N0 = N0.getOperand(0);
5963       }
5964       if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) &&
5965           N1.getConstantOperandAPInt(1) == NumBitsPerElt) {
5966         Offset1 = true;
5967         N1 = N1.getOperand(0);
5968       }
5969     } else {
5970       APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
5971       if ((!(N0.isUndef() || EltsLHS.isZero()) &&
5972            !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
5973           (!(N1.isUndef() || EltsRHS.isZero()) &&
5974            !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
5975         return false;
5976     }
5977 
5978     bool IsUnary = (N0 == N1);
5979 
5980     Ops.push_back(N0);
5981     if (!IsUnary)
5982       Ops.push_back(N1);
5983 
5984     createPackShuffleMask(VT, Mask, IsUnary);
5985 
5986     if (Offset0 || Offset1) {
5987       for (int &M : Mask)
5988         if ((Offset0 && isInRange(M, 0, NumElts)) ||
5989             (Offset1 && isInRange(M, NumElts, 2 * NumElts)))
5990           ++M;
5991     }
5992     return true;
5993   }
5994   case ISD::VSELECT:
5995   case X86ISD::BLENDV: {
5996     SDValue Cond = N.getOperand(0);
5997     if (createShuffleMaskFromVSELECT(Mask, Cond, Opcode == X86ISD::BLENDV)) {
5998       Ops.push_back(N.getOperand(1));
5999       Ops.push_back(N.getOperand(2));
6000       return true;
6001     }
6002     return false;
6003   }
6004   case X86ISD::VTRUNC: {
6005     SDValue Src = N.getOperand(0);
6006     EVT SrcVT = Src.getValueType();
6007     // Truncated source must be a simple vector.
6008     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6009         (SrcVT.getScalarSizeInBits() % 8) != 0)
6010       return false;
6011     unsigned NumSrcElts = SrcVT.getVectorNumElements();
6012     unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6013     unsigned Scale = NumBitsPerSrcElt / NumBitsPerElt;
6014     assert((NumBitsPerSrcElt % NumBitsPerElt) == 0 && "Illegal truncation");
6015     for (unsigned i = 0; i != NumSrcElts; ++i)
6016       Mask.push_back(i * Scale);
6017     Mask.append(NumElts - NumSrcElts, SM_SentinelZero);
6018     Ops.push_back(Src);
6019     return true;
6020   }
6021   case X86ISD::VSHLI:
6022   case X86ISD::VSRLI: {
6023     uint64_t ShiftVal = N.getConstantOperandVal(1);
6024     // Out of range bit shifts are guaranteed to be zero.
6025     if (NumBitsPerElt <= ShiftVal) {
6026       Mask.append(NumElts, SM_SentinelZero);
6027       return true;
6028     }
6029 
6030     // We can only decode 'whole byte' bit shifts as shuffles.
6031     if ((ShiftVal % 8) != 0)
6032       break;
6033 
6034     uint64_t ByteShift = ShiftVal / 8;
6035     Ops.push_back(N.getOperand(0));
6036 
6037     // Clear mask to all zeros and insert the shifted byte indices.
6038     Mask.append(NumSizeInBytes, SM_SentinelZero);
6039 
6040     if (X86ISD::VSHLI == Opcode) {
6041       for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6042         for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6043           Mask[i + j] = i + j - ByteShift;
6044     } else {
6045       for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6046         for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6047           Mask[i + j - ByteShift] = i + j;
6048     }
6049     return true;
6050   }
6051   case X86ISD::VROTLI:
6052   case X86ISD::VROTRI: {
6053     // We can only decode 'whole byte' bit rotates as shuffles.
6054     uint64_t RotateVal = N.getConstantOperandAPInt(1).urem(NumBitsPerElt);
6055     if ((RotateVal % 8) != 0)
6056       return false;
6057     Ops.push_back(N.getOperand(0));
6058     int Offset = RotateVal / 8;
6059     Offset = (X86ISD::VROTLI == Opcode ? NumBytesPerElt - Offset : Offset);
6060     for (int i = 0; i != (int)NumElts; ++i) {
6061       int BaseIdx = i * NumBytesPerElt;
6062       for (int j = 0; j != (int)NumBytesPerElt; ++j) {
6063         Mask.push_back(BaseIdx + ((Offset + j) % NumBytesPerElt));
6064       }
6065     }
6066     return true;
6067   }
6068   case X86ISD::VBROADCAST: {
6069     SDValue Src = N.getOperand(0);
6070     if (!Src.getSimpleValueType().isVector()) {
6071       if (Src.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6072           !isNullConstant(Src.getOperand(1)) ||
6073           Src.getOperand(0).getValueType().getScalarType() !=
6074               VT.getScalarType())
6075         return false;
6076       Src = Src.getOperand(0);
6077     }
6078     Ops.push_back(Src);
6079     Mask.append(NumElts, 0);
6080     return true;
6081   }
6082   case ISD::SIGN_EXTEND_VECTOR_INREG: {
6083     SDValue Src = N.getOperand(0);
6084     EVT SrcVT = Src.getValueType();
6085     unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6086 
6087     // Extended source must be a simple vector.
6088     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6089         (NumBitsPerSrcElt % 8) != 0)
6090       return false;
6091 
6092     // We can only handle all-signbits extensions.
6093     APInt DemandedSrcElts =
6094         DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
6095     if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt)
6096       return false;
6097 
6098     assert((NumBitsPerElt % NumBitsPerSrcElt) == 0 && "Unexpected extension");
6099     unsigned Scale = NumBitsPerElt / NumBitsPerSrcElt;
6100     for (unsigned I = 0; I != NumElts; ++I)
6101       Mask.append(Scale, I);
6102     Ops.push_back(Src);
6103     return true;
6104   }
6105   case ISD::ZERO_EXTEND:
6106   case ISD::ANY_EXTEND:
6107   case ISD::ZERO_EXTEND_VECTOR_INREG:
6108   case ISD::ANY_EXTEND_VECTOR_INREG: {
6109     SDValue Src = N.getOperand(0);
6110     EVT SrcVT = Src.getValueType();
6111 
6112     // Extended source must be a simple vector.
6113     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6114         (SrcVT.getScalarSizeInBits() % 8) != 0)
6115       return false;
6116 
6117     bool IsAnyExtend =
6118         (ISD::ANY_EXTEND == Opcode || ISD::ANY_EXTEND_VECTOR_INREG == Opcode);
6119     DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts,
6120                          IsAnyExtend, Mask);
6121     Ops.push_back(Src);
6122     return true;
6123   }
6124   }
6125 
6126   return false;
6127 }
6128 
6129 /// Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
6130 static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
6131                                               SmallVectorImpl<int> &Mask) {
6132   int MaskWidth = Mask.size();
6133   SmallVector<SDValue, 16> UsedInputs;
6134   for (int i = 0, e = Inputs.size(); i < e; ++i) {
6135     int lo = UsedInputs.size() * MaskWidth;
6136     int hi = lo + MaskWidth;
6137 
6138     // Strip UNDEF input usage.
6139     if (Inputs[i].isUndef())
6140       for (int &M : Mask)
6141         if ((lo <= M) && (M < hi))
6142           M = SM_SentinelUndef;
6143 
6144     // Check for unused inputs.
6145     if (none_of(Mask, [lo, hi](int i) { return (lo <= i) && (i < hi); })) {
6146       for (int &M : Mask)
6147         if (lo <= M)
6148           M -= MaskWidth;
6149       continue;
6150     }
6151 
6152     // Check for repeated inputs.
6153     bool IsRepeat = false;
6154     for (int j = 0, ue = UsedInputs.size(); j != ue; ++j) {
6155       if (UsedInputs[j] != Inputs[i])
6156         continue;
6157       for (int &M : Mask)
6158         if (lo <= M)
6159           M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth);
6160       IsRepeat = true;
6161       break;
6162     }
6163     if (IsRepeat)
6164       continue;
6165 
6166     UsedInputs.push_back(Inputs[i]);
6167   }
6168   Inputs = UsedInputs;
6169 }
6170 
6171 /// Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs
6172 /// and then sets the SM_SentinelUndef and SM_SentinelZero values.
6173 /// Returns true if the target shuffle mask was decoded.
6174 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6175                                    SmallVectorImpl<SDValue> &Inputs,
6176                                    SmallVectorImpl<int> &Mask,
6177                                    APInt &KnownUndef, APInt &KnownZero,
6178                                    const SelectionDAG &DAG, unsigned Depth,
6179                                    bool ResolveKnownElts) {
6180   if (Depth >= SelectionDAG::MaxRecursionDepth)
6181     return false; // Limit search depth.
6182 
6183   EVT VT = Op.getValueType();
6184   if (!VT.isSimple() || !VT.isVector())
6185     return false;
6186 
6187   if (getTargetShuffleAndZeroables(Op, Mask, Inputs, KnownUndef, KnownZero)) {
6188     if (ResolveKnownElts)
6189       resolveTargetShuffleFromZeroables(Mask, KnownUndef, KnownZero);
6190     return true;
6191   }
6192   if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
6193                          ResolveKnownElts)) {
6194     resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
6195     return true;
6196   }
6197   return false;
6198 }
6199 
6200 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6201                                    SmallVectorImpl<SDValue> &Inputs,
6202                                    SmallVectorImpl<int> &Mask,
6203                                    const SelectionDAG &DAG, unsigned Depth,
6204                                    bool ResolveKnownElts) {
6205   APInt KnownUndef, KnownZero;
6206   return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, KnownUndef,
6207                                 KnownZero, DAG, Depth, ResolveKnownElts);
6208 }
6209 
6210 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6211                                    SmallVectorImpl<int> &Mask,
6212                                    const SelectionDAG &DAG, unsigned Depth = 0,
6213                                    bool ResolveKnownElts = true) {
6214   EVT VT = Op.getValueType();
6215   if (!VT.isSimple() || !VT.isVector())
6216     return false;
6217 
6218   unsigned NumElts = Op.getValueType().getVectorNumElements();
6219   APInt DemandedElts = APInt::getAllOnes(NumElts);
6220   return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth,
6221                                 ResolveKnownElts);
6222 }
6223 
6224 // Attempt to create a scalar/subvector broadcast from the base MemSDNode.
6225 static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT,
6226                                  EVT MemVT, MemSDNode *Mem, unsigned Offset,
6227                                  SelectionDAG &DAG) {
6228   assert((Opcode == X86ISD::VBROADCAST_LOAD ||
6229           Opcode == X86ISD::SUBV_BROADCAST_LOAD) &&
6230          "Unknown broadcast load type");
6231 
6232   // Ensure this is a simple (non-atomic, non-voltile), temporal read memop.
6233   if (!Mem || !Mem->readMem() || !Mem->isSimple() || Mem->isNonTemporal())
6234     return SDValue();
6235 
6236   SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
6237                                          TypeSize::getFixed(Offset), DL);
6238   SDVTList Tys = DAG.getVTList(VT, MVT::Other);
6239   SDValue Ops[] = {Mem->getChain(), Ptr};
6240   SDValue BcstLd = DAG.getMemIntrinsicNode(
6241       Opcode, DL, Tys, Ops, MemVT,
6242       DAG.getMachineFunction().getMachineMemOperand(
6243           Mem->getMemOperand(), Offset, MemVT.getStoreSize()));
6244   DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
6245   return BcstLd;
6246 }
6247 
6248 /// Returns the scalar element that will make up the i'th
6249 /// element of the result of the vector shuffle.
6250 static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
6251                                    SelectionDAG &DAG, unsigned Depth) {
6252   if (Depth >= SelectionDAG::MaxRecursionDepth)
6253     return SDValue(); // Limit search depth.
6254 
6255   EVT VT = Op.getValueType();
6256   unsigned Opcode = Op.getOpcode();
6257   unsigned NumElems = VT.getVectorNumElements();
6258 
6259   // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
6260   if (auto *SV = dyn_cast<ShuffleVectorSDNode>(Op)) {
6261     int Elt = SV->getMaskElt(Index);
6262 
6263     if (Elt < 0)
6264       return DAG.getUNDEF(VT.getVectorElementType());
6265 
6266     SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
6267     return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6268   }
6269 
6270   // Recurse into target specific vector shuffles to find scalars.
6271   if (isTargetShuffle(Opcode)) {
6272     MVT ShufVT = VT.getSimpleVT();
6273     MVT ShufSVT = ShufVT.getVectorElementType();
6274     int NumElems = (int)ShufVT.getVectorNumElements();
6275     SmallVector<int, 16> ShuffleMask;
6276     SmallVector<SDValue, 16> ShuffleOps;
6277     if (!getTargetShuffleMask(Op.getNode(), ShufVT, true, ShuffleOps,
6278                               ShuffleMask))
6279       return SDValue();
6280 
6281     int Elt = ShuffleMask[Index];
6282     if (Elt == SM_SentinelZero)
6283       return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT)
6284                                  : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT);
6285     if (Elt == SM_SentinelUndef)
6286       return DAG.getUNDEF(ShufSVT);
6287 
6288     assert(0 <= Elt && Elt < (2 * NumElems) && "Shuffle index out of range");
6289     SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
6290     return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6291   }
6292 
6293   // Recurse into insert_subvector base/sub vector to find scalars.
6294   if (Opcode == ISD::INSERT_SUBVECTOR) {
6295     SDValue Vec = Op.getOperand(0);
6296     SDValue Sub = Op.getOperand(1);
6297     uint64_t SubIdx = Op.getConstantOperandVal(2);
6298     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
6299 
6300     if (SubIdx <= Index && Index < (SubIdx + NumSubElts))
6301       return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1);
6302     return getShuffleScalarElt(Vec, Index, DAG, Depth + 1);
6303   }
6304 
6305   // Recurse into concat_vectors sub vector to find scalars.
6306   if (Opcode == ISD::CONCAT_VECTORS) {
6307     EVT SubVT = Op.getOperand(0).getValueType();
6308     unsigned NumSubElts = SubVT.getVectorNumElements();
6309     uint64_t SubIdx = Index / NumSubElts;
6310     uint64_t SubElt = Index % NumSubElts;
6311     return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1);
6312   }
6313 
6314   // Recurse into extract_subvector src vector to find scalars.
6315   if (Opcode == ISD::EXTRACT_SUBVECTOR) {
6316     SDValue Src = Op.getOperand(0);
6317     uint64_t SrcIdx = Op.getConstantOperandVal(1);
6318     return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1);
6319   }
6320 
6321   // We only peek through bitcasts of the same vector width.
6322   if (Opcode == ISD::BITCAST) {
6323     SDValue Src = Op.getOperand(0);
6324     EVT SrcVT = Src.getValueType();
6325     if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems)
6326       return getShuffleScalarElt(Src, Index, DAG, Depth + 1);
6327     return SDValue();
6328   }
6329 
6330   // Actual nodes that may contain scalar elements
6331 
6332   // For insert_vector_elt - either return the index matching scalar or recurse
6333   // into the base vector.
6334   if (Opcode == ISD::INSERT_VECTOR_ELT &&
6335       isa<ConstantSDNode>(Op.getOperand(2))) {
6336     if (Op.getConstantOperandAPInt(2) == Index)
6337       return Op.getOperand(1);
6338     return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1);
6339   }
6340 
6341   if (Opcode == ISD::SCALAR_TO_VECTOR)
6342     return (Index == 0) ? Op.getOperand(0)
6343                         : DAG.getUNDEF(VT.getVectorElementType());
6344 
6345   if (Opcode == ISD::BUILD_VECTOR)
6346     return Op.getOperand(Index);
6347 
6348   return SDValue();
6349 }
6350 
6351 // Use PINSRB/PINSRW/PINSRD to create a build vector.
6352 static SDValue LowerBuildVectorAsInsert(SDValue Op, const APInt &NonZeroMask,
6353                                         unsigned NumNonZero, unsigned NumZero,
6354                                         SelectionDAG &DAG,
6355                                         const X86Subtarget &Subtarget) {
6356   MVT VT = Op.getSimpleValueType();
6357   unsigned NumElts = VT.getVectorNumElements();
6358   assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||
6359           ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&
6360          "Illegal vector insertion");
6361 
6362   SDLoc dl(Op);
6363   SDValue V;
6364   bool First = true;
6365 
6366   for (unsigned i = 0; i < NumElts; ++i) {
6367     bool IsNonZero = NonZeroMask[i];
6368     if (!IsNonZero)
6369       continue;
6370 
6371     // If the build vector contains zeros or our first insertion is not the
6372     // first index then insert into zero vector to break any register
6373     // dependency else use SCALAR_TO_VECTOR.
6374     if (First) {
6375       First = false;
6376       if (NumZero || 0 != i)
6377         V = getZeroVector(VT, Subtarget, DAG, dl);
6378       else {
6379         assert(0 == i && "Expected insertion into zero-index");
6380         V = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6381         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);
6382         V = DAG.getBitcast(VT, V);
6383         continue;
6384       }
6385     }
6386     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i),
6387                     DAG.getIntPtrConstant(i, dl));
6388   }
6389 
6390   return V;
6391 }
6392 
6393 /// Custom lower build_vector of v16i8.
6394 static SDValue LowerBuildVectorv16i8(SDValue Op, const APInt &NonZeroMask,
6395                                      unsigned NumNonZero, unsigned NumZero,
6396                                      SelectionDAG &DAG,
6397                                      const X86Subtarget &Subtarget) {
6398   if (NumNonZero > 8 && !Subtarget.hasSSE41())
6399     return SDValue();
6400 
6401   // SSE4.1 - use PINSRB to insert each byte directly.
6402   if (Subtarget.hasSSE41())
6403     return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG,
6404                                     Subtarget);
6405 
6406   SDLoc dl(Op);
6407   SDValue V;
6408 
6409   // Pre-SSE4.1 - merge byte pairs and insert with PINSRW.
6410   // If both the lowest 16-bits are non-zero, then convert to MOVD.
6411   if (!NonZeroMask.extractBits(2, 0).isZero() &&
6412       !NonZeroMask.extractBits(2, 2).isZero()) {
6413     for (unsigned I = 0; I != 4; ++I) {
6414       if (!NonZeroMask[I])
6415         continue;
6416       SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), dl, MVT::i32);
6417       if (I != 0)
6418         Elt = DAG.getNode(ISD::SHL, dl, MVT::i32, Elt,
6419                           DAG.getConstant(I * 8, dl, MVT::i8));
6420       V = V ? DAG.getNode(ISD::OR, dl, MVT::i32, V, Elt) : Elt;
6421     }
6422     assert(V && "Failed to fold v16i8 vector to zero");
6423     V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);
6424     V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V);
6425     V = DAG.getBitcast(MVT::v8i16, V);
6426   }
6427   for (unsigned i = V ? 4 : 0; i < 16; i += 2) {
6428     bool ThisIsNonZero = NonZeroMask[i];
6429     bool NextIsNonZero = NonZeroMask[i + 1];
6430     if (!ThisIsNonZero && !NextIsNonZero)
6431       continue;
6432 
6433     SDValue Elt;
6434     if (ThisIsNonZero) {
6435       if (NumZero || NextIsNonZero)
6436         Elt = DAG.getZExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6437       else
6438         Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6439     }
6440 
6441     if (NextIsNonZero) {
6442       SDValue NextElt = Op.getOperand(i + 1);
6443       if (i == 0 && NumZero)
6444         NextElt = DAG.getZExtOrTrunc(NextElt, dl, MVT::i32);
6445       else
6446         NextElt = DAG.getAnyExtOrTrunc(NextElt, dl, MVT::i32);
6447       NextElt = DAG.getNode(ISD::SHL, dl, MVT::i32, NextElt,
6448                             DAG.getConstant(8, dl, MVT::i8));
6449       if (ThisIsNonZero)
6450         Elt = DAG.getNode(ISD::OR, dl, MVT::i32, NextElt, Elt);
6451       else
6452         Elt = NextElt;
6453     }
6454 
6455     // If our first insertion is not the first index or zeros are needed, then
6456     // insert into zero vector. Otherwise, use SCALAR_TO_VECTOR (leaves high
6457     // elements undefined).
6458     if (!V) {
6459       if (i != 0 || NumZero)
6460         V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
6461       else {
6462         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt);
6463         V = DAG.getBitcast(MVT::v8i16, V);
6464         continue;
6465       }
6466     }
6467     Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Elt);
6468     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, Elt,
6469                     DAG.getIntPtrConstant(i / 2, dl));
6470   }
6471 
6472   return DAG.getBitcast(MVT::v16i8, V);
6473 }
6474 
6475 /// Custom lower build_vector of v8i16.
6476 static SDValue LowerBuildVectorv8i16(SDValue Op, const APInt &NonZeroMask,
6477                                      unsigned NumNonZero, unsigned NumZero,
6478                                      SelectionDAG &DAG,
6479                                      const X86Subtarget &Subtarget) {
6480   if (NumNonZero > 4 && !Subtarget.hasSSE41())
6481     return SDValue();
6482 
6483   // Use PINSRW to insert each byte directly.
6484   return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG,
6485                                   Subtarget);
6486 }
6487 
6488 /// Custom lower build_vector of v4i32 or v4f32.
6489 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
6490                                      const X86Subtarget &Subtarget) {
6491   // If this is a splat of a pair of elements, use MOVDDUP (unless the target
6492   // has XOP; in that case defer lowering to potentially use VPERMIL2PS).
6493   // Because we're creating a less complicated build vector here, we may enable
6494   // further folding of the MOVDDUP via shuffle transforms.
6495   if (Subtarget.hasSSE3() && !Subtarget.hasXOP() &&
6496       Op.getOperand(0) == Op.getOperand(2) &&
6497       Op.getOperand(1) == Op.getOperand(3) &&
6498       Op.getOperand(0) != Op.getOperand(1)) {
6499     SDLoc DL(Op);
6500     MVT VT = Op.getSimpleValueType();
6501     MVT EltVT = VT.getVectorElementType();
6502     // Create a new build vector with the first 2 elements followed by undef
6503     // padding, bitcast to v2f64, duplicate, and bitcast back.
6504     SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
6505                        DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
6506     SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
6507     SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
6508     return DAG.getBitcast(VT, Dup);
6509   }
6510 
6511   // Find all zeroable elements.
6512   std::bitset<4> Zeroable, Undefs;
6513   for (int i = 0; i < 4; ++i) {
6514     SDValue Elt = Op.getOperand(i);
6515     Undefs[i] = Elt.isUndef();
6516     Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
6517   }
6518   assert(Zeroable.size() - Zeroable.count() > 1 &&
6519          "We expect at least two non-zero elements!");
6520 
6521   // We only know how to deal with build_vector nodes where elements are either
6522   // zeroable or extract_vector_elt with constant index.
6523   SDValue FirstNonZero;
6524   unsigned FirstNonZeroIdx;
6525   for (unsigned i = 0; i < 4; ++i) {
6526     if (Zeroable[i])
6527       continue;
6528     SDValue Elt = Op.getOperand(i);
6529     if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6530         !isa<ConstantSDNode>(Elt.getOperand(1)))
6531       return SDValue();
6532     // Make sure that this node is extracting from a 128-bit vector.
6533     MVT VT = Elt.getOperand(0).getSimpleValueType();
6534     if (!VT.is128BitVector())
6535       return SDValue();
6536     if (!FirstNonZero.getNode()) {
6537       FirstNonZero = Elt;
6538       FirstNonZeroIdx = i;
6539     }
6540   }
6541 
6542   assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!");
6543   SDValue V1 = FirstNonZero.getOperand(0);
6544   MVT VT = V1.getSimpleValueType();
6545 
6546   // See if this build_vector can be lowered as a blend with zero.
6547   SDValue Elt;
6548   unsigned EltMaskIdx, EltIdx;
6549   int Mask[4];
6550   for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
6551     if (Zeroable[EltIdx]) {
6552       // The zero vector will be on the right hand side.
6553       Mask[EltIdx] = EltIdx+4;
6554       continue;
6555     }
6556 
6557     Elt = Op->getOperand(EltIdx);
6558     // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
6559     EltMaskIdx = Elt.getConstantOperandVal(1);
6560     if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
6561       break;
6562     Mask[EltIdx] = EltIdx;
6563   }
6564 
6565   if (EltIdx == 4) {
6566     // Let the shuffle legalizer deal with blend operations.
6567     SDValue VZeroOrUndef = (Zeroable == Undefs)
6568                                ? DAG.getUNDEF(VT)
6569                                : getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
6570     if (V1.getSimpleValueType() != VT)
6571       V1 = DAG.getBitcast(VT, V1);
6572     return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask);
6573   }
6574 
6575   // See if we can lower this build_vector to a INSERTPS.
6576   if (!Subtarget.hasSSE41())
6577     return SDValue();
6578 
6579   SDValue V2 = Elt.getOperand(0);
6580   if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
6581     V1 = SDValue();
6582 
6583   bool CanFold = true;
6584   for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
6585     if (Zeroable[i])
6586       continue;
6587 
6588     SDValue Current = Op->getOperand(i);
6589     SDValue SrcVector = Current->getOperand(0);
6590     if (!V1.getNode())
6591       V1 = SrcVector;
6592     CanFold = (SrcVector == V1) && (Current.getConstantOperandAPInt(1) == i);
6593   }
6594 
6595   if (!CanFold)
6596     return SDValue();
6597 
6598   assert(V1.getNode() && "Expected at least two non-zero elements!");
6599   if (V1.getSimpleValueType() != MVT::v4f32)
6600     V1 = DAG.getBitcast(MVT::v4f32, V1);
6601   if (V2.getSimpleValueType() != MVT::v4f32)
6602     V2 = DAG.getBitcast(MVT::v4f32, V2);
6603 
6604   // Ok, we can emit an INSERTPS instruction.
6605   unsigned ZMask = Zeroable.to_ulong();
6606 
6607   unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
6608   assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
6609   SDLoc DL(Op);
6610   SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6611                                DAG.getIntPtrConstant(InsertPSMask, DL, true));
6612   return DAG.getBitcast(VT, Result);
6613 }
6614 
6615 /// Return a vector logical shift node.
6616 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
6617                          SelectionDAG &DAG, const TargetLowering &TLI,
6618                          const SDLoc &dl) {
6619   assert(VT.is128BitVector() && "Unknown type for VShift");
6620   MVT ShVT = MVT::v16i8;
6621   unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
6622   SrcOp = DAG.getBitcast(ShVT, SrcOp);
6623   assert(NumBits % 8 == 0 && "Only support byte sized shifts");
6624   SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
6625   return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
6626 }
6627 
6628 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
6629                                       SelectionDAG &DAG) {
6630 
6631   // Check if the scalar load can be widened into a vector load. And if
6632   // the address is "base + cst" see if the cst can be "absorbed" into
6633   // the shuffle mask.
6634   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
6635     SDValue Ptr = LD->getBasePtr();
6636     if (!ISD::isNormalLoad(LD) || !LD->isSimple())
6637       return SDValue();
6638     EVT PVT = LD->getValueType(0);
6639     if (PVT != MVT::i32 && PVT != MVT::f32)
6640       return SDValue();
6641 
6642     int FI = -1;
6643     int64_t Offset = 0;
6644     if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
6645       FI = FINode->getIndex();
6646       Offset = 0;
6647     } else if (DAG.isBaseWithConstantOffset(Ptr) &&
6648                isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6649       FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6650       Offset = Ptr.getConstantOperandVal(1);
6651       Ptr = Ptr.getOperand(0);
6652     } else {
6653       return SDValue();
6654     }
6655 
6656     // FIXME: 256-bit vector instructions don't require a strict alignment,
6657     // improve this code to support it better.
6658     Align RequiredAlign(VT.getSizeInBits() / 8);
6659     SDValue Chain = LD->getChain();
6660     // Make sure the stack object alignment is at least 16 or 32.
6661     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6662     MaybeAlign InferredAlign = DAG.InferPtrAlign(Ptr);
6663     if (!InferredAlign || *InferredAlign < RequiredAlign) {
6664       if (MFI.isFixedObjectIndex(FI)) {
6665         // Can't change the alignment. FIXME: It's possible to compute
6666         // the exact stack offset and reference FI + adjust offset instead.
6667         // If someone *really* cares about this. That's the way to implement it.
6668         return SDValue();
6669       } else {
6670         MFI.setObjectAlignment(FI, RequiredAlign);
6671       }
6672     }
6673 
6674     // (Offset % 16 or 32) must be multiple of 4. Then address is then
6675     // Ptr + (Offset & ~15).
6676     if (Offset < 0)
6677       return SDValue();
6678     if ((Offset % RequiredAlign.value()) & 3)
6679       return SDValue();
6680     int64_t StartOffset = Offset & ~int64_t(RequiredAlign.value() - 1);
6681     if (StartOffset) {
6682       SDLoc DL(Ptr);
6683       Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
6684                         DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
6685     }
6686 
6687     int EltNo = (Offset - StartOffset) >> 2;
6688     unsigned NumElems = VT.getVectorNumElements();
6689 
6690     EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
6691     SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
6692                              LD->getPointerInfo().getWithOffset(StartOffset));
6693 
6694     SmallVector<int, 8> Mask(NumElems, EltNo);
6695 
6696     return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask);
6697   }
6698 
6699   return SDValue();
6700 }
6701 
6702 // Recurse to find a LoadSDNode source and the accumulated ByteOffest.
6703 static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
6704   if (ISD::isNON_EXTLoad(Elt.getNode())) {
6705     auto *BaseLd = cast<LoadSDNode>(Elt);
6706     if (!BaseLd->isSimple())
6707       return false;
6708     Ld = BaseLd;
6709     ByteOffset = 0;
6710     return true;
6711   }
6712 
6713   switch (Elt.getOpcode()) {
6714   case ISD::BITCAST:
6715   case ISD::TRUNCATE:
6716   case ISD::SCALAR_TO_VECTOR:
6717     return findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset);
6718   case ISD::SRL:
6719     if (auto *AmtC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6720       uint64_t Amt = AmtC->getZExtValue();
6721       if ((Amt % 8) == 0 && findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset)) {
6722         ByteOffset += Amt / 8;
6723         return true;
6724       }
6725     }
6726     break;
6727   case ISD::EXTRACT_VECTOR_ELT:
6728     if (auto *IdxC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6729       SDValue Src = Elt.getOperand(0);
6730       unsigned SrcSizeInBits = Src.getScalarValueSizeInBits();
6731       unsigned DstSizeInBits = Elt.getScalarValueSizeInBits();
6732       if (DstSizeInBits == SrcSizeInBits && (SrcSizeInBits % 8) == 0 &&
6733           findEltLoadSrc(Src, Ld, ByteOffset)) {
6734         uint64_t Idx = IdxC->getZExtValue();
6735         ByteOffset += Idx * (SrcSizeInBits / 8);
6736         return true;
6737       }
6738     }
6739     break;
6740   }
6741 
6742   return false;
6743 }
6744 
6745 /// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
6746 /// elements can be replaced by a single large load which has the same value as
6747 /// a build_vector or insert_subvector whose loaded operands are 'Elts'.
6748 ///
6749 /// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
6750 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
6751                                         const SDLoc &DL, SelectionDAG &DAG,
6752                                         const X86Subtarget &Subtarget,
6753                                         bool IsAfterLegalize) {
6754   if ((VT.getScalarSizeInBits() % 8) != 0)
6755     return SDValue();
6756 
6757   unsigned NumElems = Elts.size();
6758 
6759   int LastLoadedElt = -1;
6760   APInt LoadMask = APInt::getZero(NumElems);
6761   APInt ZeroMask = APInt::getZero(NumElems);
6762   APInt UndefMask = APInt::getZero(NumElems);
6763 
6764   SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
6765   SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
6766 
6767   // For each element in the initializer, see if we've found a load, zero or an
6768   // undef.
6769   for (unsigned i = 0; i < NumElems; ++i) {
6770     SDValue Elt = peekThroughBitcasts(Elts[i]);
6771     if (!Elt.getNode())
6772       return SDValue();
6773     if (Elt.isUndef()) {
6774       UndefMask.setBit(i);
6775       continue;
6776     }
6777     if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) {
6778       ZeroMask.setBit(i);
6779       continue;
6780     }
6781 
6782     // Each loaded element must be the correct fractional portion of the
6783     // requested vector load.
6784     unsigned EltSizeInBits = Elt.getValueSizeInBits();
6785     if ((NumElems * EltSizeInBits) != VT.getSizeInBits())
6786       return SDValue();
6787 
6788     if (!findEltLoadSrc(Elt, Loads[i], ByteOffsets[i]) || ByteOffsets[i] < 0)
6789       return SDValue();
6790     unsigned LoadSizeInBits = Loads[i]->getValueSizeInBits(0);
6791     if (((ByteOffsets[i] * 8) + EltSizeInBits) > LoadSizeInBits)
6792       return SDValue();
6793 
6794     LoadMask.setBit(i);
6795     LastLoadedElt = i;
6796   }
6797   assert((ZeroMask.popcount() + UndefMask.popcount() + LoadMask.popcount()) ==
6798              NumElems &&
6799          "Incomplete element masks");
6800 
6801   // Handle Special Cases - all undef or undef/zero.
6802   if (UndefMask.popcount() == NumElems)
6803     return DAG.getUNDEF(VT);
6804   if ((ZeroMask.popcount() + UndefMask.popcount()) == NumElems)
6805     return VT.isInteger() ? DAG.getConstant(0, DL, VT)
6806                           : DAG.getConstantFP(0.0, DL, VT);
6807 
6808   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6809   int FirstLoadedElt = LoadMask.countr_zero();
6810   SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
6811   EVT EltBaseVT = EltBase.getValueType();
6812   assert(EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() &&
6813          "Register/Memory size mismatch");
6814   LoadSDNode *LDBase = Loads[FirstLoadedElt];
6815   assert(LDBase && "Did not find base load for merging consecutive loads");
6816   unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
6817   unsigned BaseSizeInBytes = BaseSizeInBits / 8;
6818   int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt);
6819   int LoadSizeInBits = NumLoadedElts * BaseSizeInBits;
6820   assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected");
6821 
6822   // TODO: Support offsetting the base load.
6823   if (ByteOffsets[FirstLoadedElt] != 0)
6824     return SDValue();
6825 
6826   // Check to see if the element's load is consecutive to the base load
6827   // or offset from a previous (already checked) load.
6828   auto CheckConsecutiveLoad = [&](LoadSDNode *Base, int EltIdx) {
6829     LoadSDNode *Ld = Loads[EltIdx];
6830     int64_t ByteOffset = ByteOffsets[EltIdx];
6831     if (ByteOffset && (ByteOffset % BaseSizeInBytes) == 0) {
6832       int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes);
6833       return (0 <= BaseIdx && BaseIdx < (int)NumElems && LoadMask[BaseIdx] &&
6834               Loads[BaseIdx] == Ld && ByteOffsets[BaseIdx] == 0);
6835     }
6836     return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes,
6837                                               EltIdx - FirstLoadedElt);
6838   };
6839 
6840   // Consecutive loads can contain UNDEFS but not ZERO elements.
6841   // Consecutive loads with UNDEFs and ZEROs elements require a
6842   // an additional shuffle stage to clear the ZERO elements.
6843   bool IsConsecutiveLoad = true;
6844   bool IsConsecutiveLoadWithZeros = true;
6845   for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
6846     if (LoadMask[i]) {
6847       if (!CheckConsecutiveLoad(LDBase, i)) {
6848         IsConsecutiveLoad = false;
6849         IsConsecutiveLoadWithZeros = false;
6850         break;
6851       }
6852     } else if (ZeroMask[i]) {
6853       IsConsecutiveLoad = false;
6854     }
6855   }
6856 
6857   auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
6858     auto MMOFlags = LDBase->getMemOperand()->getFlags();
6859     assert(LDBase->isSimple() &&
6860            "Cannot merge volatile or atomic loads.");
6861     SDValue NewLd =
6862         DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
6863                     LDBase->getPointerInfo(), LDBase->getOriginalAlign(),
6864                     MMOFlags);
6865     for (auto *LD : Loads)
6866       if (LD)
6867         DAG.makeEquivalentMemoryOrdering(LD, NewLd);
6868     return NewLd;
6869   };
6870 
6871   // Check if the base load is entirely dereferenceable.
6872   bool IsDereferenceable = LDBase->getPointerInfo().isDereferenceable(
6873       VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout());
6874 
6875   // LOAD - all consecutive load/undefs (must start/end with a load or be
6876   // entirely dereferenceable). If we have found an entire vector of loads and
6877   // undefs, then return a large load of the entire vector width starting at the
6878   // base pointer. If the vector contains zeros, then attempt to shuffle those
6879   // elements.
6880   if (FirstLoadedElt == 0 &&
6881       (NumLoadedElts == (int)NumElems || IsDereferenceable) &&
6882       (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
6883     if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
6884       return SDValue();
6885 
6886     // Don't create 256-bit non-temporal aligned loads without AVX2 as these
6887     // will lower to regular temporal loads and use the cache.
6888     if (LDBase->isNonTemporal() && LDBase->getAlign() >= Align(32) &&
6889         VT.is256BitVector() && !Subtarget.hasInt256())
6890       return SDValue();
6891 
6892     if (NumElems == 1)
6893       return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
6894 
6895     if (!ZeroMask)
6896       return CreateLoad(VT, LDBase);
6897 
6898     // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
6899     // vector and a zero vector to clear out the zero elements.
6900     if (!IsAfterLegalize && VT.isVector()) {
6901       unsigned NumMaskElts = VT.getVectorNumElements();
6902       if ((NumMaskElts % NumElems) == 0) {
6903         unsigned Scale = NumMaskElts / NumElems;
6904         SmallVector<int, 4> ClearMask(NumMaskElts, -1);
6905         for (unsigned i = 0; i < NumElems; ++i) {
6906           if (UndefMask[i])
6907             continue;
6908           int Offset = ZeroMask[i] ? NumMaskElts : 0;
6909           for (unsigned j = 0; j != Scale; ++j)
6910             ClearMask[(i * Scale) + j] = (i * Scale) + j + Offset;
6911         }
6912         SDValue V = CreateLoad(VT, LDBase);
6913         SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
6914                                    : DAG.getConstantFP(0.0, DL, VT);
6915         return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
6916       }
6917     }
6918   }
6919 
6920   // If the upper half of a ymm/zmm load is undef then just load the lower half.
6921   if (VT.is256BitVector() || VT.is512BitVector()) {
6922     unsigned HalfNumElems = NumElems / 2;
6923     if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnes()) {
6924       EVT HalfVT =
6925           EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
6926       SDValue HalfLD =
6927           EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
6928                                    DAG, Subtarget, IsAfterLegalize);
6929       if (HalfLD)
6930         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
6931                            HalfLD, DAG.getIntPtrConstant(0, DL));
6932     }
6933   }
6934 
6935   // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs.
6936   if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
6937       ((LoadSizeInBits == 16 && Subtarget.hasFP16()) || LoadSizeInBits == 32 ||
6938        LoadSizeInBits == 64) &&
6939       ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {
6940     MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits)
6941                                       : MVT::getIntegerVT(LoadSizeInBits);
6942     MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits);
6943     // Allow v4f32 on SSE1 only targets.
6944     // FIXME: Add more isel patterns so we can just use VT directly.
6945     if (!Subtarget.hasSSE2() && VT == MVT::v4f32)
6946       VecVT = MVT::v4f32;
6947     if (TLI.isTypeLegal(VecVT)) {
6948       SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
6949       SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
6950       SDValue ResNode = DAG.getMemIntrinsicNode(
6951           X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
6952           LDBase->getOriginalAlign(), MachineMemOperand::MOLoad);
6953       for (auto *LD : Loads)
6954         if (LD)
6955           DAG.makeEquivalentMemoryOrdering(LD, ResNode);
6956       return DAG.getBitcast(VT, ResNode);
6957     }
6958   }
6959 
6960   // BROADCAST - match the smallest possible repetition pattern, load that
6961   // scalar/subvector element and then broadcast to the entire vector.
6962   if (ZeroMask.isZero() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() &&
6963       (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) {
6964     for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) {
6965       unsigned RepeatSize = SubElems * BaseSizeInBits;
6966       unsigned ScalarSize = std::min(RepeatSize, 64u);
6967       if (!Subtarget.hasAVX2() && ScalarSize < 32)
6968         continue;
6969 
6970       // Don't attempt a 1:N subvector broadcast - it should be caught by
6971       // combineConcatVectorOps, else will cause infinite loops.
6972       if (RepeatSize > ScalarSize && SubElems == 1)
6973         continue;
6974 
6975       bool Match = true;
6976       SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
6977       for (unsigned i = 0; i != NumElems && Match; ++i) {
6978         if (!LoadMask[i])
6979           continue;
6980         SDValue Elt = peekThroughBitcasts(Elts[i]);
6981         if (RepeatedLoads[i % SubElems].isUndef())
6982           RepeatedLoads[i % SubElems] = Elt;
6983         else
6984           Match &= (RepeatedLoads[i % SubElems] == Elt);
6985       }
6986 
6987       // We must have loads at both ends of the repetition.
6988       Match &= !RepeatedLoads.front().isUndef();
6989       Match &= !RepeatedLoads.back().isUndef();
6990       if (!Match)
6991         continue;
6992 
6993       EVT RepeatVT =
6994           VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64))
6995               ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize)
6996               : EVT::getFloatingPointVT(ScalarSize);
6997       if (RepeatSize > ScalarSize)
6998         RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
6999                                     RepeatSize / ScalarSize);
7000       EVT BroadcastVT =
7001           EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
7002                            VT.getSizeInBits() / ScalarSize);
7003       if (TLI.isTypeLegal(BroadcastVT)) {
7004         if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
7005                 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) {
7006           SDValue Broadcast = RepeatLoad;
7007           if (RepeatSize > ScalarSize) {
7008             while (Broadcast.getValueSizeInBits() < VT.getSizeInBits())
7009               Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL);
7010           } else {
7011             if (!Subtarget.hasAVX2() &&
7012                 !X86::mayFoldLoadIntoBroadcastFromMem(
7013                     RepeatLoad, RepeatVT.getScalarType().getSimpleVT(),
7014                     Subtarget,
7015                     /*AssumeSingleUse=*/true))
7016               return SDValue();
7017             Broadcast =
7018                 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad);
7019           }
7020           return DAG.getBitcast(VT, Broadcast);
7021         }
7022       }
7023     }
7024   }
7025 
7026   return SDValue();
7027 }
7028 
7029 // Combine a vector ops (shuffles etc.) that is equal to build_vector load1,
7030 // load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses
7031 // are consecutive, non-overlapping, and in the right order.
7032 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
7033                                          SelectionDAG &DAG,
7034                                          const X86Subtarget &Subtarget,
7035                                          bool IsAfterLegalize) {
7036   SmallVector<SDValue, 64> Elts;
7037   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
7038     if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
7039       Elts.push_back(Elt);
7040       continue;
7041     }
7042     return SDValue();
7043   }
7044   assert(Elts.size() == VT.getVectorNumElements());
7045   return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
7046                                   IsAfterLegalize);
7047 }
7048 
7049 static Constant *getConstantVector(MVT VT, ArrayRef<APInt> Bits,
7050                                    const APInt &Undefs, LLVMContext &C) {
7051   unsigned ScalarSize = VT.getScalarSizeInBits();
7052   Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C);
7053 
7054   auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7055     if (VT.isFloatingPoint()) {
7056       if (ScalarSize == 16)
7057         return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7058       if (ScalarSize == 32)
7059         return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7060       assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7061       return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7062     }
7063     return Constant::getIntegerValue(Ty, Val);
7064   };
7065 
7066   SmallVector<Constant *, 32> ConstantVec;
7067   for (unsigned I = 0, E = Bits.size(); I != E; ++I)
7068     ConstantVec.push_back(Undefs[I] ? UndefValue::get(Ty)
7069                                     : getConstantScalar(Bits[I]));
7070 
7071   return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7072 }
7073 
7074 static Constant *getConstantVector(MVT VT, const APInt &SplatValue,
7075                                    unsigned SplatBitSize, LLVMContext &C) {
7076   unsigned ScalarSize = VT.getScalarSizeInBits();
7077 
7078   auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7079     if (VT.isFloatingPoint()) {
7080       if (ScalarSize == 16)
7081         return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7082       if (ScalarSize == 32)
7083         return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7084       assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7085       return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7086     }
7087     return Constant::getIntegerValue(Type::getIntNTy(C, ScalarSize), Val);
7088   };
7089 
7090   if (ScalarSize == SplatBitSize)
7091     return getConstantScalar(SplatValue);
7092 
7093   unsigned NumElm = SplatBitSize / ScalarSize;
7094   SmallVector<Constant *, 32> ConstantVec;
7095   for (unsigned I = 0; I != NumElm; ++I) {
7096     APInt Val = SplatValue.extractBits(ScalarSize, ScalarSize * I);
7097     ConstantVec.push_back(getConstantScalar(Val));
7098   }
7099   return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7100 }
7101 
7102 static bool isFoldableUseOfShuffle(SDNode *N) {
7103   for (auto *U : N->uses()) {
7104     unsigned Opc = U->getOpcode();
7105     // VPERMV/VPERMV3 shuffles can never fold their index operands.
7106     if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N)
7107       return false;
7108     if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N)
7109       return false;
7110     if (isTargetShuffle(Opc))
7111       return true;
7112     if (Opc == ISD::BITCAST) // Ignore bitcasts
7113       return isFoldableUseOfShuffle(U);
7114     if (N->hasOneUse()) {
7115       // TODO, there may be some general way to know if a SDNode can
7116       // be folded. We now only know whether an MI is foldable.
7117       if (Opc == X86ISD::VPDPBUSD && U->getOperand(2).getNode() != N)
7118         return false;
7119       return true;
7120     }
7121   }
7122   return false;
7123 }
7124 
7125 /// Attempt to use the vbroadcast instruction to generate a splat value
7126 /// from a splat BUILD_VECTOR which uses:
7127 ///  a. A single scalar load, or a constant.
7128 ///  b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
7129 ///
7130 /// The VBROADCAST node is returned when a pattern is found,
7131 /// or SDValue() otherwise.
7132 static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
7133                                            const X86Subtarget &Subtarget,
7134                                            SelectionDAG &DAG) {
7135   // VBROADCAST requires AVX.
7136   // TODO: Splats could be generated for non-AVX CPUs using SSE
7137   // instructions, but there's less potential gain for only 128-bit vectors.
7138   if (!Subtarget.hasAVX())
7139     return SDValue();
7140 
7141   MVT VT = BVOp->getSimpleValueType(0);
7142   unsigned NumElts = VT.getVectorNumElements();
7143   SDLoc dl(BVOp);
7144 
7145   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
7146          "Unsupported vector type for broadcast.");
7147 
7148   // See if the build vector is a repeating sequence of scalars (inc. splat).
7149   SDValue Ld;
7150   BitVector UndefElements;
7151   SmallVector<SDValue, 16> Sequence;
7152   if (BVOp->getRepeatedSequence(Sequence, &UndefElements)) {
7153     assert((NumElts % Sequence.size()) == 0 && "Sequence doesn't fit.");
7154     if (Sequence.size() == 1)
7155       Ld = Sequence[0];
7156   }
7157 
7158   // Attempt to use VBROADCASTM
7159   // From this pattern:
7160   // a. t0 = (zext_i64 (bitcast_i8 v2i1 X))
7161   // b. t1 = (build_vector t0 t0)
7162   //
7163   // Create (VBROADCASTM v2i1 X)
7164   if (!Sequence.empty() && Subtarget.hasCDI()) {
7165     // If not a splat, are the upper sequence values zeroable?
7166     unsigned SeqLen = Sequence.size();
7167     bool UpperZeroOrUndef =
7168         SeqLen == 1 ||
7169         llvm::all_of(ArrayRef(Sequence).drop_front(), [](SDValue V) {
7170           return !V || V.isUndef() || isNullConstant(V);
7171         });
7172     SDValue Op0 = Sequence[0];
7173     if (UpperZeroOrUndef && ((Op0.getOpcode() == ISD::BITCAST) ||
7174                              (Op0.getOpcode() == ISD::ZERO_EXTEND &&
7175                               Op0.getOperand(0).getOpcode() == ISD::BITCAST))) {
7176       SDValue BOperand = Op0.getOpcode() == ISD::BITCAST
7177                              ? Op0.getOperand(0)
7178                              : Op0.getOperand(0).getOperand(0);
7179       MVT MaskVT = BOperand.getSimpleValueType();
7180       MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen);
7181       if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) ||  // for broadcastmb2q
7182           (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d
7183         MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen);
7184         if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
7185           unsigned Scale = 512 / VT.getSizeInBits();
7186           BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen));
7187         }
7188         SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand);
7189         if (BcstVT.getSizeInBits() != VT.getSizeInBits())
7190           Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits());
7191         return DAG.getBitcast(VT, Bcst);
7192       }
7193     }
7194   }
7195 
7196   unsigned NumUndefElts = UndefElements.count();
7197   if (!Ld || (NumElts - NumUndefElts) <= 1) {
7198     APInt SplatValue, Undef;
7199     unsigned SplatBitSize;
7200     bool HasUndef;
7201     // Check if this is a repeated constant pattern suitable for broadcasting.
7202     if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) &&
7203         SplatBitSize > VT.getScalarSizeInBits() &&
7204         SplatBitSize < VT.getSizeInBits()) {
7205       // Avoid replacing with broadcast when it's a use of a shuffle
7206       // instruction to preserve the present custom lowering of shuffles.
7207       if (isFoldableUseOfShuffle(BVOp))
7208         return SDValue();
7209       // replace BUILD_VECTOR with broadcast of the repeated constants.
7210       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7211       LLVMContext *Ctx = DAG.getContext();
7212       MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
7213       if (SplatBitSize == 32 || SplatBitSize == 64 ||
7214           (SplatBitSize < 32 && Subtarget.hasAVX2())) {
7215         // Load the constant scalar/subvector and broadcast it.
7216         MVT CVT = MVT::getIntegerVT(SplatBitSize);
7217         Constant *C = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7218         SDValue CP = DAG.getConstantPool(C, PVT);
7219         unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
7220 
7221         Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7222         SDVTList Tys = DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);
7223         SDValue Ops[] = {DAG.getEntryNode(), CP};
7224         MachinePointerInfo MPI =
7225             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7226         SDValue Brdcst =
7227             DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7228                                     MPI, Alignment, MachineMemOperand::MOLoad);
7229         return DAG.getBitcast(VT, Brdcst);
7230       }
7231       if (SplatBitSize > 64) {
7232         // Load the vector of constants and broadcast it.
7233         Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7234         SDValue VCP = DAG.getConstantPool(VecC, PVT);
7235         unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits();
7236         MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm);
7237         Align Alignment = cast<ConstantPoolSDNode>(VCP)->getAlign();
7238         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7239         SDValue Ops[] = {DAG.getEntryNode(), VCP};
7240         MachinePointerInfo MPI =
7241             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7242         return DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, dl, Tys,
7243                                        Ops, VVT, MPI, Alignment,
7244                                        MachineMemOperand::MOLoad);
7245       }
7246     }
7247 
7248     // If we are moving a scalar into a vector (Ld must be set and all elements
7249     // but 1 are undef) and that operation is not obviously supported by
7250     // vmovd/vmovq/vmovss/vmovsd, then keep trying to form a broadcast.
7251     // That's better than general shuffling and may eliminate a load to GPR and
7252     // move from scalar to vector register.
7253     if (!Ld || NumElts - NumUndefElts != 1)
7254       return SDValue();
7255     unsigned ScalarSize = Ld.getValueSizeInBits();
7256     if (!(UndefElements[0] || (ScalarSize != 32 && ScalarSize != 64)))
7257       return SDValue();
7258   }
7259 
7260   bool ConstSplatVal =
7261       (Ld.getOpcode() == ISD::Constant || Ld.getOpcode() == ISD::ConstantFP);
7262   bool IsLoad = ISD::isNormalLoad(Ld.getNode());
7263 
7264   // TODO: Handle broadcasts of non-constant sequences.
7265 
7266   // Make sure that all of the users of a non-constant load are from the
7267   // BUILD_VECTOR node.
7268   // FIXME: Is the use count needed for non-constant, non-load case?
7269   if (!ConstSplatVal && !IsLoad && !BVOp->isOnlyUserOf(Ld.getNode()))
7270     return SDValue();
7271 
7272   unsigned ScalarSize = Ld.getValueSizeInBits();
7273   bool IsGE256 = (VT.getSizeInBits() >= 256);
7274 
7275   // When optimizing for size, generate up to 5 extra bytes for a broadcast
7276   // instruction to save 8 or more bytes of constant pool data.
7277   // TODO: If multiple splats are generated to load the same constant,
7278   // it may be detrimental to overall size. There needs to be a way to detect
7279   // that condition to know if this is truly a size win.
7280   bool OptForSize = DAG.shouldOptForSize();
7281 
7282   // Handle broadcasting a single constant scalar from the constant pool
7283   // into a vector.
7284   // On Sandybridge (no AVX2), it is still better to load a constant vector
7285   // from the constant pool and not to broadcast it from a scalar.
7286   // But override that restriction when optimizing for size.
7287   // TODO: Check if splatting is recommended for other AVX-capable CPUs.
7288   if (ConstSplatVal && (Subtarget.hasAVX2() || OptForSize)) {
7289     EVT CVT = Ld.getValueType();
7290     assert(!CVT.isVector() && "Must not broadcast a vector type");
7291 
7292     // Splat f16, f32, i32, v4f64, v4i64 in all cases with AVX2.
7293     // For size optimization, also splat v2f64 and v2i64, and for size opt
7294     // with AVX2, also splat i8 and i16.
7295     // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
7296     if (ScalarSize == 32 ||
7297         (ScalarSize == 64 && (IsGE256 || Subtarget.hasVLX())) ||
7298         CVT == MVT::f16 ||
7299         (OptForSize && (ScalarSize == 64 || Subtarget.hasAVX2()))) {
7300       const Constant *C = nullptr;
7301       if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
7302         C = CI->getConstantIntValue();
7303       else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
7304         C = CF->getConstantFPValue();
7305 
7306       assert(C && "Invalid constant type");
7307 
7308       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7309       SDValue CP =
7310           DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
7311       Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7312 
7313       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7314       SDValue Ops[] = {DAG.getEntryNode(), CP};
7315       MachinePointerInfo MPI =
7316           MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7317       return DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7318                                      MPI, Alignment, MachineMemOperand::MOLoad);
7319     }
7320   }
7321 
7322   // Handle AVX2 in-register broadcasts.
7323   if (!IsLoad && Subtarget.hasInt256() &&
7324       (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
7325     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7326 
7327   // The scalar source must be a normal load.
7328   if (!IsLoad)
7329     return SDValue();
7330 
7331   // Make sure the non-chain result is only used by this build vector.
7332   if (!Ld->hasNUsesOfValue(NumElts - NumUndefElts, 0))
7333     return SDValue();
7334 
7335   if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
7336       (Subtarget.hasVLX() && ScalarSize == 64)) {
7337     auto *LN = cast<LoadSDNode>(Ld);
7338     SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7339     SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7340     SDValue BCast =
7341         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7342                                 LN->getMemoryVT(), LN->getMemOperand());
7343     DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7344     return BCast;
7345   }
7346 
7347   // The integer check is needed for the 64-bit into 128-bit so it doesn't match
7348   // double since there is no vbroadcastsd xmm
7349   if (Subtarget.hasInt256() && Ld.getValueType().isInteger() &&
7350       (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)) {
7351     auto *LN = cast<LoadSDNode>(Ld);
7352     SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7353     SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7354     SDValue BCast =
7355         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7356                                 LN->getMemoryVT(), LN->getMemOperand());
7357     DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7358     return BCast;
7359   }
7360 
7361   if (ScalarSize == 16 && Subtarget.hasFP16() && IsGE256)
7362     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7363 
7364   // Unsupported broadcast.
7365   return SDValue();
7366 }
7367 
7368 /// For an EXTRACT_VECTOR_ELT with a constant index return the real
7369 /// underlying vector and index.
7370 ///
7371 /// Modifies \p ExtractedFromVec to the real vector and returns the real
7372 /// index.
7373 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
7374                                          SDValue ExtIdx) {
7375   int Idx = ExtIdx->getAsZExtVal();
7376   if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
7377     return Idx;
7378 
7379   // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
7380   // lowered this:
7381   //   (extract_vector_elt (v8f32 %1), Constant<6>)
7382   // to:
7383   //   (extract_vector_elt (vector_shuffle<2,u,u,u>
7384   //                           (extract_subvector (v8f32 %0), Constant<4>),
7385   //                           undef)
7386   //                       Constant<0>)
7387   // In this case the vector is the extract_subvector expression and the index
7388   // is 2, as specified by the shuffle.
7389   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
7390   SDValue ShuffleVec = SVOp->getOperand(0);
7391   MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
7392   assert(ShuffleVecVT.getVectorElementType() ==
7393          ExtractedFromVec.getSimpleValueType().getVectorElementType());
7394 
7395   int ShuffleIdx = SVOp->getMaskElt(Idx);
7396   if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
7397     ExtractedFromVec = ShuffleVec;
7398     return ShuffleIdx;
7399   }
7400   return Idx;
7401 }
7402 
7403 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
7404   MVT VT = Op.getSimpleValueType();
7405 
7406   // Skip if insert_vec_elt is not supported.
7407   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7408   if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
7409     return SDValue();
7410 
7411   SDLoc DL(Op);
7412   unsigned NumElems = Op.getNumOperands();
7413 
7414   SDValue VecIn1;
7415   SDValue VecIn2;
7416   SmallVector<unsigned, 4> InsertIndices;
7417   SmallVector<int, 8> Mask(NumElems, -1);
7418 
7419   for (unsigned i = 0; i != NumElems; ++i) {
7420     unsigned Opc = Op.getOperand(i).getOpcode();
7421 
7422     if (Opc == ISD::UNDEF)
7423       continue;
7424 
7425     if (Opc != ISD::EXTRACT_VECTOR_ELT) {
7426       // Quit if more than 1 elements need inserting.
7427       if (InsertIndices.size() > 1)
7428         return SDValue();
7429 
7430       InsertIndices.push_back(i);
7431       continue;
7432     }
7433 
7434     SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
7435     SDValue ExtIdx = Op.getOperand(i).getOperand(1);
7436 
7437     // Quit if non-constant index.
7438     if (!isa<ConstantSDNode>(ExtIdx))
7439       return SDValue();
7440     int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
7441 
7442     // Quit if extracted from vector of different type.
7443     if (ExtractedFromVec.getValueType() != VT)
7444       return SDValue();
7445 
7446     if (!VecIn1.getNode())
7447       VecIn1 = ExtractedFromVec;
7448     else if (VecIn1 != ExtractedFromVec) {
7449       if (!VecIn2.getNode())
7450         VecIn2 = ExtractedFromVec;
7451       else if (VecIn2 != ExtractedFromVec)
7452         // Quit if more than 2 vectors to shuffle
7453         return SDValue();
7454     }
7455 
7456     if (ExtractedFromVec == VecIn1)
7457       Mask[i] = Idx;
7458     else if (ExtractedFromVec == VecIn2)
7459       Mask[i] = Idx + NumElems;
7460   }
7461 
7462   if (!VecIn1.getNode())
7463     return SDValue();
7464 
7465   VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7466   SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
7467 
7468   for (unsigned Idx : InsertIndices)
7469     NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
7470                      DAG.getIntPtrConstant(Idx, DL));
7471 
7472   return NV;
7473 }
7474 
7475 // Lower BUILD_VECTOR operation for v8bf16, v16bf16 and v32bf16 types.
7476 static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG,
7477                                        const X86Subtarget &Subtarget) {
7478   MVT VT = Op.getSimpleValueType();
7479   MVT IVT =
7480       VT.changeVectorElementType(Subtarget.hasFP16() ? MVT::f16 : MVT::i16);
7481   SmallVector<SDValue, 16> NewOps;
7482   for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I)
7483     NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16,
7484                                     Op.getOperand(I)));
7485   SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
7486   return DAG.getBitcast(VT, Res);
7487 }
7488 
7489 // Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
7490 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
7491                                      const X86Subtarget &Subtarget) {
7492 
7493   MVT VT = Op.getSimpleValueType();
7494   assert((VT.getVectorElementType() == MVT::i1) &&
7495          "Unexpected type in LowerBUILD_VECTORvXi1!");
7496 
7497   SDLoc dl(Op);
7498   if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
7499       ISD::isBuildVectorAllOnes(Op.getNode()))
7500     return Op;
7501 
7502   uint64_t Immediate = 0;
7503   SmallVector<unsigned, 16> NonConstIdx;
7504   bool IsSplat = true;
7505   bool HasConstElts = false;
7506   int SplatIdx = -1;
7507   for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
7508     SDValue In = Op.getOperand(idx);
7509     if (In.isUndef())
7510       continue;
7511     if (auto *InC = dyn_cast<ConstantSDNode>(In)) {
7512       Immediate |= (InC->getZExtValue() & 0x1) << idx;
7513       HasConstElts = true;
7514     } else {
7515       NonConstIdx.push_back(idx);
7516     }
7517     if (SplatIdx < 0)
7518       SplatIdx = idx;
7519     else if (In != Op.getOperand(SplatIdx))
7520       IsSplat = false;
7521   }
7522 
7523   // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
7524   if (IsSplat) {
7525     // The build_vector allows the scalar element to be larger than the vector
7526     // element type. We need to mask it to use as a condition unless we know
7527     // the upper bits are zero.
7528     // FIXME: Use computeKnownBits instead of checking specific opcode?
7529     SDValue Cond = Op.getOperand(SplatIdx);
7530     assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");
7531     if (Cond.getOpcode() != ISD::SETCC)
7532       Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
7533                          DAG.getConstant(1, dl, MVT::i8));
7534 
7535     // Perform the select in the scalar domain so we can use cmov.
7536     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7537       SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
7538                                      DAG.getAllOnesConstant(dl, MVT::i32),
7539                                      DAG.getConstant(0, dl, MVT::i32));
7540       Select = DAG.getBitcast(MVT::v32i1, Select);
7541       return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);
7542     } else {
7543       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7544       SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
7545                                      DAG.getAllOnesConstant(dl, ImmVT),
7546                                      DAG.getConstant(0, dl, ImmVT));
7547       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7548       Select = DAG.getBitcast(VecVT, Select);
7549       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select,
7550                          DAG.getIntPtrConstant(0, dl));
7551     }
7552   }
7553 
7554   // insert elements one by one
7555   SDValue DstVec;
7556   if (HasConstElts) {
7557     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7558       SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
7559       SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
7560       ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
7561       ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
7562       DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
7563     } else {
7564       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7565       SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
7566       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7567       DstVec = DAG.getBitcast(VecVT, Imm);
7568       DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
7569                            DAG.getIntPtrConstant(0, dl));
7570     }
7571   } else
7572     DstVec = DAG.getUNDEF(VT);
7573 
7574   for (unsigned InsertIdx : NonConstIdx) {
7575     DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
7576                          Op.getOperand(InsertIdx),
7577                          DAG.getIntPtrConstant(InsertIdx, dl));
7578   }
7579   return DstVec;
7580 }
7581 
7582 LLVM_ATTRIBUTE_UNUSED static bool isHorizOp(unsigned Opcode) {
7583   switch (Opcode) {
7584   case X86ISD::PACKSS:
7585   case X86ISD::PACKUS:
7586   case X86ISD::FHADD:
7587   case X86ISD::FHSUB:
7588   case X86ISD::HADD:
7589   case X86ISD::HSUB:
7590     return true;
7591   }
7592   return false;
7593 }
7594 
7595 /// This is a helper function of LowerToHorizontalOp().
7596 /// This function checks that the build_vector \p N in input implements a
7597 /// 128-bit partial horizontal operation on a 256-bit vector, but that operation
7598 /// may not match the layout of an x86 256-bit horizontal instruction.
7599 /// In other words, if this returns true, then some extraction/insertion will
7600 /// be required to produce a valid horizontal instruction.
7601 ///
7602 /// Parameter \p Opcode defines the kind of horizontal operation to match.
7603 /// For example, if \p Opcode is equal to ISD::ADD, then this function
7604 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
7605 /// is equal to ISD::SUB, then this function checks if this is a horizontal
7606 /// arithmetic sub.
7607 ///
7608 /// This function only analyzes elements of \p N whose indices are
7609 /// in range [BaseIdx, LastIdx).
7610 ///
7611 /// TODO: This function was originally used to match both real and fake partial
7612 /// horizontal operations, but the index-matching logic is incorrect for that.
7613 /// See the corrected implementation in isHopBuildVector(). Can we reduce this
7614 /// code because it is only used for partial h-op matching now?
7615 static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
7616                                   SelectionDAG &DAG,
7617                                   unsigned BaseIdx, unsigned LastIdx,
7618                                   SDValue &V0, SDValue &V1) {
7619   EVT VT = N->getValueType(0);
7620   assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");
7621   assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
7622   assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
7623          "Invalid Vector in input!");
7624 
7625   bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
7626   bool CanFold = true;
7627   unsigned ExpectedVExtractIdx = BaseIdx;
7628   unsigned NumElts = LastIdx - BaseIdx;
7629   V0 = DAG.getUNDEF(VT);
7630   V1 = DAG.getUNDEF(VT);
7631 
7632   // Check if N implements a horizontal binop.
7633   for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
7634     SDValue Op = N->getOperand(i + BaseIdx);
7635 
7636     // Skip UNDEFs.
7637     if (Op->isUndef()) {
7638       // Update the expected vector extract index.
7639       if (i * 2 == NumElts)
7640         ExpectedVExtractIdx = BaseIdx;
7641       ExpectedVExtractIdx += 2;
7642       continue;
7643     }
7644 
7645     CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
7646 
7647     if (!CanFold)
7648       break;
7649 
7650     SDValue Op0 = Op.getOperand(0);
7651     SDValue Op1 = Op.getOperand(1);
7652 
7653     // Try to match the following pattern:
7654     // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
7655     CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7656         Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7657         Op0.getOperand(0) == Op1.getOperand(0) &&
7658         isa<ConstantSDNode>(Op0.getOperand(1)) &&
7659         isa<ConstantSDNode>(Op1.getOperand(1)));
7660     if (!CanFold)
7661       break;
7662 
7663     unsigned I0 = Op0.getConstantOperandVal(1);
7664     unsigned I1 = Op1.getConstantOperandVal(1);
7665 
7666     if (i * 2 < NumElts) {
7667       if (V0.isUndef()) {
7668         V0 = Op0.getOperand(0);
7669         if (V0.getValueType() != VT)
7670           return false;
7671       }
7672     } else {
7673       if (V1.isUndef()) {
7674         V1 = Op0.getOperand(0);
7675         if (V1.getValueType() != VT)
7676           return false;
7677       }
7678       if (i * 2 == NumElts)
7679         ExpectedVExtractIdx = BaseIdx;
7680     }
7681 
7682     SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
7683     if (I0 == ExpectedVExtractIdx)
7684       CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
7685     else if (IsCommutable && I1 == ExpectedVExtractIdx) {
7686       // Try to match the following dag sequence:
7687       // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
7688       CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
7689     } else
7690       CanFold = false;
7691 
7692     ExpectedVExtractIdx += 2;
7693   }
7694 
7695   return CanFold;
7696 }
7697 
7698 /// Emit a sequence of two 128-bit horizontal add/sub followed by
7699 /// a concat_vector.
7700 ///
7701 /// This is a helper function of LowerToHorizontalOp().
7702 /// This function expects two 256-bit vectors called V0 and V1.
7703 /// At first, each vector is split into two separate 128-bit vectors.
7704 /// Then, the resulting 128-bit vectors are used to implement two
7705 /// horizontal binary operations.
7706 ///
7707 /// The kind of horizontal binary operation is defined by \p X86Opcode.
7708 ///
7709 /// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
7710 /// the two new horizontal binop.
7711 /// When Mode is set, the first horizontal binop dag node would take as input
7712 /// the lower 128-bit of V0 and the upper 128-bit of V0. The second
7713 /// horizontal binop dag node would take as input the lower 128-bit of V1
7714 /// and the upper 128-bit of V1.
7715 ///   Example:
7716 ///     HADD V0_LO, V0_HI
7717 ///     HADD V1_LO, V1_HI
7718 ///
7719 /// Otherwise, the first horizontal binop dag node takes as input the lower
7720 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
7721 /// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
7722 ///   Example:
7723 ///     HADD V0_LO, V1_LO
7724 ///     HADD V0_HI, V1_HI
7725 ///
7726 /// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
7727 /// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
7728 /// the upper 128-bits of the result.
7729 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
7730                                      const SDLoc &DL, SelectionDAG &DAG,
7731                                      unsigned X86Opcode, bool Mode,
7732                                      bool isUndefLO, bool isUndefHI) {
7733   MVT VT = V0.getSimpleValueType();
7734   assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&
7735          "Invalid nodes in input!");
7736 
7737   unsigned NumElts = VT.getVectorNumElements();
7738   SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
7739   SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
7740   SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
7741   SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
7742   MVT NewVT = V0_LO.getSimpleValueType();
7743 
7744   SDValue LO = DAG.getUNDEF(NewVT);
7745   SDValue HI = DAG.getUNDEF(NewVT);
7746 
7747   if (Mode) {
7748     // Don't emit a horizontal binop if the result is expected to be UNDEF.
7749     if (!isUndefLO && !V0->isUndef())
7750       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
7751     if (!isUndefHI && !V1->isUndef())
7752       HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
7753   } else {
7754     // Don't emit a horizontal binop if the result is expected to be UNDEF.
7755     if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef()))
7756       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
7757 
7758     if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
7759       HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
7760   }
7761 
7762   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
7763 }
7764 
7765 /// Returns true iff \p BV builds a vector with the result equivalent to
7766 /// the result of ADDSUB/SUBADD operation.
7767 /// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
7768 /// (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters
7769 /// \p Opnd0 and \p Opnd1.
7770 static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
7771                              const X86Subtarget &Subtarget, SelectionDAG &DAG,
7772                              SDValue &Opnd0, SDValue &Opnd1,
7773                              unsigned &NumExtracts,
7774                              bool &IsSubAdd) {
7775 
7776   MVT VT = BV->getSimpleValueType(0);
7777   if (!Subtarget.hasSSE3() || !VT.isFloatingPoint())
7778     return false;
7779 
7780   unsigned NumElts = VT.getVectorNumElements();
7781   SDValue InVec0 = DAG.getUNDEF(VT);
7782   SDValue InVec1 = DAG.getUNDEF(VT);
7783 
7784   NumExtracts = 0;
7785 
7786   // Odd-numbered elements in the input build vector are obtained from
7787   // adding/subtracting two integer/float elements.
7788   // Even-numbered elements in the input build vector are obtained from
7789   // subtracting/adding two integer/float elements.
7790   unsigned Opc[2] = {0, 0};
7791   for (unsigned i = 0, e = NumElts; i != e; ++i) {
7792     SDValue Op = BV->getOperand(i);
7793 
7794     // Skip 'undef' values.
7795     unsigned Opcode = Op.getOpcode();
7796     if (Opcode == ISD::UNDEF)
7797       continue;
7798 
7799     // Early exit if we found an unexpected opcode.
7800     if (Opcode != ISD::FADD && Opcode != ISD::FSUB)
7801       return false;
7802 
7803     SDValue Op0 = Op.getOperand(0);
7804     SDValue Op1 = Op.getOperand(1);
7805 
7806     // Try to match the following pattern:
7807     // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
7808     // Early exit if we cannot match that sequence.
7809     if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7810         Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7811         !isa<ConstantSDNode>(Op0.getOperand(1)) ||
7812         Op0.getOperand(1) != Op1.getOperand(1))
7813       return false;
7814 
7815     unsigned I0 = Op0.getConstantOperandVal(1);
7816     if (I0 != i)
7817       return false;
7818 
7819     // We found a valid add/sub node, make sure its the same opcode as previous
7820     // elements for this parity.
7821     if (Opc[i % 2] != 0 && Opc[i % 2] != Opcode)
7822       return false;
7823     Opc[i % 2] = Opcode;
7824 
7825     // Update InVec0 and InVec1.
7826     if (InVec0.isUndef()) {
7827       InVec0 = Op0.getOperand(0);
7828       if (InVec0.getSimpleValueType() != VT)
7829         return false;
7830     }
7831     if (InVec1.isUndef()) {
7832       InVec1 = Op1.getOperand(0);
7833       if (InVec1.getSimpleValueType() != VT)
7834         return false;
7835     }
7836 
7837     // Make sure that operands in input to each add/sub node always
7838     // come from a same pair of vectors.
7839     if (InVec0 != Op0.getOperand(0)) {
7840       if (Opcode == ISD::FSUB)
7841         return false;
7842 
7843       // FADD is commutable. Try to commute the operands
7844       // and then test again.
7845       std::swap(Op0, Op1);
7846       if (InVec0 != Op0.getOperand(0))
7847         return false;
7848     }
7849 
7850     if (InVec1 != Op1.getOperand(0))
7851       return false;
7852 
7853     // Increment the number of extractions done.
7854     ++NumExtracts;
7855   }
7856 
7857   // Ensure we have found an opcode for both parities and that they are
7858   // different. Don't try to fold this build_vector into an ADDSUB/SUBADD if the
7859   // inputs are undef.
7860   if (!Opc[0] || !Opc[1] || Opc[0] == Opc[1] ||
7861       InVec0.isUndef() || InVec1.isUndef())
7862     return false;
7863 
7864   IsSubAdd = Opc[0] == ISD::FADD;
7865 
7866   Opnd0 = InVec0;
7867   Opnd1 = InVec1;
7868   return true;
7869 }
7870 
7871 /// Returns true if is possible to fold MUL and an idiom that has already been
7872 /// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
7873 /// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
7874 /// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
7875 ///
7876 /// Prior to calling this function it should be known that there is some
7877 /// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
7878 /// using \p Opnd0 and \p Opnd1 as operands. Also, this method is called
7879 /// before replacement of such SDNode with ADDSUB operation. Thus the number
7880 /// of \p Opnd0 uses is expected to be equal to 2.
7881 /// For example, this function may be called for the following IR:
7882 ///    %AB = fmul fast <2 x double> %A, %B
7883 ///    %Sub = fsub fast <2 x double> %AB, %C
7884 ///    %Add = fadd fast <2 x double> %AB, %C
7885 ///    %Addsub = shufflevector <2 x double> %Sub, <2 x double> %Add,
7886 ///                            <2 x i32> <i32 0, i32 3>
7887 /// There is a def for %Addsub here, which potentially can be replaced by
7888 /// X86ISD::ADDSUB operation:
7889 ///    %Addsub = X86ISD::ADDSUB %AB, %C
7890 /// and such ADDSUB can further be replaced with FMADDSUB:
7891 ///    %Addsub = FMADDSUB %A, %B, %C.
7892 ///
7893 /// The main reason why this method is called before the replacement of the
7894 /// recognized ADDSUB idiom with ADDSUB operation is that such replacement
7895 /// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
7896 /// FMADDSUB is.
7897 static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
7898                                  SelectionDAG &DAG,
7899                                  SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
7900                                  unsigned ExpectedUses) {
7901   if (Opnd0.getOpcode() != ISD::FMUL ||
7902       !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
7903     return false;
7904 
7905   // FIXME: These checks must match the similar ones in
7906   // DAGCombiner::visitFADDForFMACombine. It would be good to have one
7907   // function that would answer if it is Ok to fuse MUL + ADD to FMADD
7908   // or MUL + ADDSUB to FMADDSUB.
7909   const TargetOptions &Options = DAG.getTarget().Options;
7910   bool AllowFusion =
7911       (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
7912   if (!AllowFusion)
7913     return false;
7914 
7915   Opnd2 = Opnd1;
7916   Opnd1 = Opnd0.getOperand(1);
7917   Opnd0 = Opnd0.getOperand(0);
7918 
7919   return true;
7920 }
7921 
7922 /// Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or
7923 /// 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or
7924 /// X86ISD::FMSUBADD node.
7925 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
7926                                        const X86Subtarget &Subtarget,
7927                                        SelectionDAG &DAG) {
7928   SDValue Opnd0, Opnd1;
7929   unsigned NumExtracts;
7930   bool IsSubAdd;
7931   if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
7932                         IsSubAdd))
7933     return SDValue();
7934 
7935   MVT VT = BV->getSimpleValueType(0);
7936   SDLoc DL(BV);
7937 
7938   // Try to generate X86ISD::FMADDSUB node here.
7939   SDValue Opnd2;
7940   if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
7941     unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
7942     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
7943   }
7944 
7945   // We only support ADDSUB.
7946   if (IsSubAdd)
7947     return SDValue();
7948 
7949   // There are no known X86 targets with 512-bit ADDSUB instructions!
7950   // Convert to blend(fsub,fadd).
7951   if (VT.is512BitVector()) {
7952     SmallVector<int> Mask;
7953     for (int I = 0, E = VT.getVectorNumElements(); I != E; I += 2) {
7954         Mask.push_back(I);
7955         Mask.push_back(I + E + 1);
7956     }
7957     SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1);
7958     SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1);
7959     return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask);
7960   }
7961 
7962   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
7963 }
7964 
7965 static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
7966                              unsigned &HOpcode, SDValue &V0, SDValue &V1) {
7967   // Initialize outputs to known values.
7968   MVT VT = BV->getSimpleValueType(0);
7969   HOpcode = ISD::DELETED_NODE;
7970   V0 = DAG.getUNDEF(VT);
7971   V1 = DAG.getUNDEF(VT);
7972 
7973   // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
7974   // half of the result is calculated independently from the 128-bit halves of
7975   // the inputs, so that makes the index-checking logic below more complicated.
7976   unsigned NumElts = VT.getVectorNumElements();
7977   unsigned GenericOpcode = ISD::DELETED_NODE;
7978   unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
7979   unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
7980   unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
7981   for (unsigned i = 0; i != Num128BitChunks; ++i) {
7982     for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
7983       // Ignore undef elements.
7984       SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
7985       if (Op.isUndef())
7986         continue;
7987 
7988       // If there's an opcode mismatch, we're done.
7989       if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
7990         return false;
7991 
7992       // Initialize horizontal opcode.
7993       if (HOpcode == ISD::DELETED_NODE) {
7994         GenericOpcode = Op.getOpcode();
7995         switch (GenericOpcode) {
7996         case ISD::ADD: HOpcode = X86ISD::HADD; break;
7997         case ISD::SUB: HOpcode = X86ISD::HSUB; break;
7998         case ISD::FADD: HOpcode = X86ISD::FHADD; break;
7999         case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
8000         default: return false;
8001         }
8002       }
8003 
8004       SDValue Op0 = Op.getOperand(0);
8005       SDValue Op1 = Op.getOperand(1);
8006       if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8007           Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8008           Op0.getOperand(0) != Op1.getOperand(0) ||
8009           !isa<ConstantSDNode>(Op0.getOperand(1)) ||
8010           !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
8011         return false;
8012 
8013       // The source vector is chosen based on which 64-bit half of the
8014       // destination vector is being calculated.
8015       if (j < NumEltsIn64Bits) {
8016         if (V0.isUndef())
8017           V0 = Op0.getOperand(0);
8018       } else {
8019         if (V1.isUndef())
8020           V1 = Op0.getOperand(0);
8021       }
8022 
8023       SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
8024       if (SourceVec != Op0.getOperand(0))
8025         return false;
8026 
8027       // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
8028       unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
8029       unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
8030       unsigned ExpectedIndex = i * NumEltsIn128Bits +
8031                                (j % NumEltsIn64Bits) * 2;
8032       if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
8033         continue;
8034 
8035       // If this is not a commutative op, this does not match.
8036       if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
8037         return false;
8038 
8039       // Addition is commutative, so try swapping the extract indexes.
8040       // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
8041       if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
8042         continue;
8043 
8044       // Extract indexes do not match horizontal requirement.
8045       return false;
8046     }
8047   }
8048   // We matched. Opcode and operands are returned by reference as arguments.
8049   return true;
8050 }
8051 
8052 static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
8053                                     SelectionDAG &DAG, unsigned HOpcode,
8054                                     SDValue V0, SDValue V1) {
8055   // If either input vector is not the same size as the build vector,
8056   // extract/insert the low bits to the correct size.
8057   // This is free (examples: zmm --> xmm, xmm --> ymm).
8058   MVT VT = BV->getSimpleValueType(0);
8059   unsigned Width = VT.getSizeInBits();
8060   if (V0.getValueSizeInBits() > Width)
8061     V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), Width);
8062   else if (V0.getValueSizeInBits() < Width)
8063     V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width);
8064 
8065   if (V1.getValueSizeInBits() > Width)
8066     V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), Width);
8067   else if (V1.getValueSizeInBits() < Width)
8068     V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width);
8069 
8070   unsigned NumElts = VT.getVectorNumElements();
8071   APInt DemandedElts = APInt::getAllOnes(NumElts);
8072   for (unsigned i = 0; i != NumElts; ++i)
8073     if (BV->getOperand(i).isUndef())
8074       DemandedElts.clearBit(i);
8075 
8076   // If we don't need the upper xmm, then perform as a xmm hop.
8077   unsigned HalfNumElts = NumElts / 2;
8078   if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) {
8079     MVT HalfVT = VT.getHalfNumVectorElementsVT();
8080     V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), 128);
8081     V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), 128);
8082     SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1);
8083     return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, SDLoc(BV), 256);
8084   }
8085 
8086   return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1);
8087 }
8088 
8089 /// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
8090 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
8091                                    const X86Subtarget &Subtarget,
8092                                    SelectionDAG &DAG) {
8093   // We need at least 2 non-undef elements to make this worthwhile by default.
8094   unsigned NumNonUndefs =
8095       count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
8096   if (NumNonUndefs < 2)
8097     return SDValue();
8098 
8099   // There are 4 sets of horizontal math operations distinguished by type:
8100   // int/FP at 128-bit/256-bit. Each type was introduced with a different
8101   // subtarget feature. Try to match those "native" patterns first.
8102   MVT VT = BV->getSimpleValueType(0);
8103   if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||
8104       ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||
8105       ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||
8106       ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {
8107     unsigned HOpcode;
8108     SDValue V0, V1;
8109     if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
8110       return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
8111   }
8112 
8113   // Try harder to match 256-bit ops by using extract/concat.
8114   if (!Subtarget.hasAVX() || !VT.is256BitVector())
8115     return SDValue();
8116 
8117   // Count the number of UNDEF operands in the build_vector in input.
8118   unsigned NumElts = VT.getVectorNumElements();
8119   unsigned Half = NumElts / 2;
8120   unsigned NumUndefsLO = 0;
8121   unsigned NumUndefsHI = 0;
8122   for (unsigned i = 0, e = Half; i != e; ++i)
8123     if (BV->getOperand(i)->isUndef())
8124       NumUndefsLO++;
8125 
8126   for (unsigned i = Half, e = NumElts; i != e; ++i)
8127     if (BV->getOperand(i)->isUndef())
8128       NumUndefsHI++;
8129 
8130   SDLoc DL(BV);
8131   SDValue InVec0, InVec1;
8132   if (VT == MVT::v8i32 || VT == MVT::v16i16) {
8133     SDValue InVec2, InVec3;
8134     unsigned X86Opcode;
8135     bool CanFold = true;
8136 
8137     if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
8138         isHorizontalBinOpPart(BV, ISD::ADD, DAG, Half, NumElts, InVec2,
8139                               InVec3) &&
8140         ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8141         ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8142       X86Opcode = X86ISD::HADD;
8143     else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, Half, InVec0,
8144                                    InVec1) &&
8145              isHorizontalBinOpPart(BV, ISD::SUB, DAG, Half, NumElts, InVec2,
8146                                    InVec3) &&
8147              ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8148              ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8149       X86Opcode = X86ISD::HSUB;
8150     else
8151       CanFold = false;
8152 
8153     if (CanFold) {
8154       // Do not try to expand this build_vector into a pair of horizontal
8155       // add/sub if we can emit a pair of scalar add/sub.
8156       if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8157         return SDValue();
8158 
8159       // Convert this build_vector into a pair of horizontal binops followed by
8160       // a concat vector. We must adjust the outputs from the partial horizontal
8161       // matching calls above to account for undefined vector halves.
8162       SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
8163       SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
8164       assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?");
8165       bool isUndefLO = NumUndefsLO == Half;
8166       bool isUndefHI = NumUndefsHI == Half;
8167       return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
8168                                    isUndefHI);
8169     }
8170   }
8171 
8172   if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
8173       VT == MVT::v16i16) {
8174     unsigned X86Opcode;
8175     if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
8176       X86Opcode = X86ISD::HADD;
8177     else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, NumElts, InVec0,
8178                                    InVec1))
8179       X86Opcode = X86ISD::HSUB;
8180     else if (isHorizontalBinOpPart(BV, ISD::FADD, DAG, 0, NumElts, InVec0,
8181                                    InVec1))
8182       X86Opcode = X86ISD::FHADD;
8183     else if (isHorizontalBinOpPart(BV, ISD::FSUB, DAG, 0, NumElts, InVec0,
8184                                    InVec1))
8185       X86Opcode = X86ISD::FHSUB;
8186     else
8187       return SDValue();
8188 
8189     // Don't try to expand this build_vector into a pair of horizontal add/sub
8190     // if we can simply emit a pair of scalar add/sub.
8191     if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8192       return SDValue();
8193 
8194     // Convert this build_vector into two horizontal add/sub followed by
8195     // a concat vector.
8196     bool isUndefLO = NumUndefsLO == Half;
8197     bool isUndefHI = NumUndefsHI == Half;
8198     return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
8199                                  isUndefLO, isUndefHI);
8200   }
8201 
8202   return SDValue();
8203 }
8204 
8205 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
8206                           SelectionDAG &DAG);
8207 
8208 /// If a BUILD_VECTOR's source elements all apply the same bit operation and
8209 /// one of their operands is constant, lower to a pair of BUILD_VECTOR and
8210 /// just apply the bit to the vectors.
8211 /// NOTE: Its not in our interest to start make a general purpose vectorizer
8212 /// from this, but enough scalar bit operations are created from the later
8213 /// legalization + scalarization stages to need basic support.
8214 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
8215                                        const X86Subtarget &Subtarget,
8216                                        SelectionDAG &DAG) {
8217   SDLoc DL(Op);
8218   MVT VT = Op->getSimpleValueType(0);
8219   unsigned NumElems = VT.getVectorNumElements();
8220   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8221 
8222   // Check that all elements have the same opcode.
8223   // TODO: Should we allow UNDEFS and if so how many?
8224   unsigned Opcode = Op->getOperand(0).getOpcode();
8225   for (unsigned i = 1; i < NumElems; ++i)
8226     if (Opcode != Op->getOperand(i).getOpcode())
8227       return SDValue();
8228 
8229   // TODO: We may be able to add support for other Ops (ADD/SUB + shifts).
8230   bool IsShift = false;
8231   switch (Opcode) {
8232   default:
8233     return SDValue();
8234   case ISD::SHL:
8235   case ISD::SRL:
8236   case ISD::SRA:
8237     IsShift = true;
8238     break;
8239   case ISD::AND:
8240   case ISD::XOR:
8241   case ISD::OR:
8242     // Don't do this if the buildvector is a splat - we'd replace one
8243     // constant with an entire vector.
8244     if (Op->getSplatValue())
8245       return SDValue();
8246     if (!TLI.isOperationLegalOrPromote(Opcode, VT))
8247       return SDValue();
8248     break;
8249   }
8250 
8251   SmallVector<SDValue, 4> LHSElts, RHSElts;
8252   for (SDValue Elt : Op->ops()) {
8253     SDValue LHS = Elt.getOperand(0);
8254     SDValue RHS = Elt.getOperand(1);
8255 
8256     // We expect the canonicalized RHS operand to be the constant.
8257     if (!isa<ConstantSDNode>(RHS))
8258       return SDValue();
8259 
8260     // Extend shift amounts.
8261     if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) {
8262       if (!IsShift)
8263         return SDValue();
8264       RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
8265     }
8266 
8267     LHSElts.push_back(LHS);
8268     RHSElts.push_back(RHS);
8269   }
8270 
8271   // Limit to shifts by uniform immediates.
8272   // TODO: Only accept vXi8/vXi64 special cases?
8273   // TODO: Permit non-uniform XOP/AVX2/MULLO cases?
8274   if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
8275     return SDValue();
8276 
8277   SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
8278   SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
8279   SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
8280 
8281   if (!IsShift)
8282     return Res;
8283 
8284   // Immediately lower the shift to ensure the constant build vector doesn't
8285   // get converted to a constant pool before the shift is lowered.
8286   return LowerShift(Res, Subtarget, DAG);
8287 }
8288 
8289 /// Create a vector constant without a load. SSE/AVX provide the bare minimum
8290 /// functionality to do this, so it's all zeros, all ones, or some derivation
8291 /// that is cheap to calculate.
8292 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
8293                                          const X86Subtarget &Subtarget) {
8294   SDLoc DL(Op);
8295   MVT VT = Op.getSimpleValueType();
8296 
8297   // Vectors containing all zeros can be matched by pxor and xorps.
8298   if (ISD::isBuildVectorAllZeros(Op.getNode()))
8299     return Op;
8300 
8301   // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
8302   // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
8303   // vpcmpeqd on 256-bit vectors.
8304   if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
8305     if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
8306       return Op;
8307 
8308     return getOnesVector(VT, DAG, DL);
8309   }
8310 
8311   return SDValue();
8312 }
8313 
8314 /// Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute
8315 /// from a vector of source values and a vector of extraction indices.
8316 /// The vectors might be manipulated to match the type of the permute op.
8317 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
8318                                      SDLoc &DL, SelectionDAG &DAG,
8319                                      const X86Subtarget &Subtarget) {
8320   MVT ShuffleVT = VT;
8321   EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8322   unsigned NumElts = VT.getVectorNumElements();
8323   unsigned SizeInBits = VT.getSizeInBits();
8324 
8325   // Adjust IndicesVec to match VT size.
8326   assert(IndicesVec.getValueType().getVectorNumElements() >= NumElts &&
8327          "Illegal variable permute mask size");
8328   if (IndicesVec.getValueType().getVectorNumElements() > NumElts) {
8329     // Narrow/widen the indices vector to the correct size.
8330     if (IndicesVec.getValueSizeInBits() > SizeInBits)
8331       IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec),
8332                                     NumElts * VT.getScalarSizeInBits());
8333     else if (IndicesVec.getValueSizeInBits() < SizeInBits)
8334       IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG,
8335                                   SDLoc(IndicesVec), SizeInBits);
8336     // Zero-extend the index elements within the vector.
8337     if (IndicesVec.getValueType().getVectorNumElements() > NumElts)
8338       IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec),
8339                                IndicesVT, IndicesVec);
8340   }
8341   IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT);
8342 
8343   // Handle SrcVec that don't match VT type.
8344   if (SrcVec.getValueSizeInBits() != SizeInBits) {
8345     if ((SrcVec.getValueSizeInBits() % SizeInBits) == 0) {
8346       // Handle larger SrcVec by treating it as a larger permute.
8347       unsigned Scale = SrcVec.getValueSizeInBits() / SizeInBits;
8348       VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);
8349       IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8350       IndicesVec = widenSubVector(IndicesVT.getSimpleVT(), IndicesVec, false,
8351                                   Subtarget, DAG, SDLoc(IndicesVec));
8352       SDValue NewSrcVec =
8353           createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8354       if (NewSrcVec)
8355         return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits);
8356       return SDValue();
8357     } else if (SrcVec.getValueSizeInBits() < SizeInBits) {
8358       // Widen smaller SrcVec to match VT.
8359       SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));
8360     } else
8361       return SDValue();
8362   }
8363 
8364   auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
8365     assert(isPowerOf2_64(Scale) && "Illegal variable permute shuffle scale");
8366     EVT SrcVT = Idx.getValueType();
8367     unsigned NumDstBits = SrcVT.getScalarSizeInBits() / Scale;
8368     uint64_t IndexScale = 0;
8369     uint64_t IndexOffset = 0;
8370 
8371     // If we're scaling a smaller permute op, then we need to repeat the
8372     // indices, scaling and offsetting them as well.
8373     // e.g. v4i32 -> v16i8 (Scale = 4)
8374     // IndexScale = v4i32 Splat(4 << 24 | 4 << 16 | 4 << 8 | 4)
8375     // IndexOffset = v4i32 Splat(3 << 24 | 2 << 16 | 1 << 8 | 0)
8376     for (uint64_t i = 0; i != Scale; ++i) {
8377       IndexScale |= Scale << (i * NumDstBits);
8378       IndexOffset |= i << (i * NumDstBits);
8379     }
8380 
8381     Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx,
8382                       DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT));
8383     Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx,
8384                       DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT));
8385     return Idx;
8386   };
8387 
8388   unsigned Opcode = 0;
8389   switch (VT.SimpleTy) {
8390   default:
8391     break;
8392   case MVT::v16i8:
8393     if (Subtarget.hasSSSE3())
8394       Opcode = X86ISD::PSHUFB;
8395     break;
8396   case MVT::v8i16:
8397     if (Subtarget.hasVLX() && Subtarget.hasBWI())
8398       Opcode = X86ISD::VPERMV;
8399     else if (Subtarget.hasSSSE3()) {
8400       Opcode = X86ISD::PSHUFB;
8401       ShuffleVT = MVT::v16i8;
8402     }
8403     break;
8404   case MVT::v4f32:
8405   case MVT::v4i32:
8406     if (Subtarget.hasAVX()) {
8407       Opcode = X86ISD::VPERMILPV;
8408       ShuffleVT = MVT::v4f32;
8409     } else if (Subtarget.hasSSSE3()) {
8410       Opcode = X86ISD::PSHUFB;
8411       ShuffleVT = MVT::v16i8;
8412     }
8413     break;
8414   case MVT::v2f64:
8415   case MVT::v2i64:
8416     if (Subtarget.hasAVX()) {
8417       // VPERMILPD selects using bit#1 of the index vector, so scale IndicesVec.
8418       IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8419       Opcode = X86ISD::VPERMILPV;
8420       ShuffleVT = MVT::v2f64;
8421     } else if (Subtarget.hasSSE41()) {
8422       // SSE41 can compare v2i64 - select between indices 0 and 1.
8423       return DAG.getSelectCC(
8424           DL, IndicesVec,
8425           getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
8426           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
8427           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
8428           ISD::CondCode::SETEQ);
8429     }
8430     break;
8431   case MVT::v32i8:
8432     if (Subtarget.hasVLX() && Subtarget.hasVBMI())
8433       Opcode = X86ISD::VPERMV;
8434     else if (Subtarget.hasXOP()) {
8435       SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
8436       SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
8437       SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
8438       SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
8439       return DAG.getNode(
8440           ISD::CONCAT_VECTORS, DL, VT,
8441           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
8442           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
8443     } else if (Subtarget.hasAVX()) {
8444       SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
8445       SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
8446       SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
8447       SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
8448       auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
8449                               ArrayRef<SDValue> Ops) {
8450         // Permute Lo and Hi and then select based on index range.
8451         // This works as SHUFB uses bits[3:0] to permute elements and we don't
8452         // care about the bit[7] as its just an index vector.
8453         SDValue Idx = Ops[2];
8454         EVT VT = Idx.getValueType();
8455         return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
8456                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
8457                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
8458                                ISD::CondCode::SETGT);
8459       };
8460       SDValue Ops[] = {LoLo, HiHi, IndicesVec};
8461       return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
8462                               PSHUFBBuilder);
8463     }
8464     break;
8465   case MVT::v16i16:
8466     if (Subtarget.hasVLX() && Subtarget.hasBWI())
8467       Opcode = X86ISD::VPERMV;
8468     else if (Subtarget.hasAVX()) {
8469       // Scale to v32i8 and perform as v32i8.
8470       IndicesVec = ScaleIndices(IndicesVec, 2);
8471       return DAG.getBitcast(
8472           VT, createVariablePermute(
8473                   MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),
8474                   DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
8475     }
8476     break;
8477   case MVT::v8f32:
8478   case MVT::v8i32:
8479     if (Subtarget.hasAVX2())
8480       Opcode = X86ISD::VPERMV;
8481     else if (Subtarget.hasAVX()) {
8482       SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);
8483       SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8484                                           {0, 1, 2, 3, 0, 1, 2, 3});
8485       SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8486                                           {4, 5, 6, 7, 4, 5, 6, 7});
8487       if (Subtarget.hasXOP())
8488         return DAG.getBitcast(
8489             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
8490                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8491       // Permute Lo and Hi and then select based on index range.
8492       // This works as VPERMILPS only uses index bits[0:1] to permute elements.
8493       SDValue Res = DAG.getSelectCC(
8494           DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
8495           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
8496           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
8497           ISD::CondCode::SETGT);
8498       return DAG.getBitcast(VT, Res);
8499     }
8500     break;
8501   case MVT::v4i64:
8502   case MVT::v4f64:
8503     if (Subtarget.hasAVX512()) {
8504       if (!Subtarget.hasVLX()) {
8505         MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);
8506         SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG,
8507                                 SDLoc(SrcVec));
8508         IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget,
8509                                     DAG, SDLoc(IndicesVec));
8510         SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
8511                                             DAG, Subtarget);
8512         return extract256BitVector(Res, 0, DAG, DL);
8513       }
8514       Opcode = X86ISD::VPERMV;
8515     } else if (Subtarget.hasAVX()) {
8516       SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);
8517       SDValue LoLo =
8518           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
8519       SDValue HiHi =
8520           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
8521       // VPERMIL2PD selects with bit#1 of the index vector, so scale IndicesVec.
8522       IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8523       if (Subtarget.hasXOP())
8524         return DAG.getBitcast(
8525             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
8526                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8527       // Permute Lo and Hi and then select based on index range.
8528       // This works as VPERMILPD only uses index bit[1] to permute elements.
8529       SDValue Res = DAG.getSelectCC(
8530           DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
8531           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
8532           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
8533           ISD::CondCode::SETGT);
8534       return DAG.getBitcast(VT, Res);
8535     }
8536     break;
8537   case MVT::v64i8:
8538     if (Subtarget.hasVBMI())
8539       Opcode = X86ISD::VPERMV;
8540     break;
8541   case MVT::v32i16:
8542     if (Subtarget.hasBWI())
8543       Opcode = X86ISD::VPERMV;
8544     break;
8545   case MVT::v16f32:
8546   case MVT::v16i32:
8547   case MVT::v8f64:
8548   case MVT::v8i64:
8549     if (Subtarget.hasAVX512())
8550       Opcode = X86ISD::VPERMV;
8551     break;
8552   }
8553   if (!Opcode)
8554     return SDValue();
8555 
8556   assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
8557          (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 &&
8558          "Illegal variable permute shuffle type");
8559 
8560   uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits();
8561   if (Scale > 1)
8562     IndicesVec = ScaleIndices(IndicesVec, Scale);
8563 
8564   EVT ShuffleIdxVT = EVT(ShuffleVT).changeVectorElementTypeToInteger();
8565   IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec);
8566 
8567   SrcVec = DAG.getBitcast(ShuffleVT, SrcVec);
8568   SDValue Res = Opcode == X86ISD::VPERMV
8569                     ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
8570                     : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
8571   return DAG.getBitcast(VT, Res);
8572 }
8573 
8574 // Tries to lower a BUILD_VECTOR composed of extract-extract chains that can be
8575 // reasoned to be a permutation of a vector by indices in a non-constant vector.
8576 // (build_vector (extract_elt V, (extract_elt I, 0)),
8577 //               (extract_elt V, (extract_elt I, 1)),
8578 //                    ...
8579 // ->
8580 // (vpermv I, V)
8581 //
8582 // TODO: Handle undefs
8583 // TODO: Utilize pshufb and zero mask blending to support more efficient
8584 // construction of vectors with constant-0 elements.
8585 static SDValue
8586 LowerBUILD_VECTORAsVariablePermute(SDValue V, SelectionDAG &DAG,
8587                                    const X86Subtarget &Subtarget) {
8588   SDValue SrcVec, IndicesVec;
8589   // Check for a match of the permute source vector and permute index elements.
8590   // This is done by checking that the i-th build_vector operand is of the form:
8591   // (extract_elt SrcVec, (extract_elt IndicesVec, i)).
8592   for (unsigned Idx = 0, E = V.getNumOperands(); Idx != E; ++Idx) {
8593     SDValue Op = V.getOperand(Idx);
8594     if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8595       return SDValue();
8596 
8597     // If this is the first extract encountered in V, set the source vector,
8598     // otherwise verify the extract is from the previously defined source
8599     // vector.
8600     if (!SrcVec)
8601       SrcVec = Op.getOperand(0);
8602     else if (SrcVec != Op.getOperand(0))
8603       return SDValue();
8604     SDValue ExtractedIndex = Op->getOperand(1);
8605     // Peek through extends.
8606     if (ExtractedIndex.getOpcode() == ISD::ZERO_EXTEND ||
8607         ExtractedIndex.getOpcode() == ISD::SIGN_EXTEND)
8608       ExtractedIndex = ExtractedIndex.getOperand(0);
8609     if (ExtractedIndex.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8610       return SDValue();
8611 
8612     // If this is the first extract from the index vector candidate, set the
8613     // indices vector, otherwise verify the extract is from the previously
8614     // defined indices vector.
8615     if (!IndicesVec)
8616       IndicesVec = ExtractedIndex.getOperand(0);
8617     else if (IndicesVec != ExtractedIndex.getOperand(0))
8618       return SDValue();
8619 
8620     auto *PermIdx = dyn_cast<ConstantSDNode>(ExtractedIndex.getOperand(1));
8621     if (!PermIdx || PermIdx->getAPIntValue() != Idx)
8622       return SDValue();
8623   }
8624 
8625   SDLoc DL(V);
8626   MVT VT = V.getSimpleValueType();
8627   return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8628 }
8629 
8630 SDValue
8631 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
8632   SDLoc dl(Op);
8633 
8634   MVT VT = Op.getSimpleValueType();
8635   MVT EltVT = VT.getVectorElementType();
8636   MVT OpEltVT = Op.getOperand(0).getSimpleValueType();
8637   unsigned NumElems = Op.getNumOperands();
8638 
8639   // Generate vectors for predicate vectors.
8640   if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
8641     return LowerBUILD_VECTORvXi1(Op, DAG, Subtarget);
8642 
8643   if (VT.getVectorElementType() == MVT::bf16 &&
8644       (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16()))
8645     return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget);
8646 
8647   if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
8648     return VectorConstant;
8649 
8650   unsigned EVTBits = EltVT.getSizeInBits();
8651   APInt UndefMask = APInt::getZero(NumElems);
8652   APInt FrozenUndefMask = APInt::getZero(NumElems);
8653   APInt ZeroMask = APInt::getZero(NumElems);
8654   APInt NonZeroMask = APInt::getZero(NumElems);
8655   bool IsAllConstants = true;
8656   bool OneUseFrozenUndefs = true;
8657   SmallSet<SDValue, 8> Values;
8658   unsigned NumConstants = NumElems;
8659   for (unsigned i = 0; i < NumElems; ++i) {
8660     SDValue Elt = Op.getOperand(i);
8661     if (Elt.isUndef()) {
8662       UndefMask.setBit(i);
8663       continue;
8664     }
8665     if (ISD::isFreezeUndef(Elt.getNode())) {
8666       OneUseFrozenUndefs = OneUseFrozenUndefs && Elt->hasOneUse();
8667       FrozenUndefMask.setBit(i);
8668       continue;
8669     }
8670     Values.insert(Elt);
8671     if (!isIntOrFPConstant(Elt)) {
8672       IsAllConstants = false;
8673       NumConstants--;
8674     }
8675     if (X86::isZeroNode(Elt)) {
8676       ZeroMask.setBit(i);
8677     } else {
8678       NonZeroMask.setBit(i);
8679     }
8680   }
8681 
8682   // All undef vector. Return an UNDEF.
8683   if (UndefMask.isAllOnes())
8684     return DAG.getUNDEF(VT);
8685 
8686   // All undef/freeze(undef) vector. Return a FREEZE UNDEF.
8687   if (OneUseFrozenUndefs && (UndefMask | FrozenUndefMask).isAllOnes())
8688     return DAG.getFreeze(DAG.getUNDEF(VT));
8689 
8690   // All undef/freeze(undef)/zero vector. Return a zero vector.
8691   if ((UndefMask | FrozenUndefMask | ZeroMask).isAllOnes())
8692     return getZeroVector(VT, Subtarget, DAG, dl);
8693 
8694   // If we have multiple FREEZE-UNDEF operands, we are likely going to end up
8695   // lowering into a suboptimal insertion sequence. Instead, thaw the UNDEF in
8696   // our source BUILD_VECTOR, create another FREEZE-UNDEF splat BUILD_VECTOR,
8697   // and blend the FREEZE-UNDEF operands back in.
8698   // FIXME: is this worthwhile even for a single FREEZE-UNDEF operand?
8699   if (unsigned NumFrozenUndefElts = FrozenUndefMask.popcount();
8700       NumFrozenUndefElts >= 2 && NumFrozenUndefElts < NumElems) {
8701     SmallVector<int, 16> BlendMask(NumElems, -1);
8702     SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT));
8703     for (unsigned i = 0; i < NumElems; ++i) {
8704       if (UndefMask[i]) {
8705         BlendMask[i] = -1;
8706         continue;
8707       }
8708       BlendMask[i] = i;
8709       if (!FrozenUndefMask[i])
8710         Elts[i] = Op.getOperand(i);
8711       else
8712         BlendMask[i] += NumElems;
8713     }
8714     SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts);
8715     SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT));
8716     SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt);
8717     return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask);
8718   }
8719 
8720   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
8721 
8722   // If the upper elts of a ymm/zmm are undef/freeze(undef)/zero then we might
8723   // be better off lowering to a smaller build vector and padding with
8724   // undef/zero.
8725   if ((VT.is256BitVector() || VT.is512BitVector()) &&
8726       !isFoldableUseOfShuffle(BV)) {
8727     unsigned UpperElems = NumElems / 2;
8728     APInt UndefOrZeroMask = FrozenUndefMask | UndefMask | ZeroMask;
8729     unsigned NumUpperUndefsOrZeros = UndefOrZeroMask.countl_one();
8730     if (NumUpperUndefsOrZeros >= UpperElems) {
8731       if (VT.is512BitVector() &&
8732           NumUpperUndefsOrZeros >= (NumElems - (NumElems / 4)))
8733         UpperElems = NumElems - (NumElems / 4);
8734       // If freeze(undef) is in any upper elements, force to zero.
8735       bool UndefUpper = UndefMask.countl_one() >= UpperElems;
8736       MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems);
8737       SDValue NewBV =
8738           DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems));
8739       return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl);
8740     }
8741   }
8742 
8743   if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
8744     return AddSub;
8745   if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
8746     return HorizontalOp;
8747   if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
8748     return Broadcast;
8749   if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG))
8750     return BitOp;
8751 
8752   unsigned NumZero = ZeroMask.popcount();
8753   unsigned NumNonZero = NonZeroMask.popcount();
8754 
8755   // If we are inserting one variable into a vector of non-zero constants, try
8756   // to avoid loading each constant element as a scalar. Load the constants as a
8757   // vector and then insert the variable scalar element. If insertion is not
8758   // supported, fall back to a shuffle to get the scalar blended with the
8759   // constants. Insertion into a zero vector is handled as a special-case
8760   // somewhere below here.
8761   if (NumConstants == NumElems - 1 && NumNonZero != 1 &&
8762       FrozenUndefMask.isZero() &&
8763       (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) ||
8764        isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) {
8765     // Create an all-constant vector. The variable element in the old
8766     // build vector is replaced by undef in the constant vector. Save the
8767     // variable scalar element and its index for use in the insertelement.
8768     LLVMContext &Context = *DAG.getContext();
8769     Type *EltType = Op.getValueType().getScalarType().getTypeForEVT(Context);
8770     SmallVector<Constant *, 16> ConstVecOps(NumElems, UndefValue::get(EltType));
8771     SDValue VarElt;
8772     SDValue InsIndex;
8773     for (unsigned i = 0; i != NumElems; ++i) {
8774       SDValue Elt = Op.getOperand(i);
8775       if (auto *C = dyn_cast<ConstantSDNode>(Elt))
8776         ConstVecOps[i] = ConstantInt::get(Context, C->getAPIntValue());
8777       else if (auto *C = dyn_cast<ConstantFPSDNode>(Elt))
8778         ConstVecOps[i] = ConstantFP::get(Context, C->getValueAPF());
8779       else if (!Elt.isUndef()) {
8780         assert(!VarElt.getNode() && !InsIndex.getNode() &&
8781                "Expected one variable element in this vector");
8782         VarElt = Elt;
8783         InsIndex = DAG.getVectorIdxConstant(i, dl);
8784       }
8785     }
8786     Constant *CV = ConstantVector::get(ConstVecOps);
8787     SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
8788 
8789     // The constants we just created may not be legal (eg, floating point). We
8790     // must lower the vector right here because we can not guarantee that we'll
8791     // legalize it before loading it. This is also why we could not just create
8792     // a new build vector here. If the build vector contains illegal constants,
8793     // it could get split back up into a series of insert elements.
8794     // TODO: Improve this by using shorter loads with broadcast/VZEXT_LOAD.
8795     SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
8796     MachineFunction &MF = DAG.getMachineFunction();
8797     MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
8798     SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
8799     unsigned InsertC = InsIndex->getAsZExtVal();
8800     unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits();
8801     if (InsertC < NumEltsInLow128Bits)
8802       return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);
8803 
8804     // There's no good way to insert into the high elements of a >128-bit
8805     // vector, so use shuffles to avoid an extract/insert sequence.
8806     assert(VT.getSizeInBits() > 128 && "Invalid insertion index?");
8807     assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector");
8808     SmallVector<int, 8> ShuffleMask;
8809     unsigned NumElts = VT.getVectorNumElements();
8810     for (unsigned i = 0; i != NumElts; ++i)
8811       ShuffleMask.push_back(i == InsertC ? NumElts : i);
8812     SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
8813     return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);
8814   }
8815 
8816   // Special case for single non-zero, non-undef, element.
8817   if (NumNonZero == 1) {
8818     unsigned Idx = NonZeroMask.countr_zero();
8819     SDValue Item = Op.getOperand(Idx);
8820 
8821     // If we have a constant or non-constant insertion into the low element of
8822     // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
8823     // the rest of the elements.  This will be matched as movd/movq/movss/movsd
8824     // depending on what the source datatype is.
8825     if (Idx == 0) {
8826       if (NumZero == 0)
8827         return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8828 
8829       if (EltVT == MVT::i32 || EltVT == MVT::f16 || EltVT == MVT::f32 ||
8830           EltVT == MVT::f64 || (EltVT == MVT::i64 && Subtarget.is64Bit()) ||
8831           (EltVT == MVT::i16 && Subtarget.hasFP16())) {
8832         assert((VT.is128BitVector() || VT.is256BitVector() ||
8833                 VT.is512BitVector()) &&
8834                "Expected an SSE value type!");
8835         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8836         // Turn it into a MOVL (i.e. movsh, movss, movsd, movw or movd) to a
8837         // zero vector.
8838         return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8839       }
8840 
8841       // We can't directly insert an i8 or i16 into a vector, so zero extend
8842       // it to i32 first.
8843       if (EltVT == MVT::i16 || EltVT == MVT::i8) {
8844         Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
8845         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
8846         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item);
8847         Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8848         return DAG.getBitcast(VT, Item);
8849       }
8850     }
8851 
8852     // Is it a vector logical left shift?
8853     if (NumElems == 2 && Idx == 1 &&
8854         X86::isZeroNode(Op.getOperand(0)) &&
8855         !X86::isZeroNode(Op.getOperand(1))) {
8856       unsigned NumBits = VT.getSizeInBits();
8857       return getVShift(true, VT,
8858                        DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
8859                                    VT, Op.getOperand(1)),
8860                        NumBits/2, DAG, *this, dl);
8861     }
8862 
8863     if (IsAllConstants) // Otherwise, it's better to do a constpool load.
8864       return SDValue();
8865 
8866     // Otherwise, if this is a vector with i32 or f32 elements, and the element
8867     // is a non-constant being inserted into an element other than the low one,
8868     // we can't use a constant pool load.  Instead, use SCALAR_TO_VECTOR (aka
8869     // movd/movss) to move this into the low element, then shuffle it into
8870     // place.
8871     if (EVTBits == 32) {
8872       Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8873       return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
8874     }
8875   }
8876 
8877   // Splat is obviously ok. Let legalizer expand it to a shuffle.
8878   if (Values.size() == 1) {
8879     if (EVTBits == 32) {
8880       // Instead of a shuffle like this:
8881       // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
8882       // Check if it's possible to issue this instead.
8883       // shuffle (vload ptr)), undef, <1, 1, 1, 1>
8884       unsigned Idx = NonZeroMask.countr_zero();
8885       SDValue Item = Op.getOperand(Idx);
8886       if (Op.getNode()->isOnlyUserOf(Item.getNode()))
8887         return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
8888     }
8889     return SDValue();
8890   }
8891 
8892   // A vector full of immediates; various special cases are already
8893   // handled, so this is best done with a single constant-pool load.
8894   if (IsAllConstants)
8895     return SDValue();
8896 
8897   if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget))
8898       return V;
8899 
8900   // See if we can use a vector load to get all of the elements.
8901   {
8902     SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
8903     if (SDValue LD =
8904             EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
8905       return LD;
8906   }
8907 
8908   // If this is a splat of pairs of 32-bit elements, we can use a narrower
8909   // build_vector and broadcast it.
8910   // TODO: We could probably generalize this more.
8911   if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
8912     SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
8913                        DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
8914     auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
8915       // Make sure all the even/odd operands match.
8916       for (unsigned i = 2; i != NumElems; ++i)
8917         if (Ops[i % 2] != Op.getOperand(i))
8918           return false;
8919       return true;
8920     };
8921     if (CanSplat(Op, NumElems, Ops)) {
8922       MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;
8923       MVT NarrowVT = MVT::getVectorVT(EltVT, 4);
8924       // Create a new build vector and cast to v2i64/v2f64.
8925       SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
8926                                      DAG.getBuildVector(NarrowVT, dl, Ops));
8927       // Broadcast from v2i64/v2f64 and cast to final VT.
8928       MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);
8929       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
8930                                             NewBV));
8931     }
8932   }
8933 
8934   // For AVX-length vectors, build the individual 128-bit pieces and use
8935   // shuffles to put them in place.
8936   if (VT.getSizeInBits() > 128) {
8937     MVT HVT = MVT::getVectorVT(EltVT, NumElems / 2);
8938 
8939     // Build both the lower and upper subvector.
8940     SDValue Lower =
8941         DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
8942     SDValue Upper = DAG.getBuildVector(
8943         HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
8944 
8945     // Recreate the wider vector with the lower and upper part.
8946     return concatSubVectors(Lower, Upper, DAG, dl);
8947   }
8948 
8949   // Let legalizer expand 2-wide build_vectors.
8950   if (EVTBits == 64) {
8951     if (NumNonZero == 1) {
8952       // One half is zero or undef.
8953       unsigned Idx = NonZeroMask.countr_zero();
8954       SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
8955                                Op.getOperand(Idx));
8956       return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
8957     }
8958     return SDValue();
8959   }
8960 
8961   // If element VT is < 32 bits, convert it to inserts into a zero vector.
8962   if (EVTBits == 8 && NumElems == 16)
8963     if (SDValue V = LowerBuildVectorv16i8(Op, NonZeroMask, NumNonZero, NumZero,
8964                                           DAG, Subtarget))
8965       return V;
8966 
8967   if (EltVT == MVT::i16 && NumElems == 8)
8968     if (SDValue V = LowerBuildVectorv8i16(Op, NonZeroMask, NumNonZero, NumZero,
8969                                           DAG, Subtarget))
8970       return V;
8971 
8972   // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
8973   if (EVTBits == 32 && NumElems == 4)
8974     if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget))
8975       return V;
8976 
8977   // If element VT is == 32 bits, turn it into a number of shuffles.
8978   if (NumElems == 4 && NumZero > 0) {
8979     SmallVector<SDValue, 8> Ops(NumElems);
8980     for (unsigned i = 0; i < 4; ++i) {
8981       bool isZero = !NonZeroMask[i];
8982       if (isZero)
8983         Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);
8984       else
8985         Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
8986     }
8987 
8988     for (unsigned i = 0; i < 2; ++i) {
8989       switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) {
8990         default: llvm_unreachable("Unexpected NonZero count");
8991         case 0:
8992           Ops[i] = Ops[i*2];  // Must be a zero vector.
8993           break;
8994         case 1:
8995           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
8996           break;
8997         case 2:
8998           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
8999           break;
9000         case 3:
9001           Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9002           break;
9003       }
9004     }
9005 
9006     bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2;
9007     bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2;
9008     int MaskVec[] = {
9009       Reverse1 ? 1 : 0,
9010       Reverse1 ? 0 : 1,
9011       static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
9012       static_cast<int>(Reverse2 ? NumElems   : NumElems+1)
9013     };
9014     return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
9015   }
9016 
9017   assert(Values.size() > 1 && "Expected non-undef and non-splat vector");
9018 
9019   // Check for a build vector from mostly shuffle plus few inserting.
9020   if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
9021     return Sh;
9022 
9023   // For SSE 4.1, use insertps to put the high elements into the low element.
9024   if (Subtarget.hasSSE41() && EltVT != MVT::f16) {
9025     SDValue Result;
9026     if (!Op.getOperand(0).isUndef())
9027       Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
9028     else
9029       Result = DAG.getUNDEF(VT);
9030 
9031     for (unsigned i = 1; i < NumElems; ++i) {
9032       if (Op.getOperand(i).isUndef()) continue;
9033       Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
9034                            Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
9035     }
9036     return Result;
9037   }
9038 
9039   // Otherwise, expand into a number of unpckl*, start by extending each of
9040   // our (non-undef) elements to the full vector width with the element in the
9041   // bottom slot of the vector (which generates no code for SSE).
9042   SmallVector<SDValue, 8> Ops(NumElems);
9043   for (unsigned i = 0; i < NumElems; ++i) {
9044     if (!Op.getOperand(i).isUndef())
9045       Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9046     else
9047       Ops[i] = DAG.getUNDEF(VT);
9048   }
9049 
9050   // Next, we iteratively mix elements, e.g. for v4f32:
9051   //   Step 1: unpcklps 0, 1 ==> X: <?, ?, 1, 0>
9052   //         : unpcklps 2, 3 ==> Y: <?, ?, 3, 2>
9053   //   Step 2: unpcklpd X, Y ==>    <3, 2, 1, 0>
9054   for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
9055     // Generate scaled UNPCKL shuffle mask.
9056     SmallVector<int, 16> Mask;
9057     for(unsigned i = 0; i != Scale; ++i)
9058       Mask.push_back(i);
9059     for (unsigned i = 0; i != Scale; ++i)
9060       Mask.push_back(NumElems+i);
9061     Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
9062 
9063     for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
9064       Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
9065   }
9066   return Ops[0];
9067 }
9068 
9069 // 256-bit AVX can use the vinsertf128 instruction
9070 // to create 256-bit vectors from two other 128-bit ones.
9071 // TODO: Detect subvector broadcast here instead of DAG combine?
9072 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9073                                       const X86Subtarget &Subtarget) {
9074   SDLoc dl(Op);
9075   MVT ResVT = Op.getSimpleValueType();
9076 
9077   assert((ResVT.is256BitVector() ||
9078           ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
9079 
9080   unsigned NumOperands = Op.getNumOperands();
9081   unsigned NumFreezeUndef = 0;
9082   unsigned NumZero = 0;
9083   unsigned NumNonZero = 0;
9084   unsigned NonZeros = 0;
9085   for (unsigned i = 0; i != NumOperands; ++i) {
9086     SDValue SubVec = Op.getOperand(i);
9087     if (SubVec.isUndef())
9088       continue;
9089     if (ISD::isFreezeUndef(SubVec.getNode())) {
9090         // If the freeze(undef) has multiple uses then we must fold to zero.
9091         if (SubVec.hasOneUse())
9092           ++NumFreezeUndef;
9093         else
9094           ++NumZero;
9095     }
9096     else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9097       ++NumZero;
9098     else {
9099       assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9100       NonZeros |= 1 << i;
9101       ++NumNonZero;
9102     }
9103   }
9104 
9105   // If we have more than 2 non-zeros, build each half separately.
9106   if (NumNonZero > 2) {
9107     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9108     ArrayRef<SDUse> Ops = Op->ops();
9109     SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9110                              Ops.slice(0, NumOperands/2));
9111     SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9112                              Ops.slice(NumOperands/2));
9113     return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9114   }
9115 
9116   // Otherwise, build it up through insert_subvectors.
9117   SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
9118                         : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
9119                                           : DAG.getUNDEF(ResVT));
9120 
9121   MVT SubVT = Op.getOperand(0).getSimpleValueType();
9122   unsigned NumSubElems = SubVT.getVectorNumElements();
9123   for (unsigned i = 0; i != NumOperands; ++i) {
9124     if ((NonZeros & (1 << i)) == 0)
9125       continue;
9126 
9127     Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec,
9128                       Op.getOperand(i),
9129                       DAG.getIntPtrConstant(i * NumSubElems, dl));
9130   }
9131 
9132   return Vec;
9133 }
9134 
9135 // Returns true if the given node is a type promotion (by concatenating i1
9136 // zeros) of the result of a node that already zeros all upper bits of
9137 // k-register.
9138 // TODO: Merge this with LowerAVXCONCAT_VECTORS?
9139 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
9140                                        const X86Subtarget &Subtarget,
9141                                        SelectionDAG & DAG) {
9142   SDLoc dl(Op);
9143   MVT ResVT = Op.getSimpleValueType();
9144   unsigned NumOperands = Op.getNumOperands();
9145 
9146   assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
9147          "Unexpected number of operands in CONCAT_VECTORS");
9148 
9149   uint64_t Zeros = 0;
9150   uint64_t NonZeros = 0;
9151   for (unsigned i = 0; i != NumOperands; ++i) {
9152     SDValue SubVec = Op.getOperand(i);
9153     if (SubVec.isUndef())
9154       continue;
9155     assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9156     if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9157       Zeros |= (uint64_t)1 << i;
9158     else
9159       NonZeros |= (uint64_t)1 << i;
9160   }
9161 
9162   unsigned NumElems = ResVT.getVectorNumElements();
9163 
9164   // If we are inserting non-zero vector and there are zeros in LSBs and undef
9165   // in the MSBs we need to emit a KSHIFTL. The generic lowering to
9166   // insert_subvector will give us two kshifts.
9167   if (isPowerOf2_64(NonZeros) && Zeros != 0 && NonZeros > Zeros &&
9168       Log2_64(NonZeros) != NumOperands - 1) {
9169     unsigned Idx = Log2_64(NonZeros);
9170     SDValue SubVec = Op.getOperand(Idx);
9171     unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9172     MVT ShiftVT = widenMaskVectorType(ResVT, Subtarget);
9173     Op = widenSubVector(ShiftVT, SubVec, false, Subtarget, DAG, dl);
9174     Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, Op,
9175                      DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
9176     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
9177                        DAG.getIntPtrConstant(0, dl));
9178   }
9179 
9180   // If there are zero or one non-zeros we can handle this very simply.
9181   if (NonZeros == 0 || isPowerOf2_64(NonZeros)) {
9182     SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
9183     if (!NonZeros)
9184       return Vec;
9185     unsigned Idx = Log2_64(NonZeros);
9186     SDValue SubVec = Op.getOperand(Idx);
9187     unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9188     return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec,
9189                        DAG.getIntPtrConstant(Idx * SubVecNumElts, dl));
9190   }
9191 
9192   if (NumOperands > 2) {
9193     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9194     ArrayRef<SDUse> Ops = Op->ops();
9195     SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9196                              Ops.slice(0, NumOperands/2));
9197     SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9198                              Ops.slice(NumOperands/2));
9199     return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9200   }
9201 
9202   assert(llvm::popcount(NonZeros) == 2 && "Simple cases not handled?");
9203 
9204   if (ResVT.getVectorNumElements() >= 16)
9205     return Op; // The operation is legal with KUNPCK
9206 
9207   SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
9208                             DAG.getUNDEF(ResVT), Op.getOperand(0),
9209                             DAG.getIntPtrConstant(0, dl));
9210   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1),
9211                      DAG.getIntPtrConstant(NumElems/2, dl));
9212 }
9213 
9214 static SDValue LowerCONCAT_VECTORS(SDValue Op,
9215                                    const X86Subtarget &Subtarget,
9216                                    SelectionDAG &DAG) {
9217   MVT VT = Op.getSimpleValueType();
9218   if (VT.getVectorElementType() == MVT::i1)
9219     return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
9220 
9221   assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
9222          (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
9223           Op.getNumOperands() == 4)));
9224 
9225   // AVX can use the vinsertf128 instruction to create 256-bit vectors
9226   // from two other 128-bit ones.
9227 
9228   // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
9229   return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
9230 }
9231 
9232 //===----------------------------------------------------------------------===//
9233 // Vector shuffle lowering
9234 //
9235 // This is an experimental code path for lowering vector shuffles on x86. It is
9236 // designed to handle arbitrary vector shuffles and blends, gracefully
9237 // degrading performance as necessary. It works hard to recognize idiomatic
9238 // shuffles and lower them to optimal instruction patterns without leaving
9239 // a framework that allows reasonably efficient handling of all vector shuffle
9240 // patterns.
9241 //===----------------------------------------------------------------------===//
9242 
9243 /// Tiny helper function to identify a no-op mask.
9244 ///
9245 /// This is a somewhat boring predicate function. It checks whether the mask
9246 /// array input, which is assumed to be a single-input shuffle mask of the kind
9247 /// used by the X86 shuffle instructions (not a fully general
9248 /// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
9249 /// in-place shuffle are 'no-op's.
9250 static bool isNoopShuffleMask(ArrayRef<int> Mask) {
9251   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
9252     assert(Mask[i] >= -1 && "Out of bound mask element!");
9253     if (Mask[i] >= 0 && Mask[i] != i)
9254       return false;
9255   }
9256   return true;
9257 }
9258 
9259 /// Test whether there are elements crossing LaneSizeInBits lanes in this
9260 /// shuffle mask.
9261 ///
9262 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
9263 /// and we routinely test for these.
9264 static bool isLaneCrossingShuffleMask(unsigned LaneSizeInBits,
9265                                       unsigned ScalarSizeInBits,
9266                                       ArrayRef<int> Mask) {
9267   assert(LaneSizeInBits && ScalarSizeInBits &&
9268          (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9269          "Illegal shuffle lane size");
9270   int LaneSize = LaneSizeInBits / ScalarSizeInBits;
9271   int Size = Mask.size();
9272   for (int i = 0; i < Size; ++i)
9273     if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
9274       return true;
9275   return false;
9276 }
9277 
9278 /// Test whether there are elements crossing 128-bit lanes in this
9279 /// shuffle mask.
9280 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
9281   return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask);
9282 }
9283 
9284 /// Test whether elements in each LaneSizeInBits lane in this shuffle mask come
9285 /// from multiple lanes - this is different to isLaneCrossingShuffleMask to
9286 /// better support 'repeated mask + lane permute' style shuffles.
9287 static bool isMultiLaneShuffleMask(unsigned LaneSizeInBits,
9288                                    unsigned ScalarSizeInBits,
9289                                    ArrayRef<int> Mask) {
9290   assert(LaneSizeInBits && ScalarSizeInBits &&
9291          (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9292          "Illegal shuffle lane size");
9293   int NumElts = Mask.size();
9294   int NumEltsPerLane = LaneSizeInBits / ScalarSizeInBits;
9295   int NumLanes = NumElts / NumEltsPerLane;
9296   if (NumLanes > 1) {
9297     for (int i = 0; i != NumLanes; ++i) {
9298       int SrcLane = -1;
9299       for (int j = 0; j != NumEltsPerLane; ++j) {
9300         int M = Mask[(i * NumEltsPerLane) + j];
9301         if (M < 0)
9302           continue;
9303         int Lane = (M % NumElts) / NumEltsPerLane;
9304         if (SrcLane >= 0 && SrcLane != Lane)
9305           return true;
9306         SrcLane = Lane;
9307       }
9308     }
9309   }
9310   return false;
9311 }
9312 
9313 /// Test whether a shuffle mask is equivalent within each sub-lane.
9314 ///
9315 /// This checks a shuffle mask to see if it is performing the same
9316 /// lane-relative shuffle in each sub-lane. This trivially implies
9317 /// that it is also not lane-crossing. It may however involve a blend from the
9318 /// same lane of a second vector.
9319 ///
9320 /// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
9321 /// non-trivial to compute in the face of undef lanes. The representation is
9322 /// suitable for use with existing 128-bit shuffles as entries from the second
9323 /// vector have been remapped to [LaneSize, 2*LaneSize).
9324 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
9325                                   ArrayRef<int> Mask,
9326                                   SmallVectorImpl<int> &RepeatedMask) {
9327   auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
9328   RepeatedMask.assign(LaneSize, -1);
9329   int Size = Mask.size();
9330   for (int i = 0; i < Size; ++i) {
9331     assert(Mask[i] == SM_SentinelUndef || Mask[i] >= 0);
9332     if (Mask[i] < 0)
9333       continue;
9334     if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9335       // This entry crosses lanes, so there is no way to model this shuffle.
9336       return false;
9337 
9338     // Ok, handle the in-lane shuffles by detecting if and when they repeat.
9339     // Adjust second vector indices to start at LaneSize instead of Size.
9340     int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
9341                                 : Mask[i] % LaneSize + LaneSize;
9342     if (RepeatedMask[i % LaneSize] < 0)
9343       // This is the first non-undef entry in this slot of a 128-bit lane.
9344       RepeatedMask[i % LaneSize] = LocalM;
9345     else if (RepeatedMask[i % LaneSize] != LocalM)
9346       // Found a mismatch with the repeated mask.
9347       return false;
9348   }
9349   return true;
9350 }
9351 
9352 /// Test whether a shuffle mask is equivalent within each 128-bit lane.
9353 static bool
9354 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9355                                 SmallVectorImpl<int> &RepeatedMask) {
9356   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9357 }
9358 
9359 static bool
9360 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
9361   SmallVector<int, 32> RepeatedMask;
9362   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9363 }
9364 
9365 /// Test whether a shuffle mask is equivalent within each 256-bit lane.
9366 static bool
9367 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9368                                 SmallVectorImpl<int> &RepeatedMask) {
9369   return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask);
9370 }
9371 
9372 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9373 /// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9374 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits,
9375                                         unsigned EltSizeInBits,
9376                                         ArrayRef<int> Mask,
9377                                         SmallVectorImpl<int> &RepeatedMask) {
9378   int LaneSize = LaneSizeInBits / EltSizeInBits;
9379   RepeatedMask.assign(LaneSize, SM_SentinelUndef);
9380   int Size = Mask.size();
9381   for (int i = 0; i < Size; ++i) {
9382     assert(isUndefOrZero(Mask[i]) || (Mask[i] >= 0));
9383     if (Mask[i] == SM_SentinelUndef)
9384       continue;
9385     if (Mask[i] == SM_SentinelZero) {
9386       if (!isUndefOrZero(RepeatedMask[i % LaneSize]))
9387         return false;
9388       RepeatedMask[i % LaneSize] = SM_SentinelZero;
9389       continue;
9390     }
9391     if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9392       // This entry crosses lanes, so there is no way to model this shuffle.
9393       return false;
9394 
9395     // Handle the in-lane shuffles by detecting if and when they repeat. Adjust
9396     // later vector indices to start at multiples of LaneSize instead of Size.
9397     int LaneM = Mask[i] / Size;
9398     int LocalM = (Mask[i] % LaneSize) + (LaneM * LaneSize);
9399     if (RepeatedMask[i % LaneSize] == SM_SentinelUndef)
9400       // This is the first non-undef entry in this slot of a 128-bit lane.
9401       RepeatedMask[i % LaneSize] = LocalM;
9402     else if (RepeatedMask[i % LaneSize] != LocalM)
9403       // Found a mismatch with the repeated mask.
9404       return false;
9405   }
9406   return true;
9407 }
9408 
9409 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9410 /// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9411 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,
9412                                         ArrayRef<int> Mask,
9413                                         SmallVectorImpl<int> &RepeatedMask) {
9414   return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(),
9415                                      Mask, RepeatedMask);
9416 }
9417 
9418 /// Checks whether the vector elements referenced by two shuffle masks are
9419 /// equivalent.
9420 static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
9421                                 int Idx, int ExpectedIdx) {
9422   assert(0 <= Idx && Idx < MaskSize && 0 <= ExpectedIdx &&
9423          ExpectedIdx < MaskSize && "Out of range element index");
9424   if (!Op || !ExpectedOp || Op.getOpcode() != ExpectedOp.getOpcode())
9425     return false;
9426 
9427   switch (Op.getOpcode()) {
9428   case ISD::BUILD_VECTOR:
9429     // If the values are build vectors, we can look through them to find
9430     // equivalent inputs that make the shuffles equivalent.
9431     // TODO: Handle MaskSize != Op.getNumOperands()?
9432     if (MaskSize == (int)Op.getNumOperands() &&
9433         MaskSize == (int)ExpectedOp.getNumOperands())
9434       return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx);
9435     break;
9436   case X86ISD::VBROADCAST:
9437   case X86ISD::VBROADCAST_LOAD:
9438     // TODO: Handle MaskSize != Op.getValueType().getVectorNumElements()?
9439     return (Op == ExpectedOp &&
9440             (int)Op.getValueType().getVectorNumElements() == MaskSize);
9441   case X86ISD::HADD:
9442   case X86ISD::HSUB:
9443   case X86ISD::FHADD:
9444   case X86ISD::FHSUB:
9445   case X86ISD::PACKSS:
9446   case X86ISD::PACKUS:
9447     // HOP(X,X) can refer to the elt from the lower/upper half of a lane.
9448     // TODO: Handle MaskSize != NumElts?
9449     // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases.
9450     if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) {
9451       MVT VT = Op.getSimpleValueType();
9452       int NumElts = VT.getVectorNumElements();
9453       if (MaskSize == NumElts) {
9454         int NumLanes = VT.getSizeInBits() / 128;
9455         int NumEltsPerLane = NumElts / NumLanes;
9456         int NumHalfEltsPerLane = NumEltsPerLane / 2;
9457         bool SameLane =
9458             (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane);
9459         bool SameElt =
9460             (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane);
9461         return SameLane && SameElt;
9462       }
9463     }
9464     break;
9465   }
9466 
9467   return false;
9468 }
9469 
9470 /// Checks whether a shuffle mask is equivalent to an explicit list of
9471 /// arguments.
9472 ///
9473 /// This is a fast way to test a shuffle mask against a fixed pattern:
9474 ///
9475 ///   if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
9476 ///
9477 /// It returns true if the mask is exactly as wide as the argument list, and
9478 /// each element of the mask is either -1 (signifying undef) or the value given
9479 /// in the argument.
9480 static bool isShuffleEquivalent(ArrayRef<int> Mask, ArrayRef<int> ExpectedMask,
9481                                 SDValue V1 = SDValue(),
9482                                 SDValue V2 = SDValue()) {
9483   int Size = Mask.size();
9484   if (Size != (int)ExpectedMask.size())
9485     return false;
9486 
9487   for (int i = 0; i < Size; ++i) {
9488     assert(Mask[i] >= -1 && "Out of bound mask element!");
9489     int MaskIdx = Mask[i];
9490     int ExpectedIdx = ExpectedMask[i];
9491     if (0 <= MaskIdx && MaskIdx != ExpectedIdx) {
9492       SDValue MaskV = MaskIdx < Size ? V1 : V2;
9493       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9494       MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9495       ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9496       if (!IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9497         return false;
9498     }
9499   }
9500   return true;
9501 }
9502 
9503 /// Checks whether a target shuffle mask is equivalent to an explicit pattern.
9504 ///
9505 /// The masks must be exactly the same width.
9506 ///
9507 /// If an element in Mask matches SM_SentinelUndef (-1) then the corresponding
9508 /// value in ExpectedMask is always accepted. Otherwise the indices must match.
9509 ///
9510 /// SM_SentinelZero is accepted as a valid negative index but must match in
9511 /// both, or via a known bits test.
9512 static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask,
9513                                       ArrayRef<int> ExpectedMask,
9514                                       const SelectionDAG &DAG,
9515                                       SDValue V1 = SDValue(),
9516                                       SDValue V2 = SDValue()) {
9517   int Size = Mask.size();
9518   if (Size != (int)ExpectedMask.size())
9519     return false;
9520   assert(llvm::all_of(ExpectedMask,
9521                       [Size](int M) { return isInRange(M, 0, 2 * Size); }) &&
9522          "Illegal target shuffle mask");
9523 
9524   // Check for out-of-range target shuffle mask indices.
9525   if (!isUndefOrZeroOrInRange(Mask, 0, 2 * Size))
9526     return false;
9527 
9528   // Don't use V1/V2 if they're not the same size as the shuffle mask type.
9529   if (V1 && (V1.getValueSizeInBits() != VT.getSizeInBits() ||
9530              !V1.getValueType().isVector()))
9531     V1 = SDValue();
9532   if (V2 && (V2.getValueSizeInBits() != VT.getSizeInBits() ||
9533              !V2.getValueType().isVector()))
9534     V2 = SDValue();
9535 
9536   APInt ZeroV1 = APInt::getZero(Size);
9537   APInt ZeroV2 = APInt::getZero(Size);
9538 
9539   for (int i = 0; i < Size; ++i) {
9540     int MaskIdx = Mask[i];
9541     int ExpectedIdx = ExpectedMask[i];
9542     if (MaskIdx == SM_SentinelUndef || MaskIdx == ExpectedIdx)
9543       continue;
9544     if (MaskIdx == SM_SentinelZero) {
9545       // If we need this expected index to be a zero element, then update the
9546       // relevant zero mask and perform the known bits at the end to minimize
9547       // repeated computes.
9548       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9549       if (ExpectedV &&
9550           Size == (int)ExpectedV.getValueType().getVectorNumElements()) {
9551         int BitIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9552         APInt &ZeroMask = ExpectedIdx < Size ? ZeroV1 : ZeroV2;
9553         ZeroMask.setBit(BitIdx);
9554         continue;
9555       }
9556     }
9557     if (MaskIdx >= 0) {
9558       SDValue MaskV = MaskIdx < Size ? V1 : V2;
9559       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9560       MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9561       ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9562       if (IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9563         continue;
9564     }
9565     return false;
9566   }
9567   return (ZeroV1.isZero() || DAG.MaskedVectorIsZero(V1, ZeroV1)) &&
9568          (ZeroV2.isZero() || DAG.MaskedVectorIsZero(V2, ZeroV2));
9569 }
9570 
9571 // Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd
9572 // instructions.
9573 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT,
9574                                   const SelectionDAG &DAG) {
9575   if (VT != MVT::v8i32 && VT != MVT::v8f32)
9576     return false;
9577 
9578   SmallVector<int, 8> Unpcklwd;
9579   createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true,
9580                           /* Unary = */ false);
9581   SmallVector<int, 8> Unpckhwd;
9582   createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false,
9583                           /* Unary = */ false);
9584   bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) ||
9585                          isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG));
9586   return IsUnpackwdMask;
9587 }
9588 
9589 static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask,
9590                                       const SelectionDAG &DAG) {
9591   // Create 128-bit vector type based on mask size.
9592   MVT EltVT = MVT::getIntegerVT(128 / Mask.size());
9593   MVT VT = MVT::getVectorVT(EltVT, Mask.size());
9594 
9595   // We can't assume a canonical shuffle mask, so try the commuted version too.
9596   SmallVector<int, 4> CommutedMask(Mask);
9597   ShuffleVectorSDNode::commuteMask(CommutedMask);
9598 
9599   // Match any of unary/binary or low/high.
9600   for (unsigned i = 0; i != 4; ++i) {
9601     SmallVector<int, 16> UnpackMask;
9602     createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2);
9603     if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) ||
9604         isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG))
9605       return true;
9606   }
9607   return false;
9608 }
9609 
9610 /// Return true if a shuffle mask chooses elements identically in its top and
9611 /// bottom halves. For example, any splat mask has the same top and bottom
9612 /// halves. If an element is undefined in only one half of the mask, the halves
9613 /// are not considered identical.
9614 static bool hasIdenticalHalvesShuffleMask(ArrayRef<int> Mask) {
9615   assert(Mask.size() % 2 == 0 && "Expecting even number of elements in mask");
9616   unsigned HalfSize = Mask.size() / 2;
9617   for (unsigned i = 0; i != HalfSize; ++i) {
9618     if (Mask[i] != Mask[i + HalfSize])
9619       return false;
9620   }
9621   return true;
9622 }
9623 
9624 /// Get a 4-lane 8-bit shuffle immediate for a mask.
9625 ///
9626 /// This helper function produces an 8-bit shuffle immediate corresponding to
9627 /// the ubiquitous shuffle encoding scheme used in x86 instructions for
9628 /// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
9629 /// example.
9630 ///
9631 /// NB: We rely heavily on "undef" masks preserving the input lane.
9632 static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
9633   assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
9634   assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
9635   assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
9636   assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
9637   assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
9638 
9639   // If the mask only uses one non-undef element, then fully 'splat' it to
9640   // improve later broadcast matching.
9641   int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin();
9642   assert(0 <= FirstIndex && FirstIndex < 4 && "All undef shuffle mask");
9643 
9644   int FirstElt = Mask[FirstIndex];
9645   if (all_of(Mask, [FirstElt](int M) { return M < 0 || M == FirstElt; }))
9646     return (FirstElt << 6) | (FirstElt << 4) | (FirstElt << 2) | FirstElt;
9647 
9648   unsigned Imm = 0;
9649   Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0;
9650   Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2;
9651   Imm |= (Mask[2] < 0 ? 2 : Mask[2]) << 4;
9652   Imm |= (Mask[3] < 0 ? 3 : Mask[3]) << 6;
9653   return Imm;
9654 }
9655 
9656 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
9657                                           SelectionDAG &DAG) {
9658   return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
9659 }
9660 
9661 // The Shuffle result is as follow:
9662 // 0*a[0]0*a[1]...0*a[n] , n >=0 where a[] elements in a ascending order.
9663 // Each Zeroable's element correspond to a particular Mask's element.
9664 // As described in computeZeroableShuffleElements function.
9665 //
9666 // The function looks for a sub-mask that the nonzero elements are in
9667 // increasing order. If such sub-mask exist. The function returns true.
9668 static bool isNonZeroElementsInOrder(const APInt &Zeroable,
9669                                      ArrayRef<int> Mask, const EVT &VectorType,
9670                                      bool &IsZeroSideLeft) {
9671   int NextElement = -1;
9672   // Check if the Mask's nonzero elements are in increasing order.
9673   for (int i = 0, e = Mask.size(); i < e; i++) {
9674     // Checks if the mask's zeros elements are built from only zeros.
9675     assert(Mask[i] >= -1 && "Out of bound mask element!");
9676     if (Mask[i] < 0)
9677       return false;
9678     if (Zeroable[i])
9679       continue;
9680     // Find the lowest non zero element
9681     if (NextElement < 0) {
9682       NextElement = Mask[i] != 0 ? VectorType.getVectorNumElements() : 0;
9683       IsZeroSideLeft = NextElement != 0;
9684     }
9685     // Exit if the mask's non zero elements are not in increasing order.
9686     if (NextElement != Mask[i])
9687       return false;
9688     NextElement++;
9689   }
9690   return true;
9691 }
9692 
9693 /// Try to lower a shuffle with a single PSHUFB of V1 or V2.
9694 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
9695                                       ArrayRef<int> Mask, SDValue V1,
9696                                       SDValue V2, const APInt &Zeroable,
9697                                       const X86Subtarget &Subtarget,
9698                                       SelectionDAG &DAG) {
9699   int Size = Mask.size();
9700   int LaneSize = 128 / VT.getScalarSizeInBits();
9701   const int NumBytes = VT.getSizeInBits() / 8;
9702   const int NumEltBytes = VT.getScalarSizeInBits() / 8;
9703 
9704   assert((Subtarget.hasSSSE3() && VT.is128BitVector()) ||
9705          (Subtarget.hasAVX2() && VT.is256BitVector()) ||
9706          (Subtarget.hasBWI() && VT.is512BitVector()));
9707 
9708   SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
9709   // Sign bit set in i8 mask means zero element.
9710   SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
9711 
9712   SDValue V;
9713   for (int i = 0; i < NumBytes; ++i) {
9714     int M = Mask[i / NumEltBytes];
9715     if (M < 0) {
9716       PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);
9717       continue;
9718     }
9719     if (Zeroable[i / NumEltBytes]) {
9720       PSHUFBMask[i] = ZeroMask;
9721       continue;
9722     }
9723 
9724     // We can only use a single input of V1 or V2.
9725     SDValue SrcV = (M >= Size ? V2 : V1);
9726     if (V && V != SrcV)
9727       return SDValue();
9728     V = SrcV;
9729     M %= Size;
9730 
9731     // PSHUFB can't cross lanes, ensure this doesn't happen.
9732     if ((M / LaneSize) != ((i / NumEltBytes) / LaneSize))
9733       return SDValue();
9734 
9735     M = M % LaneSize;
9736     M = M * NumEltBytes + (i % NumEltBytes);
9737     PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
9738   }
9739   assert(V && "Failed to find a source input");
9740 
9741   MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes);
9742   return DAG.getBitcast(
9743       VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
9744                       DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
9745 }
9746 
9747 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
9748                            const X86Subtarget &Subtarget, SelectionDAG &DAG,
9749                            const SDLoc &dl);
9750 
9751 // X86 has dedicated shuffle that can be lowered to VEXPAND
9752 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
9753                                     const APInt &Zeroable,
9754                                     ArrayRef<int> Mask, SDValue &V1,
9755                                     SDValue &V2, SelectionDAG &DAG,
9756                                     const X86Subtarget &Subtarget) {
9757   bool IsLeftZeroSide = true;
9758   if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
9759                                 IsLeftZeroSide))
9760     return SDValue();
9761   unsigned VEXPANDMask = (~Zeroable).getZExtValue();
9762   MVT IntegerType =
9763       MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
9764   SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
9765   unsigned NumElts = VT.getVectorNumElements();
9766   assert((NumElts == 4 || NumElts == 8 || NumElts == 16) &&
9767          "Unexpected number of vector elements");
9768   SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
9769                               Subtarget, DAG, DL);
9770   SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
9771   SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
9772   return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
9773 }
9774 
9775 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
9776                                   unsigned &UnpackOpcode, bool IsUnary,
9777                                   ArrayRef<int> TargetMask, const SDLoc &DL,
9778                                   SelectionDAG &DAG,
9779                                   const X86Subtarget &Subtarget) {
9780   int NumElts = VT.getVectorNumElements();
9781 
9782   bool Undef1 = true, Undef2 = true, Zero1 = true, Zero2 = true;
9783   for (int i = 0; i != NumElts; i += 2) {
9784     int M1 = TargetMask[i + 0];
9785     int M2 = TargetMask[i + 1];
9786     Undef1 &= (SM_SentinelUndef == M1);
9787     Undef2 &= (SM_SentinelUndef == M2);
9788     Zero1 &= isUndefOrZero(M1);
9789     Zero2 &= isUndefOrZero(M2);
9790   }
9791   assert(!((Undef1 || Zero1) && (Undef2 || Zero2)) &&
9792          "Zeroable shuffle detected");
9793 
9794   // Attempt to match the target mask against the unpack lo/hi mask patterns.
9795   SmallVector<int, 64> Unpckl, Unpckh;
9796   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary);
9797   if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1,
9798                                 (IsUnary ? V1 : V2))) {
9799     UnpackOpcode = X86ISD::UNPCKL;
9800     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9801     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9802     return true;
9803   }
9804 
9805   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary);
9806   if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1,
9807                                 (IsUnary ? V1 : V2))) {
9808     UnpackOpcode = X86ISD::UNPCKH;
9809     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9810     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9811     return true;
9812   }
9813 
9814   // If an unary shuffle, attempt to match as an unpack lo/hi with zero.
9815   if (IsUnary && (Zero1 || Zero2)) {
9816     // Don't bother if we can blend instead.
9817     if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&
9818         isSequentialOrUndefOrZeroInRange(TargetMask, 0, NumElts, 0))
9819       return false;
9820 
9821     bool MatchLo = true, MatchHi = true;
9822     for (int i = 0; (i != NumElts) && (MatchLo || MatchHi); ++i) {
9823       int M = TargetMask[i];
9824 
9825       // Ignore if the input is known to be zero or the index is undef.
9826       if ((((i & 1) == 0) && Zero1) || (((i & 1) == 1) && Zero2) ||
9827           (M == SM_SentinelUndef))
9828         continue;
9829 
9830       MatchLo &= (M == Unpckl[i]);
9831       MatchHi &= (M == Unpckh[i]);
9832     }
9833 
9834     if (MatchLo || MatchHi) {
9835       UnpackOpcode = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
9836       V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9837       V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9838       return true;
9839     }
9840   }
9841 
9842   // If a binary shuffle, commute and try again.
9843   if (!IsUnary) {
9844     ShuffleVectorSDNode::commuteMask(Unpckl);
9845     if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) {
9846       UnpackOpcode = X86ISD::UNPCKL;
9847       std::swap(V1, V2);
9848       return true;
9849     }
9850 
9851     ShuffleVectorSDNode::commuteMask(Unpckh);
9852     if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) {
9853       UnpackOpcode = X86ISD::UNPCKH;
9854       std::swap(V1, V2);
9855       return true;
9856     }
9857   }
9858 
9859   return false;
9860 }
9861 
9862 // X86 has dedicated unpack instructions that can handle specific blend
9863 // operations: UNPCKH and UNPCKL.
9864 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
9865                                      ArrayRef<int> Mask, SDValue V1, SDValue V2,
9866                                      SelectionDAG &DAG) {
9867   SmallVector<int, 8> Unpckl;
9868   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
9869   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9870     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
9871 
9872   SmallVector<int, 8> Unpckh;
9873   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false);
9874   if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9875     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
9876 
9877   // Commute and try again.
9878   ShuffleVectorSDNode::commuteMask(Unpckl);
9879   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9880     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
9881 
9882   ShuffleVectorSDNode::commuteMask(Unpckh);
9883   if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9884     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
9885 
9886   return SDValue();
9887 }
9888 
9889 /// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
9890 /// followed by unpack 256-bit.
9891 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
9892                                         ArrayRef<int> Mask, SDValue V1,
9893                                         SDValue V2, SelectionDAG &DAG) {
9894   SmallVector<int, 32> Unpckl, Unpckh;
9895   createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
9896   createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
9897 
9898   unsigned UnpackOpcode;
9899   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9900     UnpackOpcode = X86ISD::UNPCKL;
9901   else if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9902     UnpackOpcode = X86ISD::UNPCKH;
9903   else
9904     return SDValue();
9905 
9906   // This is a "natural" unpack operation (rather than the 128-bit sectored
9907   // operation implemented by AVX). We need to rearrange 64-bit chunks of the
9908   // input in order to use the x86 instruction.
9909   V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),
9910                             DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3});
9911   V1 = DAG.getBitcast(VT, V1);
9912   return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
9913 }
9914 
9915 // Check if the mask can be mapped to a TRUNCATE or VTRUNC, truncating the
9916 // source into the lower elements and zeroing the upper elements.
9917 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,
9918                                  ArrayRef<int> Mask, const APInt &Zeroable,
9919                                  const X86Subtarget &Subtarget) {
9920   if (!VT.is512BitVector() && !Subtarget.hasVLX())
9921     return false;
9922 
9923   unsigned NumElts = Mask.size();
9924   unsigned EltSizeInBits = VT.getScalarSizeInBits();
9925   unsigned MaxScale = 64 / EltSizeInBits;
9926 
9927   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
9928     unsigned SrcEltBits = EltSizeInBits * Scale;
9929     if (SrcEltBits < 32 && !Subtarget.hasBWI())
9930       continue;
9931     unsigned NumSrcElts = NumElts / Scale;
9932     if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale))
9933       continue;
9934     unsigned UpperElts = NumElts - NumSrcElts;
9935     if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
9936       continue;
9937     SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);
9938     SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);
9939     DstVT = MVT::getIntegerVT(EltSizeInBits);
9940     if ((NumSrcElts * EltSizeInBits) >= 128) {
9941       // ISD::TRUNCATE
9942       DstVT = MVT::getVectorVT(DstVT, NumSrcElts);
9943     } else {
9944       // X86ISD::VTRUNC
9945       DstVT = MVT::getVectorVT(DstVT, 128 / EltSizeInBits);
9946     }
9947     return true;
9948   }
9949 
9950   return false;
9951 }
9952 
9953 // Helper to create TRUNCATE/VTRUNC nodes, optionally with zero/undef upper
9954 // element padding to the final DstVT.
9955 static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src,
9956                                   const X86Subtarget &Subtarget,
9957                                   SelectionDAG &DAG, bool ZeroUppers) {
9958   MVT SrcVT = Src.getSimpleValueType();
9959   MVT DstSVT = DstVT.getScalarType();
9960   unsigned NumDstElts = DstVT.getVectorNumElements();
9961   unsigned NumSrcElts = SrcVT.getVectorNumElements();
9962   unsigned DstEltSizeInBits = DstVT.getScalarSizeInBits();
9963 
9964   if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT))
9965     return SDValue();
9966 
9967   // Perform a direct ISD::TRUNCATE if possible.
9968   if (NumSrcElts == NumDstElts)
9969     return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src);
9970 
9971   if (NumSrcElts > NumDstElts) {
9972     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
9973     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
9974     return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits());
9975   }
9976 
9977   if ((NumSrcElts * DstEltSizeInBits) >= 128) {
9978     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
9979     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
9980     return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
9981                           DstVT.getSizeInBits());
9982   }
9983 
9984   // Non-VLX targets must truncate from a 512-bit type, so we need to
9985   // widen, truncate and then possibly extract the original subvector.
9986   if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) {
9987     SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512);
9988     return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers);
9989   }
9990 
9991   // Fallback to a X86ISD::VTRUNC, padding if necessary.
9992   MVT TruncVT = MVT::getVectorVT(DstSVT, 128 / DstEltSizeInBits);
9993   SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src);
9994   if (DstVT != TruncVT)
9995     Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
9996                            DstVT.getSizeInBits());
9997   return Trunc;
9998 }
9999 
10000 // Try to lower trunc+vector_shuffle to a vpmovdb or a vpmovdw instruction.
10001 //
10002 // An example is the following:
10003 //
10004 // t0: ch = EntryToken
10005 //           t2: v4i64,ch = CopyFromReg t0, Register:v4i64 %0
10006 //         t25: v4i32 = truncate t2
10007 //       t41: v8i16 = bitcast t25
10008 //       t21: v8i16 = BUILD_VECTOR undef:i16, undef:i16, undef:i16, undef:i16,
10009 //       Constant:i16<0>, Constant:i16<0>, Constant:i16<0>, Constant:i16<0>
10010 //     t51: v8i16 = vector_shuffle<0,2,4,6,12,13,14,15> t41, t21
10011 //   t18: v2i64 = bitcast t51
10012 //
10013 // One can just use a single vpmovdw instruction, without avx512vl we need to
10014 // use the zmm variant and extract the lower subvector, padding with zeroes.
10015 // TODO: Merge with lowerShuffleAsVTRUNC.
10016 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,
10017                                      SDValue V2, ArrayRef<int> Mask,
10018                                      const APInt &Zeroable,
10019                                      const X86Subtarget &Subtarget,
10020                                      SelectionDAG &DAG) {
10021   assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type");
10022   if (!Subtarget.hasAVX512())
10023     return SDValue();
10024 
10025   unsigned NumElts = VT.getVectorNumElements();
10026   unsigned EltSizeInBits = VT.getScalarSizeInBits();
10027   unsigned MaxScale = 64 / EltSizeInBits;
10028   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10029     unsigned SrcEltBits = EltSizeInBits * Scale;
10030     unsigned NumSrcElts = NumElts / Scale;
10031     unsigned UpperElts = NumElts - NumSrcElts;
10032     if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) ||
10033         !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10034       continue;
10035 
10036     // Attempt to find a matching source truncation, but as a fall back VLX
10037     // cases can use the VPMOV directly.
10038     SDValue Src = peekThroughBitcasts(V1);
10039     if (Src.getOpcode() == ISD::TRUNCATE &&
10040         Src.getScalarValueSizeInBits() == SrcEltBits) {
10041       Src = Src.getOperand(0);
10042     } else if (Subtarget.hasVLX()) {
10043       MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10044       MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10045       Src = DAG.getBitcast(SrcVT, Src);
10046       // Don't do this if PACKSS/PACKUS could perform it cheaper.
10047       if (Scale == 2 &&
10048           ((DAG.ComputeNumSignBits(Src) > EltSizeInBits) ||
10049            (DAG.computeKnownBits(Src).countMinLeadingZeros() >= EltSizeInBits)))
10050         return SDValue();
10051     } else
10052       return SDValue();
10053 
10054     // VPMOVWB is only available with avx512bw.
10055     if (!Subtarget.hasBWI() && Src.getScalarValueSizeInBits() < 32)
10056       return SDValue();
10057 
10058     bool UndefUppers = isUndefInRange(Mask, NumSrcElts, UpperElts);
10059     return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10060   }
10061 
10062   return SDValue();
10063 }
10064 
10065 // Attempt to match binary shuffle patterns as a truncate.
10066 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
10067                                     SDValue V2, ArrayRef<int> Mask,
10068                                     const APInt &Zeroable,
10069                                     const X86Subtarget &Subtarget,
10070                                     SelectionDAG &DAG) {
10071   assert((VT.is128BitVector() || VT.is256BitVector()) &&
10072          "Unexpected VTRUNC type");
10073   if (!Subtarget.hasAVX512())
10074     return SDValue();
10075 
10076   unsigned NumElts = VT.getVectorNumElements();
10077   unsigned EltSizeInBits = VT.getScalarSizeInBits();
10078   unsigned MaxScale = 64 / EltSizeInBits;
10079   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10080     // TODO: Support non-BWI VPMOVWB truncations?
10081     unsigned SrcEltBits = EltSizeInBits * Scale;
10082     if (SrcEltBits < 32 && !Subtarget.hasBWI())
10083       continue;
10084 
10085     // Match shuffle <Ofs,Ofs+Scale,Ofs+2*Scale,..,undef_or_zero,undef_or_zero>
10086     // Bail if the V2 elements are undef.
10087     unsigned NumHalfSrcElts = NumElts / Scale;
10088     unsigned NumSrcElts = 2 * NumHalfSrcElts;
10089     for (unsigned Offset = 0; Offset != Scale; ++Offset) {
10090       if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, Offset, Scale) ||
10091           isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts))
10092         continue;
10093 
10094       // The elements beyond the truncation must be undef/zero.
10095       unsigned UpperElts = NumElts - NumSrcElts;
10096       if (UpperElts > 0 &&
10097           !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10098         continue;
10099       bool UndefUppers =
10100           UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
10101 
10102       // For offset truncations, ensure that the concat is cheap.
10103       if (Offset) {
10104         auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
10105           if (Lo.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
10106               Hi.getOpcode() == ISD::EXTRACT_SUBVECTOR)
10107             return Lo.getOperand(0) == Hi.getOperand(0);
10108           if (ISD::isNormalLoad(Lo.getNode()) &&
10109               ISD::isNormalLoad(Hi.getNode())) {
10110             auto *LDLo = cast<LoadSDNode>(Lo);
10111             auto *LDHi = cast<LoadSDNode>(Hi);
10112             return DAG.areNonVolatileConsecutiveLoads(
10113                 LDHi, LDLo, Lo.getValueType().getStoreSize(), 1);
10114           }
10115           return false;
10116         };
10117         if (!IsCheapConcat(V1, V2))
10118           continue;
10119       }
10120 
10121       // As we're using both sources then we need to concat them together
10122       // and truncate from the double-sized src.
10123       MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
10124       SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
10125 
10126       MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10127       MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10128       Src = DAG.getBitcast(SrcVT, Src);
10129 
10130       // Shift the offset'd elements into place for the truncation.
10131       // TODO: Use getTargetVShiftByConstNode.
10132       if (Offset)
10133         Src = DAG.getNode(
10134             X86ISD::VSRLI, DL, SrcVT, Src,
10135             DAG.getTargetConstant(Offset * EltSizeInBits, DL, MVT::i8));
10136 
10137       return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10138     }
10139   }
10140 
10141   return SDValue();
10142 }
10143 
10144 /// Check whether a compaction lowering can be done by dropping even/odd
10145 /// elements and compute how many times even/odd elements must be dropped.
10146 ///
10147 /// This handles shuffles which take every Nth element where N is a power of
10148 /// two. Example shuffle masks:
10149 ///
10150 /// (even)
10151 ///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14,  0,  2,  4,  6,  8, 10, 12, 14
10152 ///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
10153 ///  N = 2:  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12
10154 ///  N = 2:  0,  4,  8, 12, 16, 20, 24, 28,  0,  4,  8, 12, 16, 20, 24, 28
10155 ///  N = 3:  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8
10156 ///  N = 3:  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24
10157 ///
10158 /// (odd)
10159 ///  N = 1:  1,  3,  5,  7,  9, 11, 13, 15,  0,  2,  4,  6,  8, 10, 12, 14
10160 ///  N = 1:  1,  3,  5,  7,  9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
10161 ///
10162 /// Any of these lanes can of course be undef.
10163 ///
10164 /// This routine only supports N <= 3.
10165 /// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
10166 /// for larger N.
10167 ///
10168 /// \returns N above, or the number of times even/odd elements must be dropped
10169 /// if there is such a number. Otherwise returns zero.
10170 static int canLowerByDroppingElements(ArrayRef<int> Mask, bool MatchEven,
10171                                       bool IsSingleInput) {
10172   // The modulus for the shuffle vector entries is based on whether this is
10173   // a single input or not.
10174   int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
10175   assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
10176          "We should only be called with masks with a power-of-2 size!");
10177 
10178   uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
10179   int Offset = MatchEven ? 0 : 1;
10180 
10181   // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
10182   // and 2^3 simultaneously. This is because we may have ambiguity with
10183   // partially undef inputs.
10184   bool ViableForN[3] = {true, true, true};
10185 
10186   for (int i = 0, e = Mask.size(); i < e; ++i) {
10187     // Ignore undef lanes, we'll optimistically collapse them to the pattern we
10188     // want.
10189     if (Mask[i] < 0)
10190       continue;
10191 
10192     bool IsAnyViable = false;
10193     for (unsigned j = 0; j != std::size(ViableForN); ++j)
10194       if (ViableForN[j]) {
10195         uint64_t N = j + 1;
10196 
10197         // The shuffle mask must be equal to (i * 2^N) % M.
10198         if ((uint64_t)(Mask[i] - Offset) == (((uint64_t)i << N) & ModMask))
10199           IsAnyViable = true;
10200         else
10201           ViableForN[j] = false;
10202       }
10203     // Early exit if we exhaust the possible powers of two.
10204     if (!IsAnyViable)
10205       break;
10206   }
10207 
10208   for (unsigned j = 0; j != std::size(ViableForN); ++j)
10209     if (ViableForN[j])
10210       return j + 1;
10211 
10212   // Return 0 as there is no viable power of two.
10213   return 0;
10214 }
10215 
10216 // X86 has dedicated pack instructions that can handle specific truncation
10217 // operations: PACKSS and PACKUS.
10218 // Checks for compaction shuffle masks if MaxStages > 1.
10219 // TODO: Add support for matching multiple PACKSS/PACKUS stages.
10220 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
10221                                  unsigned &PackOpcode, ArrayRef<int> TargetMask,
10222                                  const SelectionDAG &DAG,
10223                                  const X86Subtarget &Subtarget,
10224                                  unsigned MaxStages = 1) {
10225   unsigned NumElts = VT.getVectorNumElements();
10226   unsigned BitSize = VT.getScalarSizeInBits();
10227   assert(0 < MaxStages && MaxStages <= 3 && (BitSize << MaxStages) <= 64 &&
10228          "Illegal maximum compaction");
10229 
10230   auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
10231     unsigned NumSrcBits = PackVT.getScalarSizeInBits();
10232     unsigned NumPackedBits = NumSrcBits - BitSize;
10233     N1 = peekThroughBitcasts(N1);
10234     N2 = peekThroughBitcasts(N2);
10235     unsigned NumBits1 = N1.getScalarValueSizeInBits();
10236     unsigned NumBits2 = N2.getScalarValueSizeInBits();
10237     bool IsZero1 = llvm::isNullOrNullSplat(N1, /*AllowUndefs*/ false);
10238     bool IsZero2 = llvm::isNullOrNullSplat(N2, /*AllowUndefs*/ false);
10239     if ((!N1.isUndef() && !IsZero1 && NumBits1 != NumSrcBits) ||
10240         (!N2.isUndef() && !IsZero2 && NumBits2 != NumSrcBits))
10241       return false;
10242     if (Subtarget.hasSSE41() || BitSize == 8) {
10243       APInt ZeroMask = APInt::getHighBitsSet(NumSrcBits, NumPackedBits);
10244       if ((N1.isUndef() || IsZero1 || DAG.MaskedValueIsZero(N1, ZeroMask)) &&
10245           (N2.isUndef() || IsZero2 || DAG.MaskedValueIsZero(N2, ZeroMask))) {
10246         V1 = N1;
10247         V2 = N2;
10248         SrcVT = PackVT;
10249         PackOpcode = X86ISD::PACKUS;
10250         return true;
10251       }
10252     }
10253     bool IsAllOnes1 = llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false);
10254     bool IsAllOnes2 = llvm::isAllOnesOrAllOnesSplat(N2, /*AllowUndefs*/ false);
10255     if ((N1.isUndef() || IsZero1 || IsAllOnes1 ||
10256          DAG.ComputeNumSignBits(N1) > NumPackedBits) &&
10257         (N2.isUndef() || IsZero2 || IsAllOnes2 ||
10258          DAG.ComputeNumSignBits(N2) > NumPackedBits)) {
10259       V1 = N1;
10260       V2 = N2;
10261       SrcVT = PackVT;
10262       PackOpcode = X86ISD::PACKSS;
10263       return true;
10264     }
10265     return false;
10266   };
10267 
10268   // Attempt to match against wider and wider compaction patterns.
10269   for (unsigned NumStages = 1; NumStages <= MaxStages; ++NumStages) {
10270     MVT PackSVT = MVT::getIntegerVT(BitSize << NumStages);
10271     MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages);
10272 
10273     // Try binary shuffle.
10274     SmallVector<int, 32> BinaryMask;
10275     createPackShuffleMask(VT, BinaryMask, false, NumStages);
10276     if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2))
10277       if (MatchPACK(V1, V2, PackVT))
10278         return true;
10279 
10280     // Try unary shuffle.
10281     SmallVector<int, 32> UnaryMask;
10282     createPackShuffleMask(VT, UnaryMask, true, NumStages);
10283     if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1))
10284       if (MatchPACK(V1, V1, PackVT))
10285         return true;
10286   }
10287 
10288   return false;
10289 }
10290 
10291 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
10292                                     SDValue V1, SDValue V2, SelectionDAG &DAG,
10293                                     const X86Subtarget &Subtarget) {
10294   MVT PackVT;
10295   unsigned PackOpcode;
10296   unsigned SizeBits = VT.getSizeInBits();
10297   unsigned EltBits = VT.getScalarSizeInBits();
10298   unsigned MaxStages = Log2_32(64 / EltBits);
10299   if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
10300                             Subtarget, MaxStages))
10301     return SDValue();
10302 
10303   unsigned CurrentEltBits = PackVT.getScalarSizeInBits();
10304   unsigned NumStages = Log2_32(CurrentEltBits / EltBits);
10305 
10306   // Don't lower multi-stage packs on AVX512, truncation is better.
10307   if (NumStages != 1 && SizeBits == 128 && Subtarget.hasVLX())
10308     return SDValue();
10309 
10310   // Pack to the largest type possible:
10311   // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
10312   unsigned MaxPackBits = 16;
10313   if (CurrentEltBits > 16 &&
10314       (PackOpcode == X86ISD::PACKSS || Subtarget.hasSSE41()))
10315     MaxPackBits = 32;
10316 
10317   // Repeatedly pack down to the target size.
10318   SDValue Res;
10319   for (unsigned i = 0; i != NumStages; ++i) {
10320     unsigned SrcEltBits = std::min(MaxPackBits, CurrentEltBits);
10321     unsigned NumSrcElts = SizeBits / SrcEltBits;
10322     MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10323     MVT DstSVT = MVT::getIntegerVT(SrcEltBits / 2);
10324     MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10325     MVT DstVT = MVT::getVectorVT(DstSVT, NumSrcElts * 2);
10326     Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
10327                       DAG.getBitcast(SrcVT, V2));
10328     V1 = V2 = Res;
10329     CurrentEltBits /= 2;
10330   }
10331   assert(Res && Res.getValueType() == VT &&
10332          "Failed to lower compaction shuffle");
10333   return Res;
10334 }
10335 
10336 /// Try to emit a bitmask instruction for a shuffle.
10337 ///
10338 /// This handles cases where we can model a blend exactly as a bitmask due to
10339 /// one of the inputs being zeroable.
10340 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10341                                      SDValue V2, ArrayRef<int> Mask,
10342                                      const APInt &Zeroable,
10343                                      const X86Subtarget &Subtarget,
10344                                      SelectionDAG &DAG) {
10345   MVT MaskVT = VT;
10346   MVT EltVT = VT.getVectorElementType();
10347   SDValue Zero, AllOnes;
10348   // Use f64 if i64 isn't legal.
10349   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
10350     EltVT = MVT::f64;
10351     MaskVT = MVT::getVectorVT(EltVT, Mask.size());
10352   }
10353 
10354   MVT LogicVT = VT;
10355   if (EltVT == MVT::f32 || EltVT == MVT::f64) {
10356     Zero = DAG.getConstantFP(0.0, DL, EltVT);
10357     APFloat AllOnesValue =
10358         APFloat::getAllOnesValue(SelectionDAG::EVTToAPFloatSemantics(EltVT));
10359     AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT);
10360     LogicVT =
10361         MVT::getVectorVT(EltVT == MVT::f64 ? MVT::i64 : MVT::i32, Mask.size());
10362   } else {
10363     Zero = DAG.getConstant(0, DL, EltVT);
10364     AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10365   }
10366 
10367   SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
10368   SDValue V;
10369   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10370     if (Zeroable[i])
10371       continue;
10372     if (Mask[i] % Size != i)
10373       return SDValue(); // Not a blend.
10374     if (!V)
10375       V = Mask[i] < Size ? V1 : V2;
10376     else if (V != (Mask[i] < Size ? V1 : V2))
10377       return SDValue(); // Can only let one input through the mask.
10378 
10379     VMaskOps[i] = AllOnes;
10380   }
10381   if (!V)
10382     return SDValue(); // No non-zeroable elements!
10383 
10384   SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10385   VMask = DAG.getBitcast(LogicVT, VMask);
10386   V = DAG.getBitcast(LogicVT, V);
10387   SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10388   return DAG.getBitcast(VT, And);
10389 }
10390 
10391 /// Try to emit a blend instruction for a shuffle using bit math.
10392 ///
10393 /// This is used as a fallback approach when first class blend instructions are
10394 /// unavailable. Currently it is only suitable for integer vectors, but could
10395 /// be generalized for floating point vectors if desirable.
10396 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10397                                       SDValue V2, ArrayRef<int> Mask,
10398                                       SelectionDAG &DAG) {
10399   assert(VT.isInteger() && "Only supports integer vector types!");
10400   MVT EltVT = VT.getVectorElementType();
10401   SDValue Zero = DAG.getConstant(0, DL, EltVT);
10402   SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10403   SmallVector<SDValue, 16> MaskOps;
10404   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10405     if (Mask[i] >= 0 && Mask[i] != i && Mask[i] != i + Size)
10406       return SDValue(); // Shuffled input!
10407     MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero);
10408   }
10409 
10410   SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10411   return getBitSelect(DL, VT, V1, V2, V1Mask, DAG);
10412 }
10413 
10414 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10415                                     SDValue PreservedSrc,
10416                                     const X86Subtarget &Subtarget,
10417                                     SelectionDAG &DAG);
10418 
10419 static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
10420                                 MutableArrayRef<int> Mask,
10421                                 const APInt &Zeroable, bool &ForceV1Zero,
10422                                 bool &ForceV2Zero, uint64_t &BlendMask) {
10423   bool V1IsZeroOrUndef =
10424       V1.isUndef() || ISD::isBuildVectorAllZeros(V1.getNode());
10425   bool V2IsZeroOrUndef =
10426       V2.isUndef() || ISD::isBuildVectorAllZeros(V2.getNode());
10427 
10428   BlendMask = 0;
10429   ForceV1Zero = false, ForceV2Zero = false;
10430   assert(Mask.size() <= 64 && "Shuffle mask too big for blend mask");
10431 
10432   int NumElts = Mask.size();
10433   int NumLanes = VT.getSizeInBits() / 128;
10434   int NumEltsPerLane = NumElts / NumLanes;
10435   assert((NumLanes * NumEltsPerLane) == NumElts && "Value type mismatch");
10436 
10437   // For 32/64-bit elements, if we only reference one input (plus any undefs),
10438   // then ensure the blend mask part for that lane just references that input.
10439   bool ForceWholeLaneMasks =
10440       VT.is256BitVector() && VT.getScalarSizeInBits() >= 32;
10441 
10442   // Attempt to generate the binary blend mask. If an input is zero then
10443   // we can use any lane.
10444   for (int Lane = 0; Lane != NumLanes; ++Lane) {
10445     // Keep track of the inputs used per lane.
10446     bool LaneV1InUse = false;
10447     bool LaneV2InUse = false;
10448     uint64_t LaneBlendMask = 0;
10449     for (int LaneElt = 0; LaneElt != NumEltsPerLane; ++LaneElt) {
10450       int Elt = (Lane * NumEltsPerLane) + LaneElt;
10451       int M = Mask[Elt];
10452       if (M == SM_SentinelUndef)
10453         continue;
10454       if (M == Elt || (0 <= M && M < NumElts &&
10455                      IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
10456         Mask[Elt] = Elt;
10457         LaneV1InUse = true;
10458         continue;
10459       }
10460       if (M == (Elt + NumElts) ||
10461           (NumElts <= M &&
10462            IsElementEquivalent(NumElts, V2, V2, M - NumElts, Elt))) {
10463         LaneBlendMask |= 1ull << LaneElt;
10464         Mask[Elt] = Elt + NumElts;
10465         LaneV2InUse = true;
10466         continue;
10467       }
10468       if (Zeroable[Elt]) {
10469         if (V1IsZeroOrUndef) {
10470           ForceV1Zero = true;
10471           Mask[Elt] = Elt;
10472           LaneV1InUse = true;
10473           continue;
10474         }
10475         if (V2IsZeroOrUndef) {
10476           ForceV2Zero = true;
10477           LaneBlendMask |= 1ull << LaneElt;
10478           Mask[Elt] = Elt + NumElts;
10479           LaneV2InUse = true;
10480           continue;
10481         }
10482       }
10483       return false;
10484     }
10485 
10486     // If we only used V2 then splat the lane blend mask to avoid any demanded
10487     // elts from V1 in this lane (the V1 equivalent is implicit with a zero
10488     // blend mask bit).
10489     if (ForceWholeLaneMasks && LaneV2InUse && !LaneV1InUse)
10490       LaneBlendMask = (1ull << NumEltsPerLane) - 1;
10491 
10492     BlendMask |= LaneBlendMask << (Lane * NumEltsPerLane);
10493   }
10494   return true;
10495 }
10496 
10497 static uint64_t scaleVectorShuffleBlendMask(uint64_t BlendMask, int Size,
10498                                             int Scale) {
10499   uint64_t ScaledMask = 0;
10500   for (int i = 0; i != Size; ++i)
10501     if (BlendMask & (1ull << i))
10502       ScaledMask |= ((1ull << Scale) - 1) << (i * Scale);
10503   return ScaledMask;
10504 }
10505 
10506 /// Try to emit a blend instruction for a shuffle.
10507 ///
10508 /// This doesn't do any checks for the availability of instructions for blending
10509 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
10510 /// be matched in the backend with the type given. What it does check for is
10511 /// that the shuffle mask is a blend, or convertible into a blend with zero.
10512 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
10513                                    SDValue V2, ArrayRef<int> Original,
10514                                    const APInt &Zeroable,
10515                                    const X86Subtarget &Subtarget,
10516                                    SelectionDAG &DAG) {
10517   uint64_t BlendMask = 0;
10518   bool ForceV1Zero = false, ForceV2Zero = false;
10519   SmallVector<int, 64> Mask(Original);
10520   if (!matchShuffleAsBlend(VT, V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero,
10521                            BlendMask))
10522     return SDValue();
10523 
10524   // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
10525   if (ForceV1Zero)
10526     V1 = getZeroVector(VT, Subtarget, DAG, DL);
10527   if (ForceV2Zero)
10528     V2 = getZeroVector(VT, Subtarget, DAG, DL);
10529 
10530   unsigned NumElts = VT.getVectorNumElements();
10531 
10532   switch (VT.SimpleTy) {
10533   case MVT::v4i64:
10534   case MVT::v8i32:
10535     assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!");
10536     [[fallthrough]];
10537   case MVT::v4f64:
10538   case MVT::v8f32:
10539     assert(Subtarget.hasAVX() && "256-bit float blends require AVX!");
10540     [[fallthrough]];
10541   case MVT::v2f64:
10542   case MVT::v2i64:
10543   case MVT::v4f32:
10544   case MVT::v4i32:
10545   case MVT::v8i16:
10546     assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!");
10547     return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
10548                        DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10549   case MVT::v16i16: {
10550     assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!");
10551     SmallVector<int, 8> RepeatedMask;
10552     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
10553       // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
10554       assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!");
10555       BlendMask = 0;
10556       for (int i = 0; i < 8; ++i)
10557         if (RepeatedMask[i] >= 8)
10558           BlendMask |= 1ull << i;
10559       return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10560                          DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10561     }
10562     // Use PBLENDW for lower/upper lanes and then blend lanes.
10563     // TODO - we should allow 2 PBLENDW here and leave shuffle combine to
10564     // merge to VSELECT where useful.
10565     uint64_t LoMask = BlendMask & 0xFF;
10566     uint64_t HiMask = (BlendMask >> 8) & 0xFF;
10567     if (LoMask == 0 || LoMask == 255 || HiMask == 0 || HiMask == 255) {
10568       SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10569                                DAG.getTargetConstant(LoMask, DL, MVT::i8));
10570       SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10571                                DAG.getTargetConstant(HiMask, DL, MVT::i8));
10572       return DAG.getVectorShuffle(
10573           MVT::v16i16, DL, Lo, Hi,
10574           {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31});
10575     }
10576     [[fallthrough]];
10577   }
10578   case MVT::v32i8:
10579     assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!");
10580     [[fallthrough]];
10581   case MVT::v16i8: {
10582     assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!");
10583 
10584     // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
10585     if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10586                                                Subtarget, DAG))
10587       return Masked;
10588 
10589     if (Subtarget.hasBWI() && Subtarget.hasVLX()) {
10590       MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10591       SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10592       return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10593     }
10594 
10595     // If we have VPTERNLOG, we can use that as a bit blend.
10596     if (Subtarget.hasVLX())
10597       if (SDValue BitBlend =
10598               lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
10599         return BitBlend;
10600 
10601     // Scale the blend by the number of bytes per element.
10602     int Scale = VT.getScalarSizeInBits() / 8;
10603 
10604     // This form of blend is always done on bytes. Compute the byte vector
10605     // type.
10606     MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
10607 
10608     // x86 allows load folding with blendvb from the 2nd source operand. But
10609     // we are still using LLVM select here (see comment below), so that's V1.
10610     // If V2 can be load-folded and V1 cannot be load-folded, then commute to
10611     // allow that load-folding possibility.
10612     if (!ISD::isNormalLoad(V1.getNode()) && ISD::isNormalLoad(V2.getNode())) {
10613       ShuffleVectorSDNode::commuteMask(Mask);
10614       std::swap(V1, V2);
10615     }
10616 
10617     // Compute the VSELECT mask. Note that VSELECT is really confusing in the
10618     // mix of LLVM's code generator and the x86 backend. We tell the code
10619     // generator that boolean values in the elements of an x86 vector register
10620     // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
10621     // mapping a select to operand #1, and 'false' mapping to operand #2. The
10622     // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
10623     // of the element (the remaining are ignored) and 0 in that high bit would
10624     // mean operand #1 while 1 in the high bit would mean operand #2. So while
10625     // the LLVM model for boolean values in vector elements gets the relevant
10626     // bit set, it is set backwards and over constrained relative to x86's
10627     // actual model.
10628     SmallVector<SDValue, 32> VSELECTMask;
10629     for (int i = 0, Size = Mask.size(); i < Size; ++i)
10630       for (int j = 0; j < Scale; ++j)
10631         VSELECTMask.push_back(
10632             Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
10633                         : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
10634                                           MVT::i8));
10635 
10636     V1 = DAG.getBitcast(BlendVT, V1);
10637     V2 = DAG.getBitcast(BlendVT, V2);
10638     return DAG.getBitcast(
10639         VT,
10640         DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
10641                       V1, V2));
10642   }
10643   case MVT::v16f32:
10644   case MVT::v8f64:
10645   case MVT::v8i64:
10646   case MVT::v16i32:
10647   case MVT::v32i16:
10648   case MVT::v64i8: {
10649     // Attempt to lower to a bitmask if we can. Only if not optimizing for size.
10650     bool OptForSize = DAG.shouldOptForSize();
10651     if (!OptForSize) {
10652       if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10653                                                  Subtarget, DAG))
10654         return Masked;
10655     }
10656 
10657     // Otherwise load an immediate into a GPR, cast to k-register, and use a
10658     // masked move.
10659     MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10660     SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10661     return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10662   }
10663   default:
10664     llvm_unreachable("Not a supported integer vector type!");
10665   }
10666 }
10667 
10668 /// Try to lower as a blend of elements from two inputs followed by
10669 /// a single-input permutation.
10670 ///
10671 /// This matches the pattern where we can blend elements from two inputs and
10672 /// then reduce the shuffle to a single-input permutation.
10673 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
10674                                              SDValue V1, SDValue V2,
10675                                              ArrayRef<int> Mask,
10676                                              SelectionDAG &DAG,
10677                                              bool ImmBlends = false) {
10678   // We build up the blend mask while checking whether a blend is a viable way
10679   // to reduce the shuffle.
10680   SmallVector<int, 32> BlendMask(Mask.size(), -1);
10681   SmallVector<int, 32> PermuteMask(Mask.size(), -1);
10682 
10683   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10684     if (Mask[i] < 0)
10685       continue;
10686 
10687     assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.");
10688 
10689     if (BlendMask[Mask[i] % Size] < 0)
10690       BlendMask[Mask[i] % Size] = Mask[i];
10691     else if (BlendMask[Mask[i] % Size] != Mask[i])
10692       return SDValue(); // Can't blend in the needed input!
10693 
10694     PermuteMask[i] = Mask[i] % Size;
10695   }
10696 
10697   // If only immediate blends, then bail if the blend mask can't be widened to
10698   // i16.
10699   unsigned EltSize = VT.getScalarSizeInBits();
10700   if (ImmBlends && EltSize == 8 && !canWidenShuffleElements(BlendMask))
10701     return SDValue();
10702 
10703   SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
10704   return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
10705 }
10706 
10707 /// Try to lower as an unpack of elements from two inputs followed by
10708 /// a single-input permutation.
10709 ///
10710 /// This matches the pattern where we can unpack elements from two inputs and
10711 /// then reduce the shuffle to a single-input (wider) permutation.
10712 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
10713                                              SDValue V1, SDValue V2,
10714                                              ArrayRef<int> Mask,
10715                                              SelectionDAG &DAG) {
10716   int NumElts = Mask.size();
10717   int NumLanes = VT.getSizeInBits() / 128;
10718   int NumLaneElts = NumElts / NumLanes;
10719   int NumHalfLaneElts = NumLaneElts / 2;
10720 
10721   bool MatchLo = true, MatchHi = true;
10722   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
10723 
10724   // Determine UNPCKL/UNPCKH type and operand order.
10725   for (int Elt = 0; Elt != NumElts; ++Elt) {
10726     int M = Mask[Elt];
10727     if (M < 0)
10728       continue;
10729 
10730     // Normalize the mask value depending on whether it's V1 or V2.
10731     int NormM = M;
10732     SDValue &Op = Ops[Elt & 1];
10733     if (M < NumElts && (Op.isUndef() || Op == V1))
10734       Op = V1;
10735     else if (NumElts <= M && (Op.isUndef() || Op == V2)) {
10736       Op = V2;
10737       NormM -= NumElts;
10738     } else
10739       return SDValue();
10740 
10741     bool MatchLoAnyLane = false, MatchHiAnyLane = false;
10742     for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) {
10743       int Lo = Lane, Mid = Lane + NumHalfLaneElts, Hi = Lane + NumLaneElts;
10744       MatchLoAnyLane |= isUndefOrInRange(NormM, Lo, Mid);
10745       MatchHiAnyLane |= isUndefOrInRange(NormM, Mid, Hi);
10746       if (MatchLoAnyLane || MatchHiAnyLane) {
10747         assert((MatchLoAnyLane ^ MatchHiAnyLane) &&
10748                "Failed to match UNPCKLO/UNPCKHI");
10749         break;
10750       }
10751     }
10752     MatchLo &= MatchLoAnyLane;
10753     MatchHi &= MatchHiAnyLane;
10754     if (!MatchLo && !MatchHi)
10755       return SDValue();
10756   }
10757   assert((MatchLo ^ MatchHi) && "Failed to match UNPCKLO/UNPCKHI");
10758 
10759   // Element indices have changed after unpacking. Calculate permute mask
10760   // so that they will be put back to the position as dictated by the
10761   // original shuffle mask indices.
10762   SmallVector<int, 32> PermuteMask(NumElts, -1);
10763   for (int Elt = 0; Elt != NumElts; ++Elt) {
10764     int M = Mask[Elt];
10765     if (M < 0)
10766       continue;
10767     int NormM = M;
10768     if (NumElts <= M)
10769       NormM -= NumElts;
10770     bool IsFirstOp = M < NumElts;
10771     int BaseMaskElt =
10772         NumLaneElts * (NormM / NumLaneElts) + (2 * (NormM % NumHalfLaneElts));
10773     if ((IsFirstOp && V1 == Ops[0]) || (!IsFirstOp && V2 == Ops[0]))
10774       PermuteMask[Elt] = BaseMaskElt;
10775     else if ((IsFirstOp && V1 == Ops[1]) || (!IsFirstOp && V2 == Ops[1]))
10776       PermuteMask[Elt] = BaseMaskElt + 1;
10777     assert(PermuteMask[Elt] != -1 &&
10778            "Input mask element is defined but failed to assign permute mask");
10779   }
10780 
10781   unsigned UnpckOp = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
10782   SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
10783   return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
10784 }
10785 
10786 /// Try to lower a shuffle as a permute of the inputs followed by an
10787 /// UNPCK instruction.
10788 ///
10789 /// This specifically targets cases where we end up with alternating between
10790 /// the two inputs, and so can permute them into something that feeds a single
10791 /// UNPCK instruction. Note that this routine only targets integer vectors
10792 /// because for floating point vectors we have a generalized SHUFPS lowering
10793 /// strategy that handles everything that doesn't *exactly* match an unpack,
10794 /// making this clever lowering unnecessary.
10795 static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
10796                                               SDValue V1, SDValue V2,
10797                                               ArrayRef<int> Mask,
10798                                               const X86Subtarget &Subtarget,
10799                                               SelectionDAG &DAG) {
10800   int Size = Mask.size();
10801   assert(Mask.size() >= 2 && "Single element masks are invalid.");
10802 
10803   // This routine only supports 128-bit integer dual input vectors.
10804   if (VT.isFloatingPoint() || !VT.is128BitVector() || V2.isUndef())
10805     return SDValue();
10806 
10807   int NumLoInputs =
10808       count_if(Mask, [Size](int M) { return M >= 0 && M % Size < Size / 2; });
10809   int NumHiInputs =
10810       count_if(Mask, [Size](int M) { return M % Size >= Size / 2; });
10811 
10812   bool UnpackLo = NumLoInputs >= NumHiInputs;
10813 
10814   auto TryUnpack = [&](int ScalarSize, int Scale) {
10815     SmallVector<int, 16> V1Mask((unsigned)Size, -1);
10816     SmallVector<int, 16> V2Mask((unsigned)Size, -1);
10817 
10818     for (int i = 0; i < Size; ++i) {
10819       if (Mask[i] < 0)
10820         continue;
10821 
10822       // Each element of the unpack contains Scale elements from this mask.
10823       int UnpackIdx = i / Scale;
10824 
10825       // We only handle the case where V1 feeds the first slots of the unpack.
10826       // We rely on canonicalization to ensure this is the case.
10827       if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
10828         return SDValue();
10829 
10830       // Setup the mask for this input. The indexing is tricky as we have to
10831       // handle the unpack stride.
10832       SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
10833       VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
10834           Mask[i] % Size;
10835     }
10836 
10837     // If we will have to shuffle both inputs to use the unpack, check whether
10838     // we can just unpack first and shuffle the result. If so, skip this unpack.
10839     if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) &&
10840         !isNoopShuffleMask(V2Mask))
10841       return SDValue();
10842 
10843     // Shuffle the inputs into place.
10844     V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
10845     V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
10846 
10847     // Cast the inputs to the type we will use to unpack them.
10848     MVT UnpackVT =
10849         MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale);
10850     V1 = DAG.getBitcast(UnpackVT, V1);
10851     V2 = DAG.getBitcast(UnpackVT, V2);
10852 
10853     // Unpack the inputs and cast the result back to the desired type.
10854     return DAG.getBitcast(
10855         VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10856                         UnpackVT, V1, V2));
10857   };
10858 
10859   // We try each unpack from the largest to the smallest to try and find one
10860   // that fits this mask.
10861   int OrigScalarSize = VT.getScalarSizeInBits();
10862   for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2)
10863     if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
10864       return Unpack;
10865 
10866   // If we're shuffling with a zero vector then we're better off not doing
10867   // VECTOR_SHUFFLE(UNPCK()) as we lose track of those zero elements.
10868   if (ISD::isBuildVectorAllZeros(V1.getNode()) ||
10869       ISD::isBuildVectorAllZeros(V2.getNode()))
10870     return SDValue();
10871 
10872   // If none of the unpack-rooted lowerings worked (or were profitable) try an
10873   // initial unpack.
10874   if (NumLoInputs == 0 || NumHiInputs == 0) {
10875     assert((NumLoInputs > 0 || NumHiInputs > 0) &&
10876            "We have to have *some* inputs!");
10877     int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0;
10878 
10879     // FIXME: We could consider the total complexity of the permute of each
10880     // possible unpacking. Or at the least we should consider how many
10881     // half-crossings are created.
10882     // FIXME: We could consider commuting the unpacks.
10883 
10884     SmallVector<int, 32> PermMask((unsigned)Size, -1);
10885     for (int i = 0; i < Size; ++i) {
10886       if (Mask[i] < 0)
10887         continue;
10888 
10889       assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!");
10890 
10891       PermMask[i] =
10892           2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1);
10893     }
10894     return DAG.getVectorShuffle(
10895         VT, DL,
10896         DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT,
10897                     V1, V2),
10898         DAG.getUNDEF(VT), PermMask);
10899   }
10900 
10901   return SDValue();
10902 }
10903 
10904 /// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
10905 /// permuting the elements of the result in place.
10906 static SDValue lowerShuffleAsByteRotateAndPermute(
10907     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
10908     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
10909   if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
10910       (VT.is256BitVector() && !Subtarget.hasAVX2()) ||
10911       (VT.is512BitVector() && !Subtarget.hasBWI()))
10912     return SDValue();
10913 
10914   // We don't currently support lane crossing permutes.
10915   if (is128BitLaneCrossingShuffleMask(VT, Mask))
10916     return SDValue();
10917 
10918   int Scale = VT.getScalarSizeInBits() / 8;
10919   int NumLanes = VT.getSizeInBits() / 128;
10920   int NumElts = VT.getVectorNumElements();
10921   int NumEltsPerLane = NumElts / NumLanes;
10922 
10923   // Determine range of mask elts.
10924   bool Blend1 = true;
10925   bool Blend2 = true;
10926   std::pair<int, int> Range1 = std::make_pair(INT_MAX, INT_MIN);
10927   std::pair<int, int> Range2 = std::make_pair(INT_MAX, INT_MIN);
10928   for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
10929     for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
10930       int M = Mask[Lane + Elt];
10931       if (M < 0)
10932         continue;
10933       if (M < NumElts) {
10934         Blend1 &= (M == (Lane + Elt));
10935         assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
10936         M = M % NumEltsPerLane;
10937         Range1.first = std::min(Range1.first, M);
10938         Range1.second = std::max(Range1.second, M);
10939       } else {
10940         M -= NumElts;
10941         Blend2 &= (M == (Lane + Elt));
10942         assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
10943         M = M % NumEltsPerLane;
10944         Range2.first = std::min(Range2.first, M);
10945         Range2.second = std::max(Range2.second, M);
10946       }
10947     }
10948   }
10949 
10950   // Bail if we don't need both elements.
10951   // TODO - it might be worth doing this for unary shuffles if the permute
10952   // can be widened.
10953   if (!(0 <= Range1.first && Range1.second < NumEltsPerLane) ||
10954       !(0 <= Range2.first && Range2.second < NumEltsPerLane))
10955     return SDValue();
10956 
10957   if (VT.getSizeInBits() > 128 && (Blend1 || Blend2))
10958     return SDValue();
10959 
10960   // Rotate the 2 ops so we can access both ranges, then permute the result.
10961   auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
10962     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
10963     SDValue Rotate = DAG.getBitcast(
10964         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
10965                         DAG.getBitcast(ByteVT, Lo),
10966                         DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
10967     SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef);
10968     for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
10969       for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
10970         int M = Mask[Lane + Elt];
10971         if (M < 0)
10972           continue;
10973         if (M < NumElts)
10974           PermMask[Lane + Elt] = Lane + ((M + Ofs - RotAmt) % NumEltsPerLane);
10975         else
10976           PermMask[Lane + Elt] = Lane + ((M - Ofs - RotAmt) % NumEltsPerLane);
10977       }
10978     }
10979     return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
10980   };
10981 
10982   // Check if the ranges are small enough to rotate from either direction.
10983   if (Range2.second < Range1.first)
10984     return RotateAndPermute(V1, V2, Range1.first, 0);
10985   if (Range1.second < Range2.first)
10986     return RotateAndPermute(V2, V1, Range2.first, NumElts);
10987   return SDValue();
10988 }
10989 
10990 static bool isBroadcastShuffleMask(ArrayRef<int> Mask) {
10991   return isUndefOrEqual(Mask, 0);
10992 }
10993 
10994 static bool isNoopOrBroadcastShuffleMask(ArrayRef<int> Mask) {
10995   return isNoopShuffleMask(Mask) || isBroadcastShuffleMask(Mask);
10996 }
10997 
10998 /// Check if the Mask consists of the same element repeated multiple times.
10999 static bool isSingleElementRepeatedMask(ArrayRef<int> Mask) {
11000   size_t NumUndefs = 0;
11001   std::optional<int> UniqueElt;
11002   for (int Elt : Mask) {
11003     if (Elt == SM_SentinelUndef) {
11004       NumUndefs++;
11005       continue;
11006     }
11007     if (UniqueElt.has_value() && UniqueElt.value() != Elt)
11008       return false;
11009     UniqueElt = Elt;
11010   }
11011   // Make sure the element is repeated enough times by checking the number of
11012   // undefs is small.
11013   return NumUndefs <= Mask.size() / 2 && UniqueElt.has_value();
11014 }
11015 
11016 /// Generic routine to decompose a shuffle and blend into independent
11017 /// blends and permutes.
11018 ///
11019 /// This matches the extremely common pattern for handling combined
11020 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
11021 /// operations. It will try to pick the best arrangement of shuffles and
11022 /// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
11023 static SDValue lowerShuffleAsDecomposedShuffleMerge(
11024     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11025     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11026   int NumElts = Mask.size();
11027   int NumLanes = VT.getSizeInBits() / 128;
11028   int NumEltsPerLane = NumElts / NumLanes;
11029 
11030   // Shuffle the input elements into the desired positions in V1 and V2 and
11031   // unpack/blend them together.
11032   bool IsAlternating = true;
11033   SmallVector<int, 32> V1Mask(NumElts, -1);
11034   SmallVector<int, 32> V2Mask(NumElts, -1);
11035   SmallVector<int, 32> FinalMask(NumElts, -1);
11036   for (int i = 0; i < NumElts; ++i) {
11037     int M = Mask[i];
11038     if (M >= 0 && M < NumElts) {
11039       V1Mask[i] = M;
11040       FinalMask[i] = i;
11041       IsAlternating &= (i & 1) == 0;
11042     } else if (M >= NumElts) {
11043       V2Mask[i] = M - NumElts;
11044       FinalMask[i] = i + NumElts;
11045       IsAlternating &= (i & 1) == 1;
11046     }
11047   }
11048 
11049   // If we effectively only demand the 0'th element of \p Input, and not only
11050   // as 0'th element, then broadcast said input,
11051   // and change \p InputMask to be a no-op (identity) mask.
11052   auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget,
11053                                          &DAG](SDValue &Input,
11054                                                MutableArrayRef<int> InputMask) {
11055     unsigned EltSizeInBits = Input.getScalarValueSizeInBits();
11056     if (!Subtarget.hasAVX2() && (!Subtarget.hasAVX() || EltSizeInBits < 32 ||
11057                                  !X86::mayFoldLoad(Input, Subtarget)))
11058       return;
11059     if (isNoopShuffleMask(InputMask))
11060       return;
11061     assert(isBroadcastShuffleMask(InputMask) &&
11062            "Expected to demand only the 0'th element.");
11063     Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input);
11064     for (auto I : enumerate(InputMask)) {
11065       int &InputMaskElt = I.value();
11066       if (InputMaskElt >= 0)
11067         InputMaskElt = I.index();
11068     }
11069   };
11070 
11071   // Currently, we may need to produce one shuffle per input, and blend results.
11072   // It is possible that the shuffle for one of the inputs is already a no-op.
11073   // See if we can simplify non-no-op shuffles into broadcasts,
11074   // which we consider to be strictly better than an arbitrary shuffle.
11075   if (isNoopOrBroadcastShuffleMask(V1Mask) &&
11076       isNoopOrBroadcastShuffleMask(V2Mask)) {
11077     canonicalizeBroadcastableInput(V1, V1Mask);
11078     canonicalizeBroadcastableInput(V2, V2Mask);
11079   }
11080 
11081   // Try to lower with the simpler initial blend/unpack/rotate strategies unless
11082   // one of the input shuffles would be a no-op. We prefer to shuffle inputs as
11083   // the shuffle may be able to fold with a load or other benefit. However, when
11084   // we'll have to do 2x as many shuffles in order to achieve this, a 2-input
11085   // pre-shuffle first is a better strategy.
11086   if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) {
11087     // Only prefer immediate blends to unpack/rotate.
11088     if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11089                                                           DAG, true))
11090       return BlendPerm;
11091     // If either input vector provides only a single element which is repeated
11092     // multiple times, unpacking from both input vectors would generate worse
11093     // code. e.g. for
11094     // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
11095     // it is better to process t4 first to create a vector of t4[0], then unpack
11096     // that vector with t2.
11097     if (!isSingleElementRepeatedMask(V1Mask) &&
11098         !isSingleElementRepeatedMask(V2Mask))
11099       if (SDValue UnpackPerm =
11100               lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
11101         return UnpackPerm;
11102     if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11103             DL, VT, V1, V2, Mask, Subtarget, DAG))
11104       return RotatePerm;
11105     // Unpack/rotate failed - try again with variable blends.
11106     if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11107                                                           DAG))
11108       return BlendPerm;
11109     if (VT.getScalarSizeInBits() >= 32)
11110       if (SDValue PermUnpack = lowerShuffleAsPermuteAndUnpack(
11111               DL, VT, V1, V2, Mask, Subtarget, DAG))
11112         return PermUnpack;
11113   }
11114 
11115   // If the final mask is an alternating blend of vXi8/vXi16, convert to an
11116   // UNPCKL(SHUFFLE, SHUFFLE) pattern.
11117   // TODO: It doesn't have to be alternating - but each lane mustn't have more
11118   // than half the elements coming from each source.
11119   if (IsAlternating && VT.getScalarSizeInBits() < 32) {
11120     V1Mask.assign(NumElts, -1);
11121     V2Mask.assign(NumElts, -1);
11122     FinalMask.assign(NumElts, -1);
11123     for (int i = 0; i != NumElts; i += NumEltsPerLane)
11124       for (int j = 0; j != NumEltsPerLane; ++j) {
11125         int M = Mask[i + j];
11126         if (M >= 0 && M < NumElts) {
11127           V1Mask[i + (j / 2)] = M;
11128           FinalMask[i + j] = i + (j / 2);
11129         } else if (M >= NumElts) {
11130           V2Mask[i + (j / 2)] = M - NumElts;
11131           FinalMask[i + j] = i + (j / 2) + NumElts;
11132         }
11133       }
11134   }
11135 
11136   V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11137   V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11138   return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask);
11139 }
11140 
11141 static int matchShuffleAsBitRotate(MVT &RotateVT, int EltSizeInBits,
11142                                    const X86Subtarget &Subtarget,
11143                                    ArrayRef<int> Mask) {
11144   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11145   assert(EltSizeInBits < 64 && "Can't rotate 64-bit integers");
11146 
11147   // AVX512 only has vXi32/vXi64 rotates, so limit the rotation sub group size.
11148   int MinSubElts = Subtarget.hasAVX512() ? std::max(32 / EltSizeInBits, 2) : 2;
11149   int MaxSubElts = 64 / EltSizeInBits;
11150   unsigned RotateAmt, NumSubElts;
11151   if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts,
11152                                           MaxSubElts, NumSubElts, RotateAmt))
11153     return -1;
11154   unsigned NumElts = Mask.size();
11155   MVT RotateSVT = MVT::getIntegerVT(EltSizeInBits * NumSubElts);
11156   RotateVT = MVT::getVectorVT(RotateSVT, NumElts / NumSubElts);
11157   return RotateAmt;
11158 }
11159 
11160 /// Lower shuffle using X86ISD::VROTLI rotations.
11161 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
11162                                        ArrayRef<int> Mask,
11163                                        const X86Subtarget &Subtarget,
11164                                        SelectionDAG &DAG) {
11165   // Only XOP + AVX512 targets have bit rotation instructions.
11166   // If we at least have SSSE3 (PSHUFB) then we shouldn't attempt to use this.
11167   bool IsLegal =
11168       (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512();
11169   if (!IsLegal && Subtarget.hasSSE3())
11170     return SDValue();
11171 
11172   MVT RotateVT;
11173   int RotateAmt = matchShuffleAsBitRotate(RotateVT, VT.getScalarSizeInBits(),
11174                                           Subtarget, Mask);
11175   if (RotateAmt < 0)
11176     return SDValue();
11177 
11178   // For pre-SSSE3 targets, if we are shuffling vXi8 elts then ISD::ROTL,
11179   // expanded to OR(SRL,SHL), will be more efficient, but if they can
11180   // widen to vXi16 or more then existing lowering should will be better.
11181   if (!IsLegal) {
11182     if ((RotateAmt % 16) == 0)
11183       return SDValue();
11184     // TODO: Use getTargetVShiftByConstNode.
11185     unsigned ShlAmt = RotateAmt;
11186     unsigned SrlAmt = RotateVT.getScalarSizeInBits() - RotateAmt;
11187     V1 = DAG.getBitcast(RotateVT, V1);
11188     SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
11189                               DAG.getTargetConstant(ShlAmt, DL, MVT::i8));
11190     SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
11191                               DAG.getTargetConstant(SrlAmt, DL, MVT::i8));
11192     SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
11193     return DAG.getBitcast(VT, Rot);
11194   }
11195 
11196   SDValue Rot =
11197       DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1),
11198                   DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
11199   return DAG.getBitcast(VT, Rot);
11200 }
11201 
11202 /// Try to match a vector shuffle as an element rotation.
11203 ///
11204 /// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
11205 static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
11206                                        ArrayRef<int> Mask) {
11207   int NumElts = Mask.size();
11208 
11209   // We need to detect various ways of spelling a rotation:
11210   //   [11, 12, 13, 14, 15,  0,  1,  2]
11211   //   [-1, 12, 13, 14, -1, -1,  1, -1]
11212   //   [-1, -1, -1, -1, -1, -1,  1,  2]
11213   //   [ 3,  4,  5,  6,  7,  8,  9, 10]
11214   //   [-1,  4,  5,  6, -1, -1,  9, -1]
11215   //   [-1,  4,  5,  6, -1, -1, -1, -1]
11216   int Rotation = 0;
11217   SDValue Lo, Hi;
11218   for (int i = 0; i < NumElts; ++i) {
11219     int M = Mask[i];
11220     assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&
11221            "Unexpected mask index.");
11222     if (M < 0)
11223       continue;
11224 
11225     // Determine where a rotated vector would have started.
11226     int StartIdx = i - (M % NumElts);
11227     if (StartIdx == 0)
11228       // The identity rotation isn't interesting, stop.
11229       return -1;
11230 
11231     // If we found the tail of a vector the rotation must be the missing
11232     // front. If we found the head of a vector, it must be how much of the
11233     // head.
11234     int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
11235 
11236     if (Rotation == 0)
11237       Rotation = CandidateRotation;
11238     else if (Rotation != CandidateRotation)
11239       // The rotations don't match, so we can't match this mask.
11240       return -1;
11241 
11242     // Compute which value this mask is pointing at.
11243     SDValue MaskV = M < NumElts ? V1 : V2;
11244 
11245     // Compute which of the two target values this index should be assigned
11246     // to. This reflects whether the high elements are remaining or the low
11247     // elements are remaining.
11248     SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11249 
11250     // Either set up this value if we've not encountered it before, or check
11251     // that it remains consistent.
11252     if (!TargetV)
11253       TargetV = MaskV;
11254     else if (TargetV != MaskV)
11255       // This may be a rotation, but it pulls from the inputs in some
11256       // unsupported interleaving.
11257       return -1;
11258   }
11259 
11260   // Check that we successfully analyzed the mask, and normalize the results.
11261   assert(Rotation != 0 && "Failed to locate a viable rotation!");
11262   assert((Lo || Hi) && "Failed to find a rotated input vector!");
11263   if (!Lo)
11264     Lo = Hi;
11265   else if (!Hi)
11266     Hi = Lo;
11267 
11268   V1 = Lo;
11269   V2 = Hi;
11270 
11271   return Rotation;
11272 }
11273 
11274 /// Try to lower a vector shuffle as a byte rotation.
11275 ///
11276 /// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
11277 /// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
11278 /// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
11279 /// try to generically lower a vector shuffle through such an pattern. It
11280 /// does not check for the profitability of lowering either as PALIGNR or
11281 /// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
11282 /// This matches shuffle vectors that look like:
11283 ///
11284 ///   v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
11285 ///
11286 /// Essentially it concatenates V1 and V2, shifts right by some number of
11287 /// elements, and takes the low elements as the result. Note that while this is
11288 /// specified as a *right shift* because x86 is little-endian, it is a *left
11289 /// rotate* of the vector lanes.
11290 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11291                                     ArrayRef<int> Mask) {
11292   // Don't accept any shuffles with zero elements.
11293   if (isAnyZero(Mask))
11294     return -1;
11295 
11296   // PALIGNR works on 128-bit lanes.
11297   SmallVector<int, 16> RepeatedMask;
11298   if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
11299     return -1;
11300 
11301   int Rotation = matchShuffleAsElementRotate(V1, V2, RepeatedMask);
11302   if (Rotation <= 0)
11303     return -1;
11304 
11305   // PALIGNR rotates bytes, so we need to scale the
11306   // rotation based on how many bytes are in the vector lane.
11307   int NumElts = RepeatedMask.size();
11308   int Scale = 16 / NumElts;
11309   return Rotation * Scale;
11310 }
11311 
11312 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11313                                         SDValue V2, ArrayRef<int> Mask,
11314                                         const X86Subtarget &Subtarget,
11315                                         SelectionDAG &DAG) {
11316   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11317 
11318   SDValue Lo = V1, Hi = V2;
11319   int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
11320   if (ByteRotation <= 0)
11321     return SDValue();
11322 
11323   // Cast the inputs to i8 vector of correct length to match PALIGNR or
11324   // PSLLDQ/PSRLDQ.
11325   MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11326   Lo = DAG.getBitcast(ByteVT, Lo);
11327   Hi = DAG.getBitcast(ByteVT, Hi);
11328 
11329   // SSSE3 targets can use the palignr instruction.
11330   if (Subtarget.hasSSSE3()) {
11331     assert((!VT.is512BitVector() || Subtarget.hasBWI()) &&
11332            "512-bit PALIGNR requires BWI instructions");
11333     return DAG.getBitcast(
11334         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
11335                         DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
11336   }
11337 
11338   assert(VT.is128BitVector() &&
11339          "Rotate-based lowering only supports 128-bit lowering!");
11340   assert(Mask.size() <= 16 &&
11341          "Can shuffle at most 16 bytes in a 128-bit vector!");
11342   assert(ByteVT == MVT::v16i8 &&
11343          "SSE2 rotate lowering only needed for v16i8!");
11344 
11345   // Default SSE2 implementation
11346   int LoByteShift = 16 - ByteRotation;
11347   int HiByteShift = ByteRotation;
11348 
11349   SDValue LoShift =
11350       DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
11351                   DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
11352   SDValue HiShift =
11353       DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
11354                   DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
11355   return DAG.getBitcast(VT,
11356                         DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
11357 }
11358 
11359 /// Try to lower a vector shuffle as a dword/qword rotation.
11360 ///
11361 /// AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary
11362 /// rotation of the concatenation of two vectors; This routine will
11363 /// try to generically lower a vector shuffle through such an pattern.
11364 ///
11365 /// Essentially it concatenates V1 and V2, shifts right by some number of
11366 /// elements, and takes the low elements as the result. Note that while this is
11367 /// specified as a *right shift* because x86 is little-endian, it is a *left
11368 /// rotate* of the vector lanes.
11369 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
11370                                     SDValue V2, ArrayRef<int> Mask,
11371                                     const APInt &Zeroable,
11372                                     const X86Subtarget &Subtarget,
11373                                     SelectionDAG &DAG) {
11374   assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
11375          "Only 32-bit and 64-bit elements are supported!");
11376 
11377   // 128/256-bit vectors are only supported with VLX.
11378   assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))
11379          && "VLX required for 128/256-bit vectors");
11380 
11381   SDValue Lo = V1, Hi = V2;
11382   int Rotation = matchShuffleAsElementRotate(Lo, Hi, Mask);
11383   if (0 < Rotation)
11384     return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
11385                        DAG.getTargetConstant(Rotation, DL, MVT::i8));
11386 
11387   // See if we can use VALIGN as a cross-lane version of VSHLDQ/VSRLDQ.
11388   // TODO: Pull this out as a matchShuffleAsElementShift helper?
11389   // TODO: We can probably make this more aggressive and use shift-pairs like
11390   // lowerShuffleAsByteShiftMask.
11391   unsigned NumElts = Mask.size();
11392   unsigned ZeroLo = Zeroable.countr_one();
11393   unsigned ZeroHi = Zeroable.countl_one();
11394   assert((ZeroLo + ZeroHi) < NumElts && "Zeroable shuffle detected");
11395   if (!ZeroLo && !ZeroHi)
11396     return SDValue();
11397 
11398   if (ZeroLo) {
11399     SDValue Src = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11400     int Low = Mask[ZeroLo] < (int)NumElts ? 0 : NumElts;
11401     if (isSequentialOrUndefInRange(Mask, ZeroLo, NumElts - ZeroLo, Low))
11402       return DAG.getNode(X86ISD::VALIGN, DL, VT, Src,
11403                          getZeroVector(VT, Subtarget, DAG, DL),
11404                          DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8));
11405   }
11406 
11407   if (ZeroHi) {
11408     SDValue Src = Mask[0] < (int)NumElts ? V1 : V2;
11409     int Low = Mask[0] < (int)NumElts ? 0 : NumElts;
11410     if (isSequentialOrUndefInRange(Mask, 0, NumElts - ZeroHi, Low + ZeroHi))
11411       return DAG.getNode(X86ISD::VALIGN, DL, VT,
11412                          getZeroVector(VT, Subtarget, DAG, DL), Src,
11413                          DAG.getTargetConstant(ZeroHi, DL, MVT::i8));
11414   }
11415 
11416   return SDValue();
11417 }
11418 
11419 /// Try to lower a vector shuffle as a byte shift sequence.
11420 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
11421                                            SDValue V2, ArrayRef<int> Mask,
11422                                            const APInt &Zeroable,
11423                                            const X86Subtarget &Subtarget,
11424                                            SelectionDAG &DAG) {
11425   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11426   assert(VT.is128BitVector() && "Only 128-bit vectors supported");
11427 
11428   // We need a shuffle that has zeros at one/both ends and a sequential
11429   // shuffle from one source within.
11430   unsigned ZeroLo = Zeroable.countr_one();
11431   unsigned ZeroHi = Zeroable.countl_one();
11432   if (!ZeroLo && !ZeroHi)
11433     return SDValue();
11434 
11435   unsigned NumElts = Mask.size();
11436   unsigned Len = NumElts - (ZeroLo + ZeroHi);
11437   if (!isSequentialOrUndefInRange(Mask, ZeroLo, Len, Mask[ZeroLo]))
11438     return SDValue();
11439 
11440   unsigned Scale = VT.getScalarSizeInBits() / 8;
11441   ArrayRef<int> StubMask = Mask.slice(ZeroLo, Len);
11442   if (!isUndefOrInRange(StubMask, 0, NumElts) &&
11443       !isUndefOrInRange(StubMask, NumElts, 2 * NumElts))
11444     return SDValue();
11445 
11446   SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11447   Res = DAG.getBitcast(MVT::v16i8, Res);
11448 
11449   // Use VSHLDQ/VSRLDQ ops to zero the ends of a vector and leave an
11450   // inner sequential set of elements, possibly offset:
11451   // 01234567 --> zzzzzz01 --> 1zzzzzzz
11452   // 01234567 --> 4567zzzz --> zzzzz456
11453   // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz
11454   if (ZeroLo == 0) {
11455     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11456     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11457                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11458     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11459                       DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
11460   } else if (ZeroHi == 0) {
11461     unsigned Shift = Mask[ZeroLo] % NumElts;
11462     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11463                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11464     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11465                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11466   } else if (!Subtarget.hasSSSE3()) {
11467     // If we don't have PSHUFB then its worth avoiding an AND constant mask
11468     // by performing 3 byte shifts. Shuffle combining can kick in above that.
11469     // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
11470     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11471     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11472                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11473     Shift += Mask[ZeroLo] % NumElts;
11474     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11475                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11476     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11477                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11478   } else
11479     return SDValue();
11480 
11481   return DAG.getBitcast(VT, Res);
11482 }
11483 
11484 /// Try to lower a vector shuffle as a bit shift (shifts in zeros).
11485 ///
11486 /// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
11487 /// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
11488 /// matches elements from one of the input vectors shuffled to the left or
11489 /// right with zeroable elements 'shifted in'. It handles both the strictly
11490 /// bit-wise element shifts and the byte shift across an entire 128-bit double
11491 /// quad word lane.
11492 ///
11493 /// PSHL : (little-endian) left bit shift.
11494 /// [ zz, 0, zz,  2 ]
11495 /// [ -1, 4, zz, -1 ]
11496 /// PSRL : (little-endian) right bit shift.
11497 /// [  1, zz,  3, zz]
11498 /// [ -1, -1,  7, zz]
11499 /// PSLLDQ : (little-endian) left byte shift
11500 /// [ zz,  0,  1,  2,  3,  4,  5,  6]
11501 /// [ zz, zz, -1, -1,  2,  3,  4, -1]
11502 /// [ zz, zz, zz, zz, zz, zz, -1,  1]
11503 /// PSRLDQ : (little-endian) right byte shift
11504 /// [  5, 6,  7, zz, zz, zz, zz, zz]
11505 /// [ -1, 5,  6,  7, zz, zz, zz, zz]
11506 /// [  1, 2, -1, -1, -1, -1, zz, zz]
11507 static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
11508                                unsigned ScalarSizeInBits, ArrayRef<int> Mask,
11509                                int MaskOffset, const APInt &Zeroable,
11510                                const X86Subtarget &Subtarget) {
11511   int Size = Mask.size();
11512   unsigned SizeInBits = Size * ScalarSizeInBits;
11513 
11514   auto CheckZeros = [&](int Shift, int Scale, bool Left) {
11515     for (int i = 0; i < Size; i += Scale)
11516       for (int j = 0; j < Shift; ++j)
11517         if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
11518           return false;
11519 
11520     return true;
11521   };
11522 
11523   auto MatchShift = [&](int Shift, int Scale, bool Left) {
11524     for (int i = 0; i != Size; i += Scale) {
11525       unsigned Pos = Left ? i + Shift : i;
11526       unsigned Low = Left ? i : i + Shift;
11527       unsigned Len = Scale - Shift;
11528       if (!isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset))
11529         return -1;
11530     }
11531 
11532     int ShiftEltBits = ScalarSizeInBits * Scale;
11533     bool ByteShift = ShiftEltBits > 64;
11534     Opcode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
11535                   : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
11536     int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
11537 
11538     // Normalize the scale for byte shifts to still produce an i64 element
11539     // type.
11540     Scale = ByteShift ? Scale / 2 : Scale;
11541 
11542     // We need to round trip through the appropriate type for the shift.
11543     MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
11544     ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
11545                         : MVT::getVectorVT(ShiftSVT, Size / Scale);
11546     return (int)ShiftAmt;
11547   };
11548 
11549   // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
11550   // keep doubling the size of the integer elements up to that. We can
11551   // then shift the elements of the integer vector by whole multiples of
11552   // their width within the elements of the larger integer vector. Test each
11553   // multiple to see if we can find a match with the moved element indices
11554   // and that the shifted in elements are all zeroable.
11555   unsigned MaxWidth = ((SizeInBits == 512) && !Subtarget.hasBWI() ? 64 : 128);
11556   for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
11557     for (int Shift = 1; Shift != Scale; ++Shift)
11558       for (bool Left : {true, false})
11559         if (CheckZeros(Shift, Scale, Left)) {
11560           int ShiftAmt = MatchShift(Shift, Scale, Left);
11561           if (0 < ShiftAmt)
11562             return ShiftAmt;
11563         }
11564 
11565   // no match
11566   return -1;
11567 }
11568 
11569 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11570                                    SDValue V2, ArrayRef<int> Mask,
11571                                    const APInt &Zeroable,
11572                                    const X86Subtarget &Subtarget,
11573                                    SelectionDAG &DAG, bool BitwiseOnly) {
11574   int Size = Mask.size();
11575   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11576 
11577   MVT ShiftVT;
11578   SDValue V = V1;
11579   unsigned Opcode;
11580 
11581   // Try to match shuffle against V1 shift.
11582   int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11583                                      Mask, 0, Zeroable, Subtarget);
11584 
11585   // If V1 failed, try to match shuffle against V2 shift.
11586   if (ShiftAmt < 0) {
11587     ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11588                                    Mask, Size, Zeroable, Subtarget);
11589     V = V2;
11590   }
11591 
11592   if (ShiftAmt < 0)
11593     return SDValue();
11594 
11595   if (BitwiseOnly && (Opcode == X86ISD::VSHLDQ || Opcode == X86ISD::VSRLDQ))
11596     return SDValue();
11597 
11598   assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
11599          "Illegal integer vector type");
11600   V = DAG.getBitcast(ShiftVT, V);
11601   V = DAG.getNode(Opcode, DL, ShiftVT, V,
11602                   DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
11603   return DAG.getBitcast(VT, V);
11604 }
11605 
11606 // EXTRQ: Extract Len elements from lower half of source, starting at Idx.
11607 // Remainder of lower half result is zero and upper half is all undef.
11608 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11609                                 ArrayRef<int> Mask, uint64_t &BitLen,
11610                                 uint64_t &BitIdx, const APInt &Zeroable) {
11611   int Size = Mask.size();
11612   int HalfSize = Size / 2;
11613   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11614   assert(!Zeroable.isAllOnes() && "Fully zeroable shuffle mask");
11615 
11616   // Upper half must be undefined.
11617   if (!isUndefUpperHalf(Mask))
11618     return false;
11619 
11620   // Determine the extraction length from the part of the
11621   // lower half that isn't zeroable.
11622   int Len = HalfSize;
11623   for (; Len > 0; --Len)
11624     if (!Zeroable[Len - 1])
11625       break;
11626   assert(Len > 0 && "Zeroable shuffle mask");
11627 
11628   // Attempt to match first Len sequential elements from the lower half.
11629   SDValue Src;
11630   int Idx = -1;
11631   for (int i = 0; i != Len; ++i) {
11632     int M = Mask[i];
11633     if (M == SM_SentinelUndef)
11634       continue;
11635     SDValue &V = (M < Size ? V1 : V2);
11636     M = M % Size;
11637 
11638     // The extracted elements must start at a valid index and all mask
11639     // elements must be in the lower half.
11640     if (i > M || M >= HalfSize)
11641       return false;
11642 
11643     if (Idx < 0 || (Src == V && Idx == (M - i))) {
11644       Src = V;
11645       Idx = M - i;
11646       continue;
11647     }
11648     return false;
11649   }
11650 
11651   if (!Src || Idx < 0)
11652     return false;
11653 
11654   assert((Idx + Len) <= HalfSize && "Illegal extraction mask");
11655   BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11656   BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11657   V1 = Src;
11658   return true;
11659 }
11660 
11661 // INSERTQ: Extract lowest Len elements from lower half of second source and
11662 // insert over first source, starting at Idx.
11663 // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
11664 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
11665                                   ArrayRef<int> Mask, uint64_t &BitLen,
11666                                   uint64_t &BitIdx) {
11667   int Size = Mask.size();
11668   int HalfSize = Size / 2;
11669   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11670 
11671   // Upper half must be undefined.
11672   if (!isUndefUpperHalf(Mask))
11673     return false;
11674 
11675   for (int Idx = 0; Idx != HalfSize; ++Idx) {
11676     SDValue Base;
11677 
11678     // Attempt to match first source from mask before insertion point.
11679     if (isUndefInRange(Mask, 0, Idx)) {
11680       /* EMPTY */
11681     } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) {
11682       Base = V1;
11683     } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) {
11684       Base = V2;
11685     } else {
11686       continue;
11687     }
11688 
11689     // Extend the extraction length looking to match both the insertion of
11690     // the second source and the remaining elements of the first.
11691     for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) {
11692       SDValue Insert;
11693       int Len = Hi - Idx;
11694 
11695       // Match insertion.
11696       if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) {
11697         Insert = V1;
11698       } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) {
11699         Insert = V2;
11700       } else {
11701         continue;
11702       }
11703 
11704       // Match the remaining elements of the lower half.
11705       if (isUndefInRange(Mask, Hi, HalfSize - Hi)) {
11706         /* EMPTY */
11707       } else if ((!Base || (Base == V1)) &&
11708                  isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) {
11709         Base = V1;
11710       } else if ((!Base || (Base == V2)) &&
11711                  isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi,
11712                                             Size + Hi)) {
11713         Base = V2;
11714       } else {
11715         continue;
11716       }
11717 
11718       BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11719       BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11720       V1 = Base;
11721       V2 = Insert;
11722       return true;
11723     }
11724   }
11725 
11726   return false;
11727 }
11728 
11729 /// Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
11730 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
11731                                      SDValue V2, ArrayRef<int> Mask,
11732                                      const APInt &Zeroable, SelectionDAG &DAG) {
11733   uint64_t BitLen, BitIdx;
11734   if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
11735     return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
11736                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
11737                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11738 
11739   if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
11740     return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
11741                        V2 ? V2 : DAG.getUNDEF(VT),
11742                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
11743                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11744 
11745   return SDValue();
11746 }
11747 
11748 /// Lower a vector shuffle as a zero or any extension.
11749 ///
11750 /// Given a specific number of elements, element bit width, and extension
11751 /// stride, produce either a zero or any extension based on the available
11752 /// features of the subtarget. The extended elements are consecutive and
11753 /// begin and can start from an offsetted element index in the input; to
11754 /// avoid excess shuffling the offset must either being in the bottom lane
11755 /// or at the start of a higher lane. All extended elements must be from
11756 /// the same lane.
11757 static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
11758     const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
11759     ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11760   assert(Scale > 1 && "Need a scale to extend.");
11761   int EltBits = VT.getScalarSizeInBits();
11762   int NumElements = VT.getVectorNumElements();
11763   int NumEltsPerLane = 128 / EltBits;
11764   int OffsetLane = Offset / NumEltsPerLane;
11765   assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
11766          "Only 8, 16, and 32 bit elements can be extended.");
11767   assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.");
11768   assert(0 <= Offset && "Extension offset must be positive.");
11769   assert((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) &&
11770          "Extension offset must be in the first lane or start an upper lane.");
11771 
11772   // Check that an index is in same lane as the base offset.
11773   auto SafeOffset = [&](int Idx) {
11774     return OffsetLane == (Idx / NumEltsPerLane);
11775   };
11776 
11777   // Shift along an input so that the offset base moves to the first element.
11778   auto ShuffleOffset = [&](SDValue V) {
11779     if (!Offset)
11780       return V;
11781 
11782     SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
11783     for (int i = 0; i * Scale < NumElements; ++i) {
11784       int SrcIdx = i + Offset;
11785       ShMask[i] = SafeOffset(SrcIdx) ? SrcIdx : -1;
11786     }
11787     return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
11788   };
11789 
11790   // Found a valid a/zext mask! Try various lowering strategies based on the
11791   // input type and available ISA extensions.
11792   if (Subtarget.hasSSE41()) {
11793     // Not worth offsetting 128-bit vectors if scale == 2, a pattern using
11794     // PUNPCK will catch this in a later shuffle match.
11795     if (Offset && Scale == 2 && VT.is128BitVector())
11796       return SDValue();
11797     MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
11798                                  NumElements / Scale);
11799     InputV = DAG.getBitcast(VT, InputV);
11800     InputV = ShuffleOffset(InputV);
11801     InputV = getEXTEND_VECTOR_INREG(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND,
11802                                     DL, ExtVT, InputV, DAG);
11803     return DAG.getBitcast(VT, InputV);
11804   }
11805 
11806   assert(VT.is128BitVector() && "Only 128-bit vectors can be extended.");
11807   InputV = DAG.getBitcast(VT, InputV);
11808 
11809   // For any extends we can cheat for larger element sizes and use shuffle
11810   // instructions that can fold with a load and/or copy.
11811   if (AnyExt && EltBits == 32) {
11812     int PSHUFDMask[4] = {Offset, -1, SafeOffset(Offset + 1) ? Offset + 1 : -1,
11813                          -1};
11814     return DAG.getBitcast(
11815         VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11816                         DAG.getBitcast(MVT::v4i32, InputV),
11817                         getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
11818   }
11819   if (AnyExt && EltBits == 16 && Scale > 2) {
11820     int PSHUFDMask[4] = {Offset / 2, -1,
11821                          SafeOffset(Offset + 1) ? (Offset + 1) / 2 : -1, -1};
11822     InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11823                          DAG.getBitcast(MVT::v4i32, InputV),
11824                          getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
11825     int PSHUFWMask[4] = {1, -1, -1, -1};
11826     unsigned OddEvenOp = (Offset & 1) ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
11827     return DAG.getBitcast(
11828         VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
11829                         DAG.getBitcast(MVT::v8i16, InputV),
11830                         getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
11831   }
11832 
11833   // The SSE4A EXTRQ instruction can efficiently extend the first 2 lanes
11834   // to 64-bits.
11835   if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget.hasSSE4A()) {
11836     assert(NumElements == (int)Mask.size() && "Unexpected shuffle mask size!");
11837     assert(VT.is128BitVector() && "Unexpected vector width!");
11838 
11839     int LoIdx = Offset * EltBits;
11840     SDValue Lo = DAG.getBitcast(
11841         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11842                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11843                                 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
11844 
11845     if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
11846       return DAG.getBitcast(VT, Lo);
11847 
11848     int HiIdx = (Offset + 1) * EltBits;
11849     SDValue Hi = DAG.getBitcast(
11850         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11851                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11852                                 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
11853     return DAG.getBitcast(VT,
11854                           DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
11855   }
11856 
11857   // If this would require more than 2 unpack instructions to expand, use
11858   // pshufb when available. We can only use more than 2 unpack instructions
11859   // when zero extending i8 elements which also makes it easier to use pshufb.
11860   if (Scale > 4 && EltBits == 8 && Subtarget.hasSSSE3()) {
11861     assert(NumElements == 16 && "Unexpected byte vector width!");
11862     SDValue PSHUFBMask[16];
11863     for (int i = 0; i < 16; ++i) {
11864       int Idx = Offset + (i / Scale);
11865       if ((i % Scale == 0 && SafeOffset(Idx))) {
11866         PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
11867         continue;
11868       }
11869       PSHUFBMask[i] =
11870           AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
11871     }
11872     InputV = DAG.getBitcast(MVT::v16i8, InputV);
11873     return DAG.getBitcast(
11874         VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
11875                         DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
11876   }
11877 
11878   // If we are extending from an offset, ensure we start on a boundary that
11879   // we can unpack from.
11880   int AlignToUnpack = Offset % (NumElements / Scale);
11881   if (AlignToUnpack) {
11882     SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
11883     for (int i = AlignToUnpack; i < NumElements; ++i)
11884       ShMask[i - AlignToUnpack] = i;
11885     InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
11886     Offset -= AlignToUnpack;
11887   }
11888 
11889   // Otherwise emit a sequence of unpacks.
11890   do {
11891     unsigned UnpackLoHi = X86ISD::UNPCKL;
11892     if (Offset >= (NumElements / 2)) {
11893       UnpackLoHi = X86ISD::UNPCKH;
11894       Offset -= (NumElements / 2);
11895     }
11896 
11897     MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
11898     SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
11899                          : getZeroVector(InputVT, Subtarget, DAG, DL);
11900     InputV = DAG.getBitcast(InputVT, InputV);
11901     InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
11902     Scale /= 2;
11903     EltBits *= 2;
11904     NumElements /= 2;
11905   } while (Scale > 1);
11906   return DAG.getBitcast(VT, InputV);
11907 }
11908 
11909 /// Try to lower a vector shuffle as a zero extension on any microarch.
11910 ///
11911 /// This routine will try to do everything in its power to cleverly lower
11912 /// a shuffle which happens to match the pattern of a zero extend. It doesn't
11913 /// check for the profitability of this lowering,  it tries to aggressively
11914 /// match this pattern. It will use all of the micro-architectural details it
11915 /// can to emit an efficient lowering. It handles both blends with all-zero
11916 /// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
11917 /// masking out later).
11918 ///
11919 /// The reason we have dedicated lowering for zext-style shuffles is that they
11920 /// are both incredibly common and often quite performance sensitive.
11921 static SDValue lowerShuffleAsZeroOrAnyExtend(
11922     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11923     const APInt &Zeroable, const X86Subtarget &Subtarget,
11924     SelectionDAG &DAG) {
11925   int Bits = VT.getSizeInBits();
11926   int NumLanes = Bits / 128;
11927   int NumElements = VT.getVectorNumElements();
11928   int NumEltsPerLane = NumElements / NumLanes;
11929   assert(VT.getScalarSizeInBits() <= 32 &&
11930          "Exceeds 32-bit integer zero extension limit");
11931   assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size");
11932 
11933   // Define a helper function to check a particular ext-scale and lower to it if
11934   // valid.
11935   auto Lower = [&](int Scale) -> SDValue {
11936     SDValue InputV;
11937     bool AnyExt = true;
11938     int Offset = 0;
11939     int Matches = 0;
11940     for (int i = 0; i < NumElements; ++i) {
11941       int M = Mask[i];
11942       if (M < 0)
11943         continue; // Valid anywhere but doesn't tell us anything.
11944       if (i % Scale != 0) {
11945         // Each of the extended elements need to be zeroable.
11946         if (!Zeroable[i])
11947           return SDValue();
11948 
11949         // We no longer are in the anyext case.
11950         AnyExt = false;
11951         continue;
11952       }
11953 
11954       // Each of the base elements needs to be consecutive indices into the
11955       // same input vector.
11956       SDValue V = M < NumElements ? V1 : V2;
11957       M = M % NumElements;
11958       if (!InputV) {
11959         InputV = V;
11960         Offset = M - (i / Scale);
11961       } else if (InputV != V)
11962         return SDValue(); // Flip-flopping inputs.
11963 
11964       // Offset must start in the lowest 128-bit lane or at the start of an
11965       // upper lane.
11966       // FIXME: Is it ever worth allowing a negative base offset?
11967       if (!((0 <= Offset && Offset < NumEltsPerLane) ||
11968             (Offset % NumEltsPerLane) == 0))
11969         return SDValue();
11970 
11971       // If we are offsetting, all referenced entries must come from the same
11972       // lane.
11973       if (Offset && (Offset / NumEltsPerLane) != (M / NumEltsPerLane))
11974         return SDValue();
11975 
11976       if ((M % NumElements) != (Offset + (i / Scale)))
11977         return SDValue(); // Non-consecutive strided elements.
11978       Matches++;
11979     }
11980 
11981     // If we fail to find an input, we have a zero-shuffle which should always
11982     // have already been handled.
11983     // FIXME: Maybe handle this here in case during blending we end up with one?
11984     if (!InputV)
11985       return SDValue();
11986 
11987     // If we are offsetting, don't extend if we only match a single input, we
11988     // can always do better by using a basic PSHUF or PUNPCK.
11989     if (Offset != 0 && Matches < 2)
11990       return SDValue();
11991 
11992     return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
11993                                                  InputV, Mask, Subtarget, DAG);
11994   };
11995 
11996   // The widest scale possible for extending is to a 64-bit integer.
11997   assert(Bits % 64 == 0 &&
11998          "The number of bits in a vector must be divisible by 64 on x86!");
11999   int NumExtElements = Bits / 64;
12000 
12001   // Each iteration, try extending the elements half as much, but into twice as
12002   // many elements.
12003   for (; NumExtElements < NumElements; NumExtElements *= 2) {
12004     assert(NumElements % NumExtElements == 0 &&
12005            "The input vector size must be divisible by the extended size.");
12006     if (SDValue V = Lower(NumElements / NumExtElements))
12007       return V;
12008   }
12009 
12010   // General extends failed, but 128-bit vectors may be able to use MOVQ.
12011   if (Bits != 128)
12012     return SDValue();
12013 
12014   // Returns one of the source operands if the shuffle can be reduced to a
12015   // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
12016   auto CanZExtLowHalf = [&]() {
12017     for (int i = NumElements / 2; i != NumElements; ++i)
12018       if (!Zeroable[i])
12019         return SDValue();
12020     if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
12021       return V1;
12022     if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
12023       return V2;
12024     return SDValue();
12025   };
12026 
12027   if (SDValue V = CanZExtLowHalf()) {
12028     V = DAG.getBitcast(MVT::v2i64, V);
12029     V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12030     return DAG.getBitcast(VT, V);
12031   }
12032 
12033   // No viable ext lowering found.
12034   return SDValue();
12035 }
12036 
12037 /// Try to get a scalar value for a specific element of a vector.
12038 ///
12039 /// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
12040 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12041                                               SelectionDAG &DAG) {
12042   MVT VT = V.getSimpleValueType();
12043   MVT EltVT = VT.getVectorElementType();
12044   V = peekThroughBitcasts(V);
12045 
12046   // If the bitcasts shift the element size, we can't extract an equivalent
12047   // element from it.
12048   MVT NewVT = V.getSimpleValueType();
12049   if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
12050     return SDValue();
12051 
12052   if (V.getOpcode() == ISD::BUILD_VECTOR ||
12053       (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) {
12054     // Ensure the scalar operand is the same size as the destination.
12055     // FIXME: Add support for scalar truncation where possible.
12056     SDValue S = V.getOperand(Idx);
12057     if (EltVT.getSizeInBits() == S.getSimpleValueType().getSizeInBits())
12058       return DAG.getBitcast(EltVT, S);
12059   }
12060 
12061   return SDValue();
12062 }
12063 
12064 /// Helper to test for a load that can be folded with x86 shuffles.
12065 ///
12066 /// This is particularly important because the set of instructions varies
12067 /// significantly based on whether the operand is a load or not.
12068 static bool isShuffleFoldableLoad(SDValue V) {
12069   return V->hasOneUse() &&
12070          ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode());
12071 }
12072 
12073 template<typename T>
12074 static bool isSoftF16(T VT, const X86Subtarget &Subtarget) {
12075   T EltVT = VT.getScalarType();
12076   return EltVT == MVT::bf16 || (EltVT == MVT::f16 && !Subtarget.hasFP16());
12077 }
12078 
12079 /// Try to lower insertion of a single element into a zero vector.
12080 ///
12081 /// This is a common pattern that we have especially efficient patterns to lower
12082 /// across all subtarget feature sets.
12083 static SDValue lowerShuffleAsElementInsertion(
12084     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12085     const APInt &Zeroable, const X86Subtarget &Subtarget,
12086     SelectionDAG &DAG) {
12087   MVT ExtVT = VT;
12088   MVT EltVT = VT.getVectorElementType();
12089   unsigned NumElts = VT.getVectorNumElements();
12090   unsigned EltBits = VT.getScalarSizeInBits();
12091 
12092   if (isSoftF16(EltVT, Subtarget))
12093     return SDValue();
12094 
12095   int V2Index =
12096       find_if(Mask, [&Mask](int M) { return M >= (int)Mask.size(); }) -
12097       Mask.begin();
12098   bool IsV1Constant = getTargetConstantFromNode(V1) != nullptr;
12099   bool IsV1Zeroable = true;
12100   for (int i = 0, Size = Mask.size(); i < Size; ++i)
12101     if (i != V2Index && !Zeroable[i]) {
12102       IsV1Zeroable = false;
12103       break;
12104     }
12105 
12106   // Bail if a non-zero V1 isn't used in place.
12107   if (!IsV1Zeroable) {
12108     SmallVector<int, 8> V1Mask(Mask);
12109     V1Mask[V2Index] = -1;
12110     if (!isNoopShuffleMask(V1Mask))
12111       return SDValue();
12112   }
12113 
12114   // Check for a single input from a SCALAR_TO_VECTOR node.
12115   // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
12116   // all the smarts here sunk into that routine. However, the current
12117   // lowering of BUILD_VECTOR makes that nearly impossible until the old
12118   // vector shuffle lowering is dead.
12119   SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12120                                                DAG);
12121   if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
12122     // We need to zext the scalar if it is smaller than an i32.
12123     V2S = DAG.getBitcast(EltVT, V2S);
12124     if (EltVT == MVT::i8 || (EltVT == MVT::i16 && !Subtarget.hasFP16())) {
12125       // Using zext to expand a narrow element won't work for non-zero
12126       // insertions. But we can use a masked constant vector if we're
12127       // inserting V2 into the bottom of V1.
12128       if (!IsV1Zeroable && !(IsV1Constant && V2Index == 0))
12129         return SDValue();
12130 
12131       // Zero-extend directly to i32.
12132       ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32);
12133       V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
12134 
12135       // If we're inserting into a constant, mask off the inserted index
12136       // and OR with the zero-extended scalar.
12137       if (!IsV1Zeroable) {
12138         SmallVector<APInt> Bits(NumElts, APInt::getAllOnes(EltBits));
12139         Bits[V2Index] = APInt::getZero(EltBits);
12140         SDValue BitMask = getConstVector(Bits, VT, DAG, DL);
12141         V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask);
12142         V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12143         V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2));
12144         return DAG.getNode(ISD::OR, DL, VT, V1, V2);
12145       }
12146     }
12147     V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12148   } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
12149              EltVT == MVT::i16) {
12150     // Either not inserting from the low element of the input or the input
12151     // element size is too small to use VZEXT_MOVL to clear the high bits.
12152     return SDValue();
12153   }
12154 
12155   if (!IsV1Zeroable) {
12156     // If V1 can't be treated as a zero vector we have fewer options to lower
12157     // this. We can't support integer vectors or non-zero targets cheaply.
12158     assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
12159     if (!VT.isFloatingPoint() || V2Index != 0)
12160       return SDValue();
12161     if (!VT.is128BitVector())
12162       return SDValue();
12163 
12164     // Otherwise, use MOVSD, MOVSS or MOVSH.
12165     unsigned MovOpc = 0;
12166     if (EltVT == MVT::f16)
12167       MovOpc = X86ISD::MOVSH;
12168     else if (EltVT == MVT::f32)
12169       MovOpc = X86ISD::MOVSS;
12170     else if (EltVT == MVT::f64)
12171       MovOpc = X86ISD::MOVSD;
12172     else
12173       llvm_unreachable("Unsupported floating point element type to handle!");
12174     return DAG.getNode(MovOpc, DL, ExtVT, V1, V2);
12175   }
12176 
12177   // This lowering only works for the low element with floating point vectors.
12178   if (VT.isFloatingPoint() && V2Index != 0)
12179     return SDValue();
12180 
12181   V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
12182   if (ExtVT != VT)
12183     V2 = DAG.getBitcast(VT, V2);
12184 
12185   if (V2Index != 0) {
12186     // If we have 4 or fewer lanes we can cheaply shuffle the element into
12187     // the desired position. Otherwise it is more efficient to do a vector
12188     // shift left. We know that we can do a vector shift left because all
12189     // the inputs are zero.
12190     if (VT.isFloatingPoint() || NumElts <= 4) {
12191       SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
12192       V2Shuffle[V2Index] = 0;
12193       V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
12194     } else {
12195       V2 = DAG.getBitcast(MVT::v16i8, V2);
12196       V2 = DAG.getNode(
12197           X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
12198           DAG.getTargetConstant(V2Index * EltBits / 8, DL, MVT::i8));
12199       V2 = DAG.getBitcast(VT, V2);
12200     }
12201   }
12202   return V2;
12203 }
12204 
12205 /// Try to lower broadcast of a single - truncated - integer element,
12206 /// coming from a scalar_to_vector/build_vector node \p V0 with larger elements.
12207 ///
12208 /// This assumes we have AVX2.
12209 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12210                                             int BroadcastIdx,
12211                                             const X86Subtarget &Subtarget,
12212                                             SelectionDAG &DAG) {
12213   assert(Subtarget.hasAVX2() &&
12214          "We can only lower integer broadcasts with AVX2!");
12215 
12216   MVT EltVT = VT.getVectorElementType();
12217   MVT V0VT = V0.getSimpleValueType();
12218 
12219   assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!");
12220   assert(V0VT.isVector() && "Unexpected non-vector vector-sized value!");
12221 
12222   MVT V0EltVT = V0VT.getVectorElementType();
12223   if (!V0EltVT.isInteger())
12224     return SDValue();
12225 
12226   const unsigned EltSize = EltVT.getSizeInBits();
12227   const unsigned V0EltSize = V0EltVT.getSizeInBits();
12228 
12229   // This is only a truncation if the original element type is larger.
12230   if (V0EltSize <= EltSize)
12231     return SDValue();
12232 
12233   assert(((V0EltSize % EltSize) == 0) &&
12234          "Scalar type sizes must all be powers of 2 on x86!");
12235 
12236   const unsigned V0Opc = V0.getOpcode();
12237   const unsigned Scale = V0EltSize / EltSize;
12238   const unsigned V0BroadcastIdx = BroadcastIdx / Scale;
12239 
12240   if ((V0Opc != ISD::SCALAR_TO_VECTOR || V0BroadcastIdx != 0) &&
12241       V0Opc != ISD::BUILD_VECTOR)
12242     return SDValue();
12243 
12244   SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12245 
12246   // If we're extracting non-least-significant bits, shift so we can truncate.
12247   // Hopefully, we can fold away the trunc/srl/load into the broadcast.
12248   // Even if we can't (and !isShuffleFoldableLoad(Scalar)), prefer
12249   // vpbroadcast+vmovd+shr to vpshufb(m)+vmovd.
12250   if (const int OffsetIdx = BroadcastIdx % Scale)
12251     Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
12252                          DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
12253 
12254   return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
12255                      DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
12256 }
12257 
12258 /// Test whether this can be lowered with a single SHUFPS instruction.
12259 ///
12260 /// This is used to disable more specialized lowerings when the shufps lowering
12261 /// will happen to be efficient.
12262 static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
12263   // This routine only handles 128-bit shufps.
12264   assert(Mask.size() == 4 && "Unsupported mask size!");
12265   assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!");
12266   assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!");
12267   assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!");
12268   assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!");
12269 
12270   // To lower with a single SHUFPS we need to have the low half and high half
12271   // each requiring a single input.
12272   if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
12273     return false;
12274   if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
12275     return false;
12276 
12277   return true;
12278 }
12279 
12280 /// Test whether the specified input (0 or 1) is in-place blended by the
12281 /// given mask.
12282 ///
12283 /// This returns true if the elements from a particular input are already in the
12284 /// slot required by the given mask and require no permutation.
12285 static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
12286   assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
12287   int Size = Mask.size();
12288   for (int i = 0; i < Size; ++i)
12289     if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
12290       return false;
12291 
12292   return true;
12293 }
12294 
12295 /// If we are extracting two 128-bit halves of a vector and shuffling the
12296 /// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
12297 /// multi-shuffle lowering.
12298 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12299                                              SDValue N1, ArrayRef<int> Mask,
12300                                              SelectionDAG &DAG) {
12301   MVT VT = N0.getSimpleValueType();
12302   assert((VT.is128BitVector() &&
12303           (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) &&
12304          "VPERM* family of shuffles requires 32-bit or 64-bit elements");
12305 
12306   // Check that both sources are extracts of the same source vector.
12307   if (N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12308       N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12309       N0.getOperand(0) != N1.getOperand(0) ||
12310       !N0.hasOneUse() || !N1.hasOneUse())
12311     return SDValue();
12312 
12313   SDValue WideVec = N0.getOperand(0);
12314   MVT WideVT = WideVec.getSimpleValueType();
12315   if (!WideVT.is256BitVector())
12316     return SDValue();
12317 
12318   // Match extracts of each half of the wide source vector. Commute the shuffle
12319   // if the extract of the low half is N1.
12320   unsigned NumElts = VT.getVectorNumElements();
12321   SmallVector<int, 4> NewMask(Mask);
12322   const APInt &ExtIndex0 = N0.getConstantOperandAPInt(1);
12323   const APInt &ExtIndex1 = N1.getConstantOperandAPInt(1);
12324   if (ExtIndex1 == 0 && ExtIndex0 == NumElts)
12325     ShuffleVectorSDNode::commuteMask(NewMask);
12326   else if (ExtIndex0 != 0 || ExtIndex1 != NumElts)
12327     return SDValue();
12328 
12329   // Final bailout: if the mask is simple, we are better off using an extract
12330   // and a simple narrow shuffle. Prefer extract+unpack(h/l)ps to vpermps
12331   // because that avoids a constant load from memory.
12332   if (NumElts == 4 &&
12333       (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask, DAG)))
12334     return SDValue();
12335 
12336   // Extend the shuffle mask with undef elements.
12337   NewMask.append(NumElts, -1);
12338 
12339   // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
12340   SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12341                                       NewMask);
12342   // This is free: ymm -> xmm.
12343   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
12344                      DAG.getIntPtrConstant(0, DL));
12345 }
12346 
12347 /// Try to lower broadcast of a single element.
12348 ///
12349 /// For convenience, this code also bundles all of the subtarget feature set
12350 /// filtering. While a little annoying to re-dispatch on type here, there isn't
12351 /// a convenient way to factor it out.
12352 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12353                                        SDValue V2, ArrayRef<int> Mask,
12354                                        const X86Subtarget &Subtarget,
12355                                        SelectionDAG &DAG) {
12356   MVT EltVT = VT.getVectorElementType();
12357   if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||
12358         (Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
12359         (Subtarget.hasAVX2() && (VT.isInteger() || EltVT == MVT::f16))))
12360     return SDValue();
12361 
12362   // With MOVDDUP (v2f64) we can broadcast from a register or a load, otherwise
12363   // we can only broadcast from a register with AVX2.
12364   unsigned NumEltBits = VT.getScalarSizeInBits();
12365   unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())
12366                         ? X86ISD::MOVDDUP
12367                         : X86ISD::VBROADCAST;
12368   bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
12369 
12370   // Check that the mask is a broadcast.
12371   int BroadcastIdx = getSplatIndex(Mask);
12372   if (BroadcastIdx < 0)
12373     return SDValue();
12374   assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
12375                                             "a sorted mask where the broadcast "
12376                                             "comes from V1.");
12377 
12378   // Go up the chain of (vector) values to find a scalar load that we can
12379   // combine with the broadcast.
12380   // TODO: Combine this logic with findEltLoadSrc() used by
12381   //       EltsFromConsecutiveLoads().
12382   int BitOffset = BroadcastIdx * NumEltBits;
12383   SDValue V = V1;
12384   for (;;) {
12385     switch (V.getOpcode()) {
12386     case ISD::BITCAST: {
12387       V = V.getOperand(0);
12388       continue;
12389     }
12390     case ISD::CONCAT_VECTORS: {
12391       int OpBitWidth = V.getOperand(0).getValueSizeInBits();
12392       int OpIdx = BitOffset / OpBitWidth;
12393       V = V.getOperand(OpIdx);
12394       BitOffset %= OpBitWidth;
12395       continue;
12396     }
12397     case ISD::EXTRACT_SUBVECTOR: {
12398       // The extraction index adds to the existing offset.
12399       unsigned EltBitWidth = V.getScalarValueSizeInBits();
12400       unsigned Idx = V.getConstantOperandVal(1);
12401       unsigned BeginOffset = Idx * EltBitWidth;
12402       BitOffset += BeginOffset;
12403       V = V.getOperand(0);
12404       continue;
12405     }
12406     case ISD::INSERT_SUBVECTOR: {
12407       SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12408       int EltBitWidth = VOuter.getScalarValueSizeInBits();
12409       int Idx = (int)V.getConstantOperandVal(2);
12410       int NumSubElts = (int)VInner.getSimpleValueType().getVectorNumElements();
12411       int BeginOffset = Idx * EltBitWidth;
12412       int EndOffset = BeginOffset + NumSubElts * EltBitWidth;
12413       if (BeginOffset <= BitOffset && BitOffset < EndOffset) {
12414         BitOffset -= BeginOffset;
12415         V = VInner;
12416       } else {
12417         V = VOuter;
12418       }
12419       continue;
12420     }
12421     }
12422     break;
12423   }
12424   assert((BitOffset % NumEltBits) == 0 && "Illegal bit-offset");
12425   BroadcastIdx = BitOffset / NumEltBits;
12426 
12427   // Do we need to bitcast the source to retrieve the original broadcast index?
12428   bool BitCastSrc = V.getScalarValueSizeInBits() != NumEltBits;
12429 
12430   // Check if this is a broadcast of a scalar. We special case lowering
12431   // for scalars so that we can more effectively fold with loads.
12432   // If the original value has a larger element type than the shuffle, the
12433   // broadcast element is in essence truncated. Make that explicit to ease
12434   // folding.
12435   if (BitCastSrc && VT.isInteger())
12436     if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12437             DL, VT, V, BroadcastIdx, Subtarget, DAG))
12438       return TruncBroadcast;
12439 
12440   // Also check the simpler case, where we can directly reuse the scalar.
12441   if (!BitCastSrc &&
12442       ((V.getOpcode() == ISD::BUILD_VECTOR && V.hasOneUse()) ||
12443        (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0))) {
12444     V = V.getOperand(BroadcastIdx);
12445 
12446     // If we can't broadcast from a register, check that the input is a load.
12447     if (!BroadcastFromReg && !isShuffleFoldableLoad(V))
12448       return SDValue();
12449   } else if (ISD::isNormalLoad(V.getNode()) &&
12450              cast<LoadSDNode>(V)->isSimple()) {
12451     // We do not check for one-use of the vector load because a broadcast load
12452     // is expected to be a win for code size, register pressure, and possibly
12453     // uops even if the original vector load is not eliminated.
12454 
12455     // Reduce the vector load and shuffle to a broadcasted scalar load.
12456     LoadSDNode *Ld = cast<LoadSDNode>(V);
12457     SDValue BaseAddr = Ld->getOperand(1);
12458     MVT SVT = VT.getScalarType();
12459     unsigned Offset = BroadcastIdx * SVT.getStoreSize();
12460     assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset");
12461     SDValue NewAddr =
12462         DAG.getMemBasePlusOffset(BaseAddr, TypeSize::getFixed(Offset), DL);
12463 
12464     // Directly form VBROADCAST_LOAD if we're using VBROADCAST opcode rather
12465     // than MOVDDUP.
12466     // FIXME: Should we add VBROADCAST_LOAD isel patterns for pre-AVX?
12467     if (Opcode == X86ISD::VBROADCAST) {
12468       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
12469       SDValue Ops[] = {Ld->getChain(), NewAddr};
12470       V = DAG.getMemIntrinsicNode(
12471           X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SVT,
12472           DAG.getMachineFunction().getMachineMemOperand(
12473               Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12474       DAG.makeEquivalentMemoryOrdering(Ld, V);
12475       return DAG.getBitcast(VT, V);
12476     }
12477     assert(SVT == MVT::f64 && "Unexpected VT!");
12478     V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
12479                     DAG.getMachineFunction().getMachineMemOperand(
12480                         Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12481     DAG.makeEquivalentMemoryOrdering(Ld, V);
12482   } else if (!BroadcastFromReg) {
12483     // We can't broadcast from a vector register.
12484     return SDValue();
12485   } else if (BitOffset != 0) {
12486     // We can only broadcast from the zero-element of a vector register,
12487     // but it can be advantageous to broadcast from the zero-element of a
12488     // subvector.
12489     if (!VT.is256BitVector() && !VT.is512BitVector())
12490       return SDValue();
12491 
12492     // VPERMQ/VPERMPD can perform the cross-lane shuffle directly.
12493     if (VT == MVT::v4f64 || VT == MVT::v4i64)
12494       return SDValue();
12495 
12496     // Only broadcast the zero-element of a 128-bit subvector.
12497     if ((BitOffset % 128) != 0)
12498       return SDValue();
12499 
12500     assert((BitOffset % V.getScalarValueSizeInBits()) == 0 &&
12501            "Unexpected bit-offset");
12502     assert((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) &&
12503            "Unexpected vector size");
12504     unsigned ExtractIdx = BitOffset / V.getScalarValueSizeInBits();
12505     V = extract128BitVector(V, ExtractIdx, DAG, DL);
12506   }
12507 
12508   // On AVX we can use VBROADCAST directly for scalar sources.
12509   if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector()) {
12510     V = DAG.getBitcast(MVT::f64, V);
12511     if (Subtarget.hasAVX()) {
12512       V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);
12513       return DAG.getBitcast(VT, V);
12514     }
12515     V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);
12516   }
12517 
12518   // If this is a scalar, do the broadcast on this type and bitcast.
12519   if (!V.getValueType().isVector()) {
12520     assert(V.getScalarValueSizeInBits() == NumEltBits &&
12521            "Unexpected scalar size");
12522     MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),
12523                                        VT.getVectorNumElements());
12524     return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
12525   }
12526 
12527   // We only support broadcasting from 128-bit vectors to minimize the
12528   // number of patterns we need to deal with in isel. So extract down to
12529   // 128-bits, removing as many bitcasts as possible.
12530   if (V.getValueSizeInBits() > 128)
12531     V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
12532 
12533   // Otherwise cast V to a vector with the same element type as VT, but
12534   // possibly narrower than VT. Then perform the broadcast.
12535   unsigned NumSrcElts = V.getValueSizeInBits() / NumEltBits;
12536   MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts);
12537   return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V));
12538 }
12539 
12540 // Check for whether we can use INSERTPS to perform the shuffle. We only use
12541 // INSERTPS when the V1 elements are already in the correct locations
12542 // because otherwise we can just always use two SHUFPS instructions which
12543 // are much smaller to encode than a SHUFPS and an INSERTPS. We can also
12544 // perform INSERTPS if a single V1 element is out of place and all V2
12545 // elements are zeroable.
12546 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12547                                    unsigned &InsertPSMask,
12548                                    const APInt &Zeroable,
12549                                    ArrayRef<int> Mask, SelectionDAG &DAG) {
12550   assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
12551   assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
12552   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
12553 
12554   // Attempt to match INSERTPS with one element from VA or VB being
12555   // inserted into VA (or undef). If successful, V1, V2 and InsertPSMask
12556   // are updated.
12557   auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
12558                              ArrayRef<int> CandidateMask) {
12559     unsigned ZMask = 0;
12560     int VADstIndex = -1;
12561     int VBDstIndex = -1;
12562     bool VAUsedInPlace = false;
12563 
12564     for (int i = 0; i < 4; ++i) {
12565       // Synthesize a zero mask from the zeroable elements (includes undefs).
12566       if (Zeroable[i]) {
12567         ZMask |= 1 << i;
12568         continue;
12569       }
12570 
12571       // Flag if we use any VA inputs in place.
12572       if (i == CandidateMask[i]) {
12573         VAUsedInPlace = true;
12574         continue;
12575       }
12576 
12577       // We can only insert a single non-zeroable element.
12578       if (VADstIndex >= 0 || VBDstIndex >= 0)
12579         return false;
12580 
12581       if (CandidateMask[i] < 4) {
12582         // VA input out of place for insertion.
12583         VADstIndex = i;
12584       } else {
12585         // VB input for insertion.
12586         VBDstIndex = i;
12587       }
12588     }
12589 
12590     // Don't bother if we have no (non-zeroable) element for insertion.
12591     if (VADstIndex < 0 && VBDstIndex < 0)
12592       return false;
12593 
12594     // Determine element insertion src/dst indices. The src index is from the
12595     // start of the inserted vector, not the start of the concatenated vector.
12596     unsigned VBSrcIndex = 0;
12597     if (VADstIndex >= 0) {
12598       // If we have a VA input out of place, we use VA as the V2 element
12599       // insertion and don't use the original V2 at all.
12600       VBSrcIndex = CandidateMask[VADstIndex];
12601       VBDstIndex = VADstIndex;
12602       VB = VA;
12603     } else {
12604       VBSrcIndex = CandidateMask[VBDstIndex] - 4;
12605     }
12606 
12607     // If no V1 inputs are used in place, then the result is created only from
12608     // the zero mask and the V2 insertion - so remove V1 dependency.
12609     if (!VAUsedInPlace)
12610       VA = DAG.getUNDEF(MVT::v4f32);
12611 
12612     // Update V1, V2 and InsertPSMask accordingly.
12613     V1 = VA;
12614     V2 = VB;
12615 
12616     // Insert the V2 element into the desired position.
12617     InsertPSMask = VBSrcIndex << 6 | VBDstIndex << 4 | ZMask;
12618     assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
12619     return true;
12620   };
12621 
12622   if (matchAsInsertPS(V1, V2, Mask))
12623     return true;
12624 
12625   // Commute and try again.
12626   SmallVector<int, 4> CommutedMask(Mask);
12627   ShuffleVectorSDNode::commuteMask(CommutedMask);
12628   if (matchAsInsertPS(V2, V1, CommutedMask))
12629     return true;
12630 
12631   return false;
12632 }
12633 
12634 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12635                                       ArrayRef<int> Mask, const APInt &Zeroable,
12636                                       SelectionDAG &DAG) {
12637   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12638   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12639 
12640   // Attempt to match the insertps pattern.
12641   unsigned InsertPSMask = 0;
12642   if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
12643     return SDValue();
12644 
12645   // Insert the V2 element into the desired position.
12646   return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
12647                      DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
12648 }
12649 
12650 /// Handle lowering of 2-lane 64-bit floating point shuffles.
12651 ///
12652 /// This is the basis function for the 2-lane 64-bit shuffles as we have full
12653 /// support for floating point shuffles but not integer shuffles. These
12654 /// instructions will incur a domain crossing penalty on some chips though so
12655 /// it is better to avoid lowering through this for integer vectors where
12656 /// possible.
12657 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12658                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12659                                  const X86Subtarget &Subtarget,
12660                                  SelectionDAG &DAG) {
12661   assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12662   assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12663   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12664 
12665   if (V2.isUndef()) {
12666     // Check for being able to broadcast a single element.
12667     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
12668                                                     Mask, Subtarget, DAG))
12669       return Broadcast;
12670 
12671     // Straight shuffle of a single input vector. Simulate this by using the
12672     // single input as both of the "inputs" to this instruction..
12673     unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
12674 
12675     if (Subtarget.hasAVX()) {
12676       // If we have AVX, we can use VPERMILPS which will allow folding a load
12677       // into the shuffle.
12678       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
12679                          DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12680     }
12681 
12682     return DAG.getNode(
12683         X86ISD::SHUFP, DL, MVT::v2f64,
12684         Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12685         Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12686         DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12687   }
12688   assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12689   assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12690   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12691   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12692 
12693   if (Subtarget.hasAVX2())
12694     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12695       return Extract;
12696 
12697   // When loading a scalar and then shuffling it into a vector we can often do
12698   // the insertion cheaply.
12699   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12700           DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12701     return Insertion;
12702   // Try inverting the insertion since for v2 masks it is easy to do and we
12703   // can't reliably sort the mask one way or the other.
12704   int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
12705                         Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
12706   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12707           DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12708     return Insertion;
12709 
12710   // Try to use one of the special instruction patterns to handle two common
12711   // blend patterns if a zero-blend above didn't work.
12712   if (isShuffleEquivalent(Mask, {0, 3}, V1, V2) ||
12713       isShuffleEquivalent(Mask, {1, 3}, V1, V2))
12714     if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
12715       // We can either use a special instruction to load over the low double or
12716       // to move just the low double.
12717       return DAG.getNode(
12718           X86ISD::MOVSD, DL, MVT::v2f64, V2,
12719           DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
12720 
12721   if (Subtarget.hasSSE41())
12722     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
12723                                             Zeroable, Subtarget, DAG))
12724       return Blend;
12725 
12726   // Use dedicated unpack instructions for masks that match their pattern.
12727   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
12728     return V;
12729 
12730   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
12731   return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
12732                      DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12733 }
12734 
12735 /// Handle lowering of 2-lane 64-bit integer shuffles.
12736 ///
12737 /// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
12738 /// the integer unit to minimize domain crossing penalties. However, for blends
12739 /// it falls back to the floating point shuffle operation with appropriate bit
12740 /// casting.
12741 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12742                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12743                                  const X86Subtarget &Subtarget,
12744                                  SelectionDAG &DAG) {
12745   assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12746   assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12747   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12748 
12749   if (V2.isUndef()) {
12750     // Check for being able to broadcast a single element.
12751     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
12752                                                     Mask, Subtarget, DAG))
12753       return Broadcast;
12754 
12755     // Straight shuffle of a single input vector. For everything from SSE2
12756     // onward this has a single fast instruction with no scary immediates.
12757     // We have to map the mask as it is actually a v4i32 shuffle instruction.
12758     V1 = DAG.getBitcast(MVT::v4i32, V1);
12759     int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2),
12760                           Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1),
12761                           Mask[1] < 0 ? -1 : (Mask[1] * 2),
12762                           Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)};
12763     return DAG.getBitcast(
12764         MVT::v2i64,
12765         DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
12766                     getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
12767   }
12768   assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!");
12769   assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!");
12770   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12771   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12772 
12773   if (Subtarget.hasAVX2())
12774     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12775       return Extract;
12776 
12777   // Try to use shift instructions.
12778   if (SDValue Shift =
12779           lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget,
12780                               DAG, /*BitwiseOnly*/ false))
12781     return Shift;
12782 
12783   // When loading a scalar and then shuffling it into a vector we can often do
12784   // the insertion cheaply.
12785   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12786           DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12787     return Insertion;
12788   // Try inverting the insertion since for v2 masks it is easy to do and we
12789   // can't reliably sort the mask one way or the other.
12790   int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2};
12791   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12792           DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12793     return Insertion;
12794 
12795   // We have different paths for blend lowering, but they all must use the
12796   // *exact* same predicate.
12797   bool IsBlendSupported = Subtarget.hasSSE41();
12798   if (IsBlendSupported)
12799     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
12800                                             Zeroable, Subtarget, DAG))
12801       return Blend;
12802 
12803   // Use dedicated unpack instructions for masks that match their pattern.
12804   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
12805     return V;
12806 
12807   // Try to use byte rotation instructions.
12808   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
12809   if (Subtarget.hasSSSE3()) {
12810     if (Subtarget.hasVLX())
12811       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
12812                                                 Zeroable, Subtarget, DAG))
12813         return Rotate;
12814 
12815     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
12816                                                   Subtarget, DAG))
12817       return Rotate;
12818   }
12819 
12820   // If we have direct support for blends, we should lower by decomposing into
12821   // a permute. That will be faster than the domain cross.
12822   if (IsBlendSupported)
12823     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,
12824                                                 Subtarget, DAG);
12825 
12826   // We implement this with SHUFPD which is pretty lame because it will likely
12827   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
12828   // However, all the alternatives are still more cycles and newer chips don't
12829   // have this problem. It would be really nice if x86 had better shuffles here.
12830   V1 = DAG.getBitcast(MVT::v2f64, V1);
12831   V2 = DAG.getBitcast(MVT::v2f64, V2);
12832   return DAG.getBitcast(MVT::v2i64,
12833                         DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
12834 }
12835 
12836 /// Lower a vector shuffle using the SHUFPS instruction.
12837 ///
12838 /// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
12839 /// It makes no assumptions about whether this is the *best* lowering, it simply
12840 /// uses it.
12841 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
12842                                       ArrayRef<int> Mask, SDValue V1,
12843                                       SDValue V2, SelectionDAG &DAG) {
12844   SDValue LowV = V1, HighV = V2;
12845   SmallVector<int, 4> NewMask(Mask);
12846   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
12847 
12848   if (NumV2Elements == 1) {
12849     int V2Index = find_if(Mask, [](int M) { return M >= 4; }) - Mask.begin();
12850 
12851     // Compute the index adjacent to V2Index and in the same half by toggling
12852     // the low bit.
12853     int V2AdjIndex = V2Index ^ 1;
12854 
12855     if (Mask[V2AdjIndex] < 0) {
12856       // Handles all the cases where we have a single V2 element and an undef.
12857       // This will only ever happen in the high lanes because we commute the
12858       // vector otherwise.
12859       if (V2Index < 2)
12860         std::swap(LowV, HighV);
12861       NewMask[V2Index] -= 4;
12862     } else {
12863       // Handle the case where the V2 element ends up adjacent to a V1 element.
12864       // To make this work, blend them together as the first step.
12865       int V1Index = V2AdjIndex;
12866       int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
12867       V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
12868                        getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
12869 
12870       // Now proceed to reconstruct the final blend as we have the necessary
12871       // high or low half formed.
12872       if (V2Index < 2) {
12873         LowV = V2;
12874         HighV = V1;
12875       } else {
12876         HighV = V2;
12877       }
12878       NewMask[V1Index] = 2; // We put the V1 element in V2[2].
12879       NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
12880     }
12881   } else if (NumV2Elements == 2) {
12882     if (Mask[0] < 4 && Mask[1] < 4) {
12883       // Handle the easy case where we have V1 in the low lanes and V2 in the
12884       // high lanes.
12885       NewMask[2] -= 4;
12886       NewMask[3] -= 4;
12887     } else if (Mask[2] < 4 && Mask[3] < 4) {
12888       // We also handle the reversed case because this utility may get called
12889       // when we detect a SHUFPS pattern but can't easily commute the shuffle to
12890       // arrange things in the right direction.
12891       NewMask[0] -= 4;
12892       NewMask[1] -= 4;
12893       HighV = V1;
12894       LowV = V2;
12895     } else {
12896       // We have a mixture of V1 and V2 in both low and high lanes. Rather than
12897       // trying to place elements directly, just blend them and set up the final
12898       // shuffle to place them.
12899 
12900       // The first two blend mask elements are for V1, the second two are for
12901       // V2.
12902       int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
12903                           Mask[2] < 4 ? Mask[2] : Mask[3],
12904                           (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
12905                           (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
12906       V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
12907                        getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
12908 
12909       // Now we do a normal shuffle of V1 by giving V1 as both operands to
12910       // a blend.
12911       LowV = HighV = V1;
12912       NewMask[0] = Mask[0] < 4 ? 0 : 2;
12913       NewMask[1] = Mask[0] < 4 ? 2 : 0;
12914       NewMask[2] = Mask[2] < 4 ? 1 : 3;
12915       NewMask[3] = Mask[2] < 4 ? 3 : 1;
12916     }
12917   } else if (NumV2Elements == 3) {
12918     // Ideally canonicalizeShuffleMaskWithCommute should have caught this, but
12919     // we can get here due to other paths (e.g repeated mask matching) that we
12920     // don't want to do another round of lowerVECTOR_SHUFFLE.
12921     ShuffleVectorSDNode::commuteMask(NewMask);
12922     return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG);
12923   }
12924   return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
12925                      getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
12926 }
12927 
12928 /// Lower 4-lane 32-bit floating point shuffles.
12929 ///
12930 /// Uses instructions exclusively from the floating point unit to minimize
12931 /// domain crossing penalties, as these are sufficient to implement all v4f32
12932 /// shuffles.
12933 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12934                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12935                                  const X86Subtarget &Subtarget,
12936                                  SelectionDAG &DAG) {
12937   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12938   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12939   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
12940 
12941   if (Subtarget.hasSSE41())
12942     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
12943                                             Zeroable, Subtarget, DAG))
12944       return Blend;
12945 
12946   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
12947 
12948   if (NumV2Elements == 0) {
12949     // Check for being able to broadcast a single element.
12950     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
12951                                                     Mask, Subtarget, DAG))
12952       return Broadcast;
12953 
12954     // Use even/odd duplicate instructions for masks that match their pattern.
12955     if (Subtarget.hasSSE3()) {
12956       if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
12957         return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
12958       if (isShuffleEquivalent(Mask, {1, 1, 3, 3}, V1, V2))
12959         return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
12960     }
12961 
12962     if (Subtarget.hasAVX()) {
12963       // If we have AVX, we can use VPERMILPS which will allow folding a load
12964       // into the shuffle.
12965       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
12966                          getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
12967     }
12968 
12969     // Use MOVLHPS/MOVHLPS to simulate unary shuffles. These are only valid
12970     // in SSE1 because otherwise they are widened to v2f64 and never get here.
12971     if (!Subtarget.hasSSE2()) {
12972       if (isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2))
12973         return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
12974       if (isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1, V2))
12975         return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
12976     }
12977 
12978     // Otherwise, use a straight shuffle of a single input vector. We pass the
12979     // input vector to both operands to simulate this with a SHUFPS.
12980     return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
12981                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
12982   }
12983 
12984   if (Subtarget.hasSSE2())
12985     if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
12986             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) {
12987       ZExt = DAG.getBitcast(MVT::v4f32, ZExt);
12988       return ZExt;
12989     }
12990 
12991   if (Subtarget.hasAVX2())
12992     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12993       return Extract;
12994 
12995   // There are special ways we can lower some single-element blends. However, we
12996   // have custom ways we can lower more complex single-element blends below that
12997   // we defer to if both this and BLENDPS fail to match, so restrict this to
12998   // when the V2 input is targeting element 0 of the mask -- that is the fast
12999   // case here.
13000   if (NumV2Elements == 1 && Mask[0] >= 4)
13001     if (SDValue V = lowerShuffleAsElementInsertion(
13002             DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13003       return V;
13004 
13005   if (Subtarget.hasSSE41()) {
13006     // Use INSERTPS if we can complete the shuffle efficiently.
13007     if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13008       return V;
13009 
13010     if (!isSingleSHUFPSMask(Mask))
13011       if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13012                                                             V2, Mask, DAG))
13013         return BlendPerm;
13014   }
13015 
13016   // Use low/high mov instructions. These are only valid in SSE1 because
13017   // otherwise they are widened to v2f64 and never get here.
13018   if (!Subtarget.hasSSE2()) {
13019     if (isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2))
13020       return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13021     if (isShuffleEquivalent(Mask, {2, 3, 6, 7}, V1, V2))
13022       return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13023   }
13024 
13025   // Use dedicated unpack instructions for masks that match their pattern.
13026   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13027     return V;
13028 
13029   // Otherwise fall back to a SHUFPS lowering strategy.
13030   return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13031 }
13032 
13033 /// Lower 4-lane i32 vector shuffles.
13034 ///
13035 /// We try to handle these with integer-domain shuffles where we can, but for
13036 /// blends we use the floating point domain blend instructions.
13037 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13038                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13039                                  const X86Subtarget &Subtarget,
13040                                  SelectionDAG &DAG) {
13041   assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13042   assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13043   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
13044 
13045   // Whenever we can lower this as a zext, that instruction is strictly faster
13046   // than any alternative. It also allows us to fold memory operands into the
13047   // shuffle in many cases.
13048   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13049                                                    Zeroable, Subtarget, DAG))
13050     return ZExt;
13051 
13052   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13053 
13054   // Try to use shift instructions if fast.
13055   if (Subtarget.preferLowerShuffleAsShift()) {
13056     if (SDValue Shift =
13057             lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable,
13058                                 Subtarget, DAG, /*BitwiseOnly*/ true))
13059       return Shift;
13060     if (NumV2Elements == 0)
13061       if (SDValue Rotate =
13062               lowerShuffleAsBitRotate(DL, MVT::v4i32, V1, Mask, Subtarget, DAG))
13063         return Rotate;
13064   }
13065 
13066   if (NumV2Elements == 0) {
13067     // Try to use broadcast unless the mask only has one non-undef element.
13068     if (count_if(Mask, [](int M) { return M >= 0 && M < 4; }) > 1) {
13069       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13070                                                       Mask, Subtarget, DAG))
13071         return Broadcast;
13072     }
13073 
13074     // Straight shuffle of a single input vector. For everything from SSE2
13075     // onward this has a single fast instruction with no scary immediates.
13076     // We coerce the shuffle pattern to be compatible with UNPCK instructions
13077     // but we aren't actually going to use the UNPCK instruction because doing
13078     // so prevents folding a load into this instruction or making a copy.
13079     const int UnpackLoMask[] = {0, 0, 1, 1};
13080     const int UnpackHiMask[] = {2, 2, 3, 3};
13081     if (isShuffleEquivalent(Mask, {0, 0, 1, 1}, V1, V2))
13082       Mask = UnpackLoMask;
13083     else if (isShuffleEquivalent(Mask, {2, 2, 3, 3}, V1, V2))
13084       Mask = UnpackHiMask;
13085 
13086     return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13087                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13088   }
13089 
13090   if (Subtarget.hasAVX2())
13091     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13092       return Extract;
13093 
13094   // Try to use shift instructions.
13095   if (SDValue Shift =
13096           lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget,
13097                               DAG, /*BitwiseOnly*/ false))
13098     return Shift;
13099 
13100   // There are special ways we can lower some single-element blends.
13101   if (NumV2Elements == 1)
13102     if (SDValue V = lowerShuffleAsElementInsertion(
13103             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13104       return V;
13105 
13106   // We have different paths for blend lowering, but they all must use the
13107   // *exact* same predicate.
13108   bool IsBlendSupported = Subtarget.hasSSE41();
13109   if (IsBlendSupported)
13110     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13111                                             Zeroable, Subtarget, DAG))
13112       return Blend;
13113 
13114   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13115                                              Zeroable, Subtarget, DAG))
13116     return Masked;
13117 
13118   // Use dedicated unpack instructions for masks that match their pattern.
13119   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
13120     return V;
13121 
13122   // Try to use byte rotation instructions.
13123   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
13124   if (Subtarget.hasSSSE3()) {
13125     if (Subtarget.hasVLX())
13126       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
13127                                                 Zeroable, Subtarget, DAG))
13128         return Rotate;
13129 
13130     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13131                                                   Subtarget, DAG))
13132       return Rotate;
13133   }
13134 
13135   // Assume that a single SHUFPS is faster than an alternative sequence of
13136   // multiple instructions (even if the CPU has a domain penalty).
13137   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
13138   if (!isSingleSHUFPSMask(Mask)) {
13139     // If we have direct support for blends, we should lower by decomposing into
13140     // a permute. That will be faster than the domain cross.
13141     if (IsBlendSupported)
13142       return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,
13143                                                   Subtarget, DAG);
13144 
13145     // Try to lower by permuting the inputs into an unpack instruction.
13146     if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13147                                                         Mask, Subtarget, DAG))
13148       return Unpack;
13149   }
13150 
13151   // We implement this with SHUFPS because it can blend from two vectors.
13152   // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
13153   // up the inputs, bypassing domain shift penalties that we would incur if we
13154   // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
13155   // relevant.
13156   SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13157   SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13158   SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13159   return DAG.getBitcast(MVT::v4i32, ShufPS);
13160 }
13161 
13162 /// Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
13163 /// shuffle lowering, and the most complex part.
13164 ///
13165 /// The lowering strategy is to try to form pairs of input lanes which are
13166 /// targeted at the same half of the final vector, and then use a dword shuffle
13167 /// to place them onto the right half, and finally unpack the paired lanes into
13168 /// their final position.
13169 ///
13170 /// The exact breakdown of how to form these dword pairs and align them on the
13171 /// correct sides is really tricky. See the comments within the function for
13172 /// more of the details.
13173 ///
13174 /// This code also handles repeated 128-bit lanes of v8i16 shuffles, but each
13175 /// lane must shuffle the *exact* same way. In fact, you must pass a v8 Mask to
13176 /// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
13177 /// vector, form the analogous 128-bit 8-element Mask.
13178 static SDValue lowerV8I16GeneralSingleInputShuffle(
13179     const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
13180     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
13181   assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");
13182   MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
13183 
13184   assert(Mask.size() == 8 && "Shuffle mask length doesn't match!");
13185   MutableArrayRef<int> LoMask = Mask.slice(0, 4);
13186   MutableArrayRef<int> HiMask = Mask.slice(4, 4);
13187 
13188   // Attempt to directly match PSHUFLW or PSHUFHW.
13189   if (isUndefOrInRange(LoMask, 0, 4) &&
13190       isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
13191     return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13192                        getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13193   }
13194   if (isUndefOrInRange(HiMask, 4, 8) &&
13195       isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
13196     for (int i = 0; i != 4; ++i)
13197       HiMask[i] = (HiMask[i] < 0 ? HiMask[i] : (HiMask[i] - 4));
13198     return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13199                        getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13200   }
13201 
13202   SmallVector<int, 4> LoInputs;
13203   copy_if(LoMask, std::back_inserter(LoInputs), [](int M) { return M >= 0; });
13204   array_pod_sort(LoInputs.begin(), LoInputs.end());
13205   LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
13206   SmallVector<int, 4> HiInputs;
13207   copy_if(HiMask, std::back_inserter(HiInputs), [](int M) { return M >= 0; });
13208   array_pod_sort(HiInputs.begin(), HiInputs.end());
13209   HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
13210   int NumLToL = llvm::lower_bound(LoInputs, 4) - LoInputs.begin();
13211   int NumHToL = LoInputs.size() - NumLToL;
13212   int NumLToH = llvm::lower_bound(HiInputs, 4) - HiInputs.begin();
13213   int NumHToH = HiInputs.size() - NumLToH;
13214   MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
13215   MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
13216   MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
13217   MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
13218 
13219   // If we are shuffling values from one half - check how many different DWORD
13220   // pairs we need to create. If only 1 or 2 then we can perform this as a
13221   // PSHUFLW/PSHUFHW + PSHUFD instead of the PSHUFD+PSHUFLW+PSHUFHW chain below.
13222   auto ShuffleDWordPairs = [&](ArrayRef<int> PSHUFHalfMask,
13223                                ArrayRef<int> PSHUFDMask, unsigned ShufWOp) {
13224     V = DAG.getNode(ShufWOp, DL, VT, V,
13225                     getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13226     V = DAG.getBitcast(PSHUFDVT, V);
13227     V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
13228                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
13229     return DAG.getBitcast(VT, V);
13230   };
13231 
13232   if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
13233     int PSHUFDMask[4] = { -1, -1, -1, -1 };
13234     SmallVector<std::pair<int, int>, 4> DWordPairs;
13235     int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
13236 
13237     // Collect the different DWORD pairs.
13238     for (int DWord = 0; DWord != 4; ++DWord) {
13239       int M0 = Mask[2 * DWord + 0];
13240       int M1 = Mask[2 * DWord + 1];
13241       M0 = (M0 >= 0 ? M0 % 4 : M0);
13242       M1 = (M1 >= 0 ? M1 % 4 : M1);
13243       if (M0 < 0 && M1 < 0)
13244         continue;
13245 
13246       bool Match = false;
13247       for (int j = 0, e = DWordPairs.size(); j < e; ++j) {
13248         auto &DWordPair = DWordPairs[j];
13249         if ((M0 < 0 || isUndefOrEqual(DWordPair.first, M0)) &&
13250             (M1 < 0 || isUndefOrEqual(DWordPair.second, M1))) {
13251           DWordPair.first = (M0 >= 0 ? M0 : DWordPair.first);
13252           DWordPair.second = (M1 >= 0 ? M1 : DWordPair.second);
13253           PSHUFDMask[DWord] = DOffset + j;
13254           Match = true;
13255           break;
13256         }
13257       }
13258       if (!Match) {
13259         PSHUFDMask[DWord] = DOffset + DWordPairs.size();
13260         DWordPairs.push_back(std::make_pair(M0, M1));
13261       }
13262     }
13263 
13264     if (DWordPairs.size() <= 2) {
13265       DWordPairs.resize(2, std::make_pair(-1, -1));
13266       int PSHUFHalfMask[4] = {DWordPairs[0].first, DWordPairs[0].second,
13267                               DWordPairs[1].first, DWordPairs[1].second};
13268       if ((NumHToL + NumHToH) == 0)
13269         return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFLW);
13270       if ((NumLToL + NumLToH) == 0)
13271         return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFHW);
13272     }
13273   }
13274 
13275   // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
13276   // such inputs we can swap two of the dwords across the half mark and end up
13277   // with <=2 inputs to each half in each half. Once there, we can fall through
13278   // to the generic code below. For example:
13279   //
13280   // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13281   // Mask:  [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
13282   //
13283   // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
13284   // and an existing 2-into-2 on the other half. In this case we may have to
13285   // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
13286   // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
13287   // Fortunately, we don't have to handle anything but a 2-into-2 pattern
13288   // because any other situation (including a 3-into-1 or 1-into-3 in the other
13289   // half than the one we target for fixing) will be fixed when we re-enter this
13290   // path. We will also combine away any sequence of PSHUFD instructions that
13291   // result into a single instruction. Here is an example of the tricky case:
13292   //
13293   // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13294   // Mask:  [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
13295   //
13296   // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
13297   //
13298   // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
13299   // Mask:  [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
13300   //
13301   // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
13302   // Mask:  [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
13303   //
13304   // The result is fine to be handled by the generic logic.
13305   auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
13306                           ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
13307                           int AOffset, int BOffset) {
13308     assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
13309            "Must call this with A having 3 or 1 inputs from the A half.");
13310     assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
13311            "Must call this with B having 1 or 3 inputs from the B half.");
13312     assert(AToAInputs.size() + BToAInputs.size() == 4 &&
13313            "Must call this with either 3:1 or 1:3 inputs (summing to 4).");
13314 
13315     bool ThreeAInputs = AToAInputs.size() == 3;
13316 
13317     // Compute the index of dword with only one word among the three inputs in
13318     // a half by taking the sum of the half with three inputs and subtracting
13319     // the sum of the actual three inputs. The difference is the remaining
13320     // slot.
13321     int ADWord = 0, BDWord = 0;
13322     int &TripleDWord = ThreeAInputs ? ADWord : BDWord;
13323     int &OneInputDWord = ThreeAInputs ? BDWord : ADWord;
13324     int TripleInputOffset = ThreeAInputs ? AOffset : BOffset;
13325     ArrayRef<int> TripleInputs = ThreeAInputs ? AToAInputs : BToAInputs;
13326     int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
13327     int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
13328     int TripleNonInputIdx =
13329         TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
13330     TripleDWord = TripleNonInputIdx / 2;
13331 
13332     // We use xor with one to compute the adjacent DWord to whichever one the
13333     // OneInput is in.
13334     OneInputDWord = (OneInput / 2) ^ 1;
13335 
13336     // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
13337     // and BToA inputs. If there is also such a problem with the BToB and AToB
13338     // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
13339     // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
13340     // is essential that we don't *create* a 3<-1 as then we might oscillate.
13341     if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
13342       // Compute how many inputs will be flipped by swapping these DWords. We
13343       // need
13344       // to balance this to ensure we don't form a 3-1 shuffle in the other
13345       // half.
13346       int NumFlippedAToBInputs = llvm::count(AToBInputs, 2 * ADWord) +
13347                                  llvm::count(AToBInputs, 2 * ADWord + 1);
13348       int NumFlippedBToBInputs = llvm::count(BToBInputs, 2 * BDWord) +
13349                                  llvm::count(BToBInputs, 2 * BDWord + 1);
13350       if ((NumFlippedAToBInputs == 1 &&
13351            (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
13352           (NumFlippedBToBInputs == 1 &&
13353            (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
13354         // We choose whether to fix the A half or B half based on whether that
13355         // half has zero flipped inputs. At zero, we may not be able to fix it
13356         // with that half. We also bias towards fixing the B half because that
13357         // will more commonly be the high half, and we have to bias one way.
13358         auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
13359                                                        ArrayRef<int> Inputs) {
13360           int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
13361           bool IsFixIdxInput = is_contained(Inputs, PinnedIdx ^ 1);
13362           // Determine whether the free index is in the flipped dword or the
13363           // unflipped dword based on where the pinned index is. We use this bit
13364           // in an xor to conditionally select the adjacent dword.
13365           int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
13366           bool IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13367           if (IsFixIdxInput == IsFixFreeIdxInput)
13368             FixFreeIdx += 1;
13369           IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13370           assert(IsFixIdxInput != IsFixFreeIdxInput &&
13371                  "We need to be changing the number of flipped inputs!");
13372           int PSHUFHalfMask[] = {0, 1, 2, 3};
13373           std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
13374           V = DAG.getNode(
13375               FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
13376               MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V,
13377               getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13378 
13379           for (int &M : Mask)
13380             if (M >= 0 && M == FixIdx)
13381               M = FixFreeIdx;
13382             else if (M >= 0 && M == FixFreeIdx)
13383               M = FixIdx;
13384         };
13385         if (NumFlippedBToBInputs != 0) {
13386           int BPinnedIdx =
13387               BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
13388           FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
13389         } else {
13390           assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!");
13391           int APinnedIdx = ThreeAInputs ? TripleNonInputIdx : OneInput;
13392           FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
13393         }
13394       }
13395     }
13396 
13397     int PSHUFDMask[] = {0, 1, 2, 3};
13398     PSHUFDMask[ADWord] = BDWord;
13399     PSHUFDMask[BDWord] = ADWord;
13400     V = DAG.getBitcast(
13401         VT,
13402         DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13403                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13404 
13405     // Adjust the mask to match the new locations of A and B.
13406     for (int &M : Mask)
13407       if (M >= 0 && M/2 == ADWord)
13408         M = 2 * BDWord + M % 2;
13409       else if (M >= 0 && M/2 == BDWord)
13410         M = 2 * ADWord + M % 2;
13411 
13412     // Recurse back into this routine to re-compute state now that this isn't
13413     // a 3 and 1 problem.
13414     return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
13415   };
13416   if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
13417     return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
13418   if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
13419     return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
13420 
13421   // At this point there are at most two inputs to the low and high halves from
13422   // each half. That means the inputs can always be grouped into dwords and
13423   // those dwords can then be moved to the correct half with a dword shuffle.
13424   // We use at most one low and one high word shuffle to collect these paired
13425   // inputs into dwords, and finally a dword shuffle to place them.
13426   int PSHUFLMask[4] = {-1, -1, -1, -1};
13427   int PSHUFHMask[4] = {-1, -1, -1, -1};
13428   int PSHUFDMask[4] = {-1, -1, -1, -1};
13429 
13430   // First fix the masks for all the inputs that are staying in their
13431   // original halves. This will then dictate the targets of the cross-half
13432   // shuffles.
13433   auto fixInPlaceInputs =
13434       [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
13435                     MutableArrayRef<int> SourceHalfMask,
13436                     MutableArrayRef<int> HalfMask, int HalfOffset) {
13437     if (InPlaceInputs.empty())
13438       return;
13439     if (InPlaceInputs.size() == 1) {
13440       SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13441           InPlaceInputs[0] - HalfOffset;
13442       PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
13443       return;
13444     }
13445     if (IncomingInputs.empty()) {
13446       // Just fix all of the in place inputs.
13447       for (int Input : InPlaceInputs) {
13448         SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
13449         PSHUFDMask[Input / 2] = Input / 2;
13450       }
13451       return;
13452     }
13453 
13454     assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
13455     SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13456         InPlaceInputs[0] - HalfOffset;
13457     // Put the second input next to the first so that they are packed into
13458     // a dword. We find the adjacent index by toggling the low bit.
13459     int AdjIndex = InPlaceInputs[0] ^ 1;
13460     SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
13461     std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
13462     PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
13463   };
13464   fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
13465   fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
13466 
13467   // Now gather the cross-half inputs and place them into a free dword of
13468   // their target half.
13469   // FIXME: This operation could almost certainly be simplified dramatically to
13470   // look more like the 3-1 fixing operation.
13471   auto moveInputsToRightHalf = [&PSHUFDMask](
13472       MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
13473       MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
13474       MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
13475       int DestOffset) {
13476     auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
13477       return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
13478     };
13479     auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
13480                                                int Word) {
13481       int LowWord = Word & ~1;
13482       int HighWord = Word | 1;
13483       return isWordClobbered(SourceHalfMask, LowWord) ||
13484              isWordClobbered(SourceHalfMask, HighWord);
13485     };
13486 
13487     if (IncomingInputs.empty())
13488       return;
13489 
13490     if (ExistingInputs.empty()) {
13491       // Map any dwords with inputs from them into the right half.
13492       for (int Input : IncomingInputs) {
13493         // If the source half mask maps over the inputs, turn those into
13494         // swaps and use the swapped lane.
13495         if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
13496           if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] < 0) {
13497             SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
13498                 Input - SourceOffset;
13499             // We have to swap the uses in our half mask in one sweep.
13500             for (int &M : HalfMask)
13501               if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
13502                 M = Input;
13503               else if (M == Input)
13504                 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13505           } else {
13506             assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
13507                        Input - SourceOffset &&
13508                    "Previous placement doesn't match!");
13509           }
13510           // Note that this correctly re-maps both when we do a swap and when
13511           // we observe the other side of the swap above. We rely on that to
13512           // avoid swapping the members of the input list directly.
13513           Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13514         }
13515 
13516         // Map the input's dword into the correct half.
13517         if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] < 0)
13518           PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
13519         else
13520           assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
13521                      Input / 2 &&
13522                  "Previous placement doesn't match!");
13523       }
13524 
13525       // And just directly shift any other-half mask elements to be same-half
13526       // as we will have mirrored the dword containing the element into the
13527       // same position within that half.
13528       for (int &M : HalfMask)
13529         if (M >= SourceOffset && M < SourceOffset + 4) {
13530           M = M - SourceOffset + DestOffset;
13531           assert(M >= 0 && "This should never wrap below zero!");
13532         }
13533       return;
13534     }
13535 
13536     // Ensure we have the input in a viable dword of its current half. This
13537     // is particularly tricky because the original position may be clobbered
13538     // by inputs being moved and *staying* in that half.
13539     if (IncomingInputs.size() == 1) {
13540       if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13541         int InputFixed = find(SourceHalfMask, -1) - std::begin(SourceHalfMask) +
13542                          SourceOffset;
13543         SourceHalfMask[InputFixed - SourceOffset] =
13544             IncomingInputs[0] - SourceOffset;
13545         std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
13546                      InputFixed);
13547         IncomingInputs[0] = InputFixed;
13548       }
13549     } else if (IncomingInputs.size() == 2) {
13550       if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
13551           isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13552         // We have two non-adjacent or clobbered inputs we need to extract from
13553         // the source half. To do this, we need to map them into some adjacent
13554         // dword slot in the source mask.
13555         int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
13556                               IncomingInputs[1] - SourceOffset};
13557 
13558         // If there is a free slot in the source half mask adjacent to one of
13559         // the inputs, place the other input in it. We use (Index XOR 1) to
13560         // compute an adjacent index.
13561         if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
13562             SourceHalfMask[InputsFixed[0] ^ 1] < 0) {
13563           SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
13564           SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13565           InputsFixed[1] = InputsFixed[0] ^ 1;
13566         } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
13567                    SourceHalfMask[InputsFixed[1] ^ 1] < 0) {
13568           SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
13569           SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
13570           InputsFixed[0] = InputsFixed[1] ^ 1;
13571         } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] < 0 &&
13572                    SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] < 0) {
13573           // The two inputs are in the same DWord but it is clobbered and the
13574           // adjacent DWord isn't used at all. Move both inputs to the free
13575           // slot.
13576           SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
13577           SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
13578           InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
13579           InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
13580         } else {
13581           // The only way we hit this point is if there is no clobbering
13582           // (because there are no off-half inputs to this half) and there is no
13583           // free slot adjacent to one of the inputs. In this case, we have to
13584           // swap an input with a non-input.
13585           for (int i = 0; i < 4; ++i)
13586             assert((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&
13587                    "We can't handle any clobbers here!");
13588           assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&
13589                  "Cannot have adjacent inputs here!");
13590 
13591           SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13592           SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
13593 
13594           // We also have to update the final source mask in this case because
13595           // it may need to undo the above swap.
13596           for (int &M : FinalSourceHalfMask)
13597             if (M == (InputsFixed[0] ^ 1) + SourceOffset)
13598               M = InputsFixed[1] + SourceOffset;
13599             else if (M == InputsFixed[1] + SourceOffset)
13600               M = (InputsFixed[0] ^ 1) + SourceOffset;
13601 
13602           InputsFixed[1] = InputsFixed[0] ^ 1;
13603         }
13604 
13605         // Point everything at the fixed inputs.
13606         for (int &M : HalfMask)
13607           if (M == IncomingInputs[0])
13608             M = InputsFixed[0] + SourceOffset;
13609           else if (M == IncomingInputs[1])
13610             M = InputsFixed[1] + SourceOffset;
13611 
13612         IncomingInputs[0] = InputsFixed[0] + SourceOffset;
13613         IncomingInputs[1] = InputsFixed[1] + SourceOffset;
13614       }
13615     } else {
13616       llvm_unreachable("Unhandled input size!");
13617     }
13618 
13619     // Now hoist the DWord down to the right half.
13620     int FreeDWord = (PSHUFDMask[DestOffset / 2] < 0 ? 0 : 1) + DestOffset / 2;
13621     assert(PSHUFDMask[FreeDWord] < 0 && "DWord not free");
13622     PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
13623     for (int &M : HalfMask)
13624       for (int Input : IncomingInputs)
13625         if (M == Input)
13626           M = FreeDWord * 2 + Input % 2;
13627   };
13628   moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
13629                         /*SourceOffset*/ 4, /*DestOffset*/ 0);
13630   moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
13631                         /*SourceOffset*/ 0, /*DestOffset*/ 4);
13632 
13633   // Now enact all the shuffles we've computed to move the inputs into their
13634   // target half.
13635   if (!isNoopShuffleMask(PSHUFLMask))
13636     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13637                     getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
13638   if (!isNoopShuffleMask(PSHUFHMask))
13639     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13640                     getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
13641   if (!isNoopShuffleMask(PSHUFDMask))
13642     V = DAG.getBitcast(
13643         VT,
13644         DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13645                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13646 
13647   // At this point, each half should contain all its inputs, and we can then
13648   // just shuffle them into their final position.
13649   assert(count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
13650          "Failed to lift all the high half inputs to the low mask!");
13651   assert(count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 &&
13652          "Failed to lift all the low half inputs to the high mask!");
13653 
13654   // Do a half shuffle for the low mask.
13655   if (!isNoopShuffleMask(LoMask))
13656     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13657                     getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13658 
13659   // Do a half shuffle with the high mask after shifting its values down.
13660   for (int &M : HiMask)
13661     if (M >= 0)
13662       M -= 4;
13663   if (!isNoopShuffleMask(HiMask))
13664     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13665                     getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13666 
13667   return V;
13668 }
13669 
13670 /// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
13671 /// blend if only one input is used.
13672 static SDValue lowerShuffleAsBlendOfPSHUFBs(
13673     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13674     const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
13675   assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
13676          "Lane crossing shuffle masks not supported");
13677 
13678   int NumBytes = VT.getSizeInBits() / 8;
13679   int Size = Mask.size();
13680   int Scale = NumBytes / Size;
13681 
13682   SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13683   SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13684   V1InUse = false;
13685   V2InUse = false;
13686 
13687   for (int i = 0; i < NumBytes; ++i) {
13688     int M = Mask[i / Scale];
13689     if (M < 0)
13690       continue;
13691 
13692     const int ZeroMask = 0x80;
13693     int V1Idx = M < Size ? M * Scale + i % Scale : ZeroMask;
13694     int V2Idx = M < Size ? ZeroMask : (M - Size) * Scale + i % Scale;
13695     if (Zeroable[i / Scale])
13696       V1Idx = V2Idx = ZeroMask;
13697 
13698     V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
13699     V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
13700     V1InUse |= (ZeroMask != V1Idx);
13701     V2InUse |= (ZeroMask != V2Idx);
13702   }
13703 
13704   MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes);
13705   if (V1InUse)
13706     V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
13707                      DAG.getBuildVector(ShufVT, DL, V1Mask));
13708   if (V2InUse)
13709     V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
13710                      DAG.getBuildVector(ShufVT, DL, V2Mask));
13711 
13712   // If we need shuffled inputs from both, blend the two.
13713   SDValue V;
13714   if (V1InUse && V2InUse)
13715     V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
13716   else
13717     V = V1InUse ? V1 : V2;
13718 
13719   // Cast the result back to the correct type.
13720   return DAG.getBitcast(VT, V);
13721 }
13722 
13723 /// Generic lowering of 8-lane i16 shuffles.
13724 ///
13725 /// This handles both single-input shuffles and combined shuffle/blends with
13726 /// two inputs. The single input shuffles are immediately delegated to
13727 /// a dedicated lowering routine.
13728 ///
13729 /// The blends are lowered in one of three fundamental ways. If there are few
13730 /// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
13731 /// of the input is significantly cheaper when lowered as an interleaving of
13732 /// the two inputs, try to interleave them. Otherwise, blend the low and high
13733 /// halves of the inputs separately (making them have relatively few inputs)
13734 /// and then concatenate them.
13735 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13736                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13737                                  const X86Subtarget &Subtarget,
13738                                  SelectionDAG &DAG) {
13739   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13740   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13741   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
13742 
13743   // Whenever we can lower this as a zext, that instruction is strictly faster
13744   // than any alternative.
13745   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
13746                                                    Zeroable, Subtarget, DAG))
13747     return ZExt;
13748 
13749   // Try to use lower using a truncation.
13750   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13751                                         Subtarget, DAG))
13752     return V;
13753 
13754   int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; });
13755 
13756   if (NumV2Inputs == 0) {
13757     // Try to use shift instructions.
13758     if (SDValue Shift =
13759             lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, Zeroable,
13760                                 Subtarget, DAG, /*BitwiseOnly*/ false))
13761       return Shift;
13762 
13763     // Check for being able to broadcast a single element.
13764     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
13765                                                     Mask, Subtarget, DAG))
13766       return Broadcast;
13767 
13768     // Try to use bit rotation instructions.
13769     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
13770                                                  Subtarget, DAG))
13771       return Rotate;
13772 
13773     // Use dedicated unpack instructions for masks that match their pattern.
13774     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13775       return V;
13776 
13777     // Use dedicated pack instructions for masks that match their pattern.
13778     if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13779                                          Subtarget))
13780       return V;
13781 
13782     // Try to use byte rotation instructions.
13783     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
13784                                                   Subtarget, DAG))
13785       return Rotate;
13786 
13787     // Make a copy of the mask so it can be modified.
13788     SmallVector<int, 8> MutableMask(Mask);
13789     return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
13790                                                Subtarget, DAG);
13791   }
13792 
13793   assert(llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) &&
13794          "All single-input shuffles should be canonicalized to be V1-input "
13795          "shuffles.");
13796 
13797   // Try to use shift instructions.
13798   if (SDValue Shift =
13799           lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget,
13800                               DAG, /*BitwiseOnly*/ false))
13801     return Shift;
13802 
13803   // See if we can use SSE4A Extraction / Insertion.
13804   if (Subtarget.hasSSE4A())
13805     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
13806                                           Zeroable, DAG))
13807       return V;
13808 
13809   // There are special ways we can lower some single-element blends.
13810   if (NumV2Inputs == 1)
13811     if (SDValue V = lowerShuffleAsElementInsertion(
13812             DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
13813       return V;
13814 
13815   // We have different paths for blend lowering, but they all must use the
13816   // *exact* same predicate.
13817   bool IsBlendSupported = Subtarget.hasSSE41();
13818   if (IsBlendSupported)
13819     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
13820                                             Zeroable, Subtarget, DAG))
13821       return Blend;
13822 
13823   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
13824                                              Zeroable, Subtarget, DAG))
13825     return Masked;
13826 
13827   // Use dedicated unpack instructions for masks that match their pattern.
13828   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13829     return V;
13830 
13831   // Use dedicated pack instructions for masks that match their pattern.
13832   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13833                                        Subtarget))
13834     return V;
13835 
13836   // Try to use lower using a truncation.
13837   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13838                                        Subtarget, DAG))
13839     return V;
13840 
13841   // Try to use byte rotation instructions.
13842   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
13843                                                 Subtarget, DAG))
13844     return Rotate;
13845 
13846   if (SDValue BitBlend =
13847           lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
13848     return BitBlend;
13849 
13850   // Try to use byte shift instructions to mask.
13851   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
13852                                               Zeroable, Subtarget, DAG))
13853     return V;
13854 
13855   // Attempt to lower using compaction, SSE41 is necessary for PACKUSDW.
13856   int NumEvenDrops = canLowerByDroppingElements(Mask, true, false);
13857   if ((NumEvenDrops == 1 || (NumEvenDrops == 2 && Subtarget.hasSSE41())) &&
13858       !Subtarget.hasVLX()) {
13859     // Check if this is part of a 256-bit vector truncation.
13860     unsigned PackOpc = 0;
13861     if (NumEvenDrops == 2 && Subtarget.hasAVX2() &&
13862         peekThroughBitcasts(V1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13863         peekThroughBitcasts(V2).getOpcode() == ISD::EXTRACT_SUBVECTOR) {
13864       SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL);
13865       V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2,
13866                          getZeroVector(MVT::v16i16, Subtarget, DAG, DL),
13867                          DAG.getTargetConstant(0xEE, DL, MVT::i8));
13868       V1V2 = DAG.getBitcast(MVT::v8i32, V1V2);
13869       V1 = extract128BitVector(V1V2, 0, DAG, DL);
13870       V2 = extract128BitVector(V1V2, 4, DAG, DL);
13871       PackOpc = X86ISD::PACKUS;
13872     } else if (Subtarget.hasSSE41()) {
13873       SmallVector<SDValue, 4> DWordClearOps(4,
13874                                             DAG.getConstant(0, DL, MVT::i32));
13875       for (unsigned i = 0; i != 4; i += 1 << (NumEvenDrops - 1))
13876         DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
13877       SDValue DWordClearMask =
13878           DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
13879       V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),
13880                        DWordClearMask);
13881       V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),
13882                        DWordClearMask);
13883       PackOpc = X86ISD::PACKUS;
13884     } else if (!Subtarget.hasSSSE3()) {
13885       SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8);
13886       V1 = DAG.getBitcast(MVT::v4i32, V1);
13887       V2 = DAG.getBitcast(MVT::v4i32, V2);
13888       V1 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V1, ShAmt);
13889       V2 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V2, ShAmt);
13890       V1 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V1, ShAmt);
13891       V2 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V2, ShAmt);
13892       PackOpc = X86ISD::PACKSS;
13893     }
13894     if (PackOpc) {
13895       // Now pack things back together.
13896       SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2);
13897       if (NumEvenDrops == 2) {
13898         Result = DAG.getBitcast(MVT::v4i32, Result);
13899         Result = DAG.getNode(PackOpc, DL, MVT::v8i16, Result, Result);
13900       }
13901       return Result;
13902     }
13903   }
13904 
13905   // When compacting odd (upper) elements, use PACKSS pre-SSE41.
13906   int NumOddDrops = canLowerByDroppingElements(Mask, false, false);
13907   if (NumOddDrops == 1) {
13908     bool HasSSE41 = Subtarget.hasSSE41();
13909     V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
13910                      DAG.getBitcast(MVT::v4i32, V1),
13911                      DAG.getTargetConstant(16, DL, MVT::i8));
13912     V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
13913                      DAG.getBitcast(MVT::v4i32, V2),
13914                      DAG.getTargetConstant(16, DL, MVT::i8));
13915     return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL,
13916                        MVT::v8i16, V1, V2);
13917   }
13918 
13919   // Try to lower by permuting the inputs into an unpack instruction.
13920   if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
13921                                                       Mask, Subtarget, DAG))
13922     return Unpack;
13923 
13924   // If we can't directly blend but can use PSHUFB, that will be better as it
13925   // can both shuffle and set up the inefficient blend.
13926   if (!IsBlendSupported && Subtarget.hasSSSE3()) {
13927     bool V1InUse, V2InUse;
13928     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
13929                                         Zeroable, DAG, V1InUse, V2InUse);
13930   }
13931 
13932   // We can always bit-blend if we have to so the fallback strategy is to
13933   // decompose into single-input permutes and blends/unpacks.
13934   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2,
13935                                               Mask, Subtarget, DAG);
13936 }
13937 
13938 /// Lower 8-lane 16-bit floating point shuffles.
13939 static SDValue lowerV8F16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13940                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13941                                  const X86Subtarget &Subtarget,
13942                                  SelectionDAG &DAG) {
13943   assert(V1.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
13944   assert(V2.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
13945   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
13946   int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
13947 
13948   if (Subtarget.hasFP16()) {
13949     if (NumV2Elements == 0) {
13950       // Check for being able to broadcast a single element.
13951       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f16, V1, V2,
13952                                                       Mask, Subtarget, DAG))
13953         return Broadcast;
13954     }
13955     if (NumV2Elements == 1 && Mask[0] >= 8)
13956       if (SDValue V = lowerShuffleAsElementInsertion(
13957               DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))
13958         return V;
13959   }
13960 
13961   V1 = DAG.getBitcast(MVT::v8i16, V1);
13962   V2 = DAG.getBitcast(MVT::v8i16, V2);
13963   return DAG.getBitcast(MVT::v8f16,
13964                         DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
13965 }
13966 
13967 // Lowers unary/binary shuffle as VPERMV/VPERMV3, for non-VLX targets,
13968 // sub-512-bit shuffles are padded to 512-bits for the shuffle and then
13969 // the active subvector is extracted.
13970 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
13971                                      ArrayRef<int> Mask, SDValue V1, SDValue V2,
13972                                      const X86Subtarget &Subtarget,
13973                                      SelectionDAG &DAG) {
13974   MVT MaskVT = VT.changeTypeToInteger();
13975   SDValue MaskNode;
13976   MVT ShuffleVT = VT;
13977   if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
13978     V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512);
13979     V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512);
13980     ShuffleVT = V1.getSimpleValueType();
13981 
13982     // Adjust mask to correct indices for the second input.
13983     int NumElts = VT.getVectorNumElements();
13984     unsigned Scale = 512 / VT.getSizeInBits();
13985     SmallVector<int, 32> AdjustedMask(Mask);
13986     for (int &M : AdjustedMask)
13987       if (NumElts <= M)
13988         M += (Scale - 1) * NumElts;
13989     MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true);
13990     MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512);
13991   } else {
13992     MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true);
13993   }
13994 
13995   SDValue Result;
13996   if (V2.isUndef())
13997     Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1);
13998   else
13999     Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2);
14000 
14001   if (VT != ShuffleVT)
14002     Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits());
14003 
14004   return Result;
14005 }
14006 
14007 /// Generic lowering of v16i8 shuffles.
14008 ///
14009 /// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
14010 /// detect any complexity reducing interleaving. If that doesn't help, it uses
14011 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
14012 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
14013 /// back together.
14014 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14015                                  const APInt &Zeroable, SDValue V1, SDValue V2,
14016                                  const X86Subtarget &Subtarget,
14017                                  SelectionDAG &DAG) {
14018   assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14019   assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14020   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
14021 
14022   // Try to use shift instructions.
14023   if (SDValue Shift =
14024           lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget,
14025                               DAG, /*BitwiseOnly*/ false))
14026     return Shift;
14027 
14028   // Try to use byte rotation instructions.
14029   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14030                                                 Subtarget, DAG))
14031     return Rotate;
14032 
14033   // Use dedicated pack instructions for masks that match their pattern.
14034   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14035                                        Subtarget))
14036     return V;
14037 
14038   // Try to use a zext lowering.
14039   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14040                                                    Zeroable, Subtarget, DAG))
14041     return ZExt;
14042 
14043   // Try to use lower using a truncation.
14044   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14045                                         Subtarget, DAG))
14046     return V;
14047 
14048   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14049                                        Subtarget, DAG))
14050     return V;
14051 
14052   // See if we can use SSE4A Extraction / Insertion.
14053   if (Subtarget.hasSSE4A())
14054     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14055                                           Zeroable, DAG))
14056       return V;
14057 
14058   int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
14059 
14060   // For single-input shuffles, there are some nicer lowering tricks we can use.
14061   if (NumV2Elements == 0) {
14062     // Check for being able to broadcast a single element.
14063     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14064                                                     Mask, Subtarget, DAG))
14065       return Broadcast;
14066 
14067     // Try to use bit rotation instructions.
14068     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14069                                                  Subtarget, DAG))
14070       return Rotate;
14071 
14072     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14073       return V;
14074 
14075     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
14076     // Notably, this handles splat and partial-splat shuffles more efficiently.
14077     // However, it only makes sense if the pre-duplication shuffle simplifies
14078     // things significantly. Currently, this means we need to be able to
14079     // express the pre-duplication shuffle as an i16 shuffle.
14080     //
14081     // FIXME: We should check for other patterns which can be widened into an
14082     // i16 shuffle as well.
14083     auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
14084       for (int i = 0; i < 16; i += 2)
14085         if (Mask[i] >= 0 && Mask[i + 1] >= 0 && Mask[i] != Mask[i + 1])
14086           return false;
14087 
14088       return true;
14089     };
14090     auto tryToWidenViaDuplication = [&]() -> SDValue {
14091       if (!canWidenViaDuplication(Mask))
14092         return SDValue();
14093       SmallVector<int, 4> LoInputs;
14094       copy_if(Mask, std::back_inserter(LoInputs),
14095               [](int M) { return M >= 0 && M < 8; });
14096       array_pod_sort(LoInputs.begin(), LoInputs.end());
14097       LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
14098                      LoInputs.end());
14099       SmallVector<int, 4> HiInputs;
14100       copy_if(Mask, std::back_inserter(HiInputs), [](int M) { return M >= 8; });
14101       array_pod_sort(HiInputs.begin(), HiInputs.end());
14102       HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
14103                      HiInputs.end());
14104 
14105       bool TargetLo = LoInputs.size() >= HiInputs.size();
14106       ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
14107       ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
14108 
14109       int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
14110       SmallDenseMap<int, int, 8> LaneMap;
14111       for (int I : InPlaceInputs) {
14112         PreDupI16Shuffle[I/2] = I/2;
14113         LaneMap[I] = I;
14114       }
14115       int j = TargetLo ? 0 : 4, je = j + 4;
14116       for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
14117         // Check if j is already a shuffle of this input. This happens when
14118         // there are two adjacent bytes after we move the low one.
14119         if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
14120           // If we haven't yet mapped the input, search for a slot into which
14121           // we can map it.
14122           while (j < je && PreDupI16Shuffle[j] >= 0)
14123             ++j;
14124 
14125           if (j == je)
14126             // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
14127             return SDValue();
14128 
14129           // Map this input with the i16 shuffle.
14130           PreDupI16Shuffle[j] = MovingInputs[i] / 2;
14131         }
14132 
14133         // Update the lane map based on the mapping we ended up with.
14134         LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
14135       }
14136       V1 = DAG.getBitcast(
14137           MVT::v16i8,
14138           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14139                                DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
14140 
14141       // Unpack the bytes to form the i16s that will be shuffled into place.
14142       bool EvenInUse = false, OddInUse = false;
14143       for (int i = 0; i < 16; i += 2) {
14144         EvenInUse |= (Mask[i + 0] >= 0);
14145         OddInUse |= (Mask[i + 1] >= 0);
14146         if (EvenInUse && OddInUse)
14147           break;
14148       }
14149       V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14150                        MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),
14151                        OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));
14152 
14153       int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
14154       for (int i = 0; i < 16; ++i)
14155         if (Mask[i] >= 0) {
14156           int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
14157           assert(MappedMask < 8 && "Invalid v8 shuffle mask!");
14158           if (PostDupI16Shuffle[i / 2] < 0)
14159             PostDupI16Shuffle[i / 2] = MappedMask;
14160           else
14161             assert(PostDupI16Shuffle[i / 2] == MappedMask &&
14162                    "Conflicting entries in the original shuffle!");
14163         }
14164       return DAG.getBitcast(
14165           MVT::v16i8,
14166           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14167                                DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
14168     };
14169     if (SDValue V = tryToWidenViaDuplication())
14170       return V;
14171   }
14172 
14173   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14174                                              Zeroable, Subtarget, DAG))
14175     return Masked;
14176 
14177   // Use dedicated unpack instructions for masks that match their pattern.
14178   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14179     return V;
14180 
14181   // Try to use byte shift instructions to mask.
14182   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
14183                                               Zeroable, Subtarget, DAG))
14184     return V;
14185 
14186   // Check for compaction patterns.
14187   bool IsSingleInput = V2.isUndef();
14188   int NumEvenDrops = canLowerByDroppingElements(Mask, true, IsSingleInput);
14189 
14190   // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
14191   // with PSHUFB. It is important to do this before we attempt to generate any
14192   // blends but after all of the single-input lowerings. If the single input
14193   // lowerings can find an instruction sequence that is faster than a PSHUFB, we
14194   // want to preserve that and we can DAG combine any longer sequences into
14195   // a PSHUFB in the end. But once we start blending from multiple inputs,
14196   // the complexity of DAG combining bad patterns back into PSHUFB is too high,
14197   // and there are *very* few patterns that would actually be faster than the
14198   // PSHUFB approach because of its ability to zero lanes.
14199   //
14200   // If the mask is a binary compaction, we can more efficiently perform this
14201   // as a PACKUS(AND(),AND()) - which is quicker than UNPACK(PSHUFB(),PSHUFB()).
14202   //
14203   // FIXME: The only exceptions to the above are blends which are exact
14204   // interleavings with direct instructions supporting them. We currently don't
14205   // handle those well here.
14206   if (Subtarget.hasSSSE3() && (IsSingleInput || NumEvenDrops != 1)) {
14207     bool V1InUse = false;
14208     bool V2InUse = false;
14209 
14210     SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14211         DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14212 
14213     // If both V1 and V2 are in use and we can use a direct blend or an unpack,
14214     // do so. This avoids using them to handle blends-with-zero which is
14215     // important as a single pshufb is significantly faster for that.
14216     if (V1InUse && V2InUse) {
14217       if (Subtarget.hasSSE41())
14218         if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14219                                                 Zeroable, Subtarget, DAG))
14220           return Blend;
14221 
14222       // We can use an unpack to do the blending rather than an or in some
14223       // cases. Even though the or may be (very minorly) more efficient, we
14224       // preference this lowering because there are common cases where part of
14225       // the complexity of the shuffles goes away when we do the final blend as
14226       // an unpack.
14227       // FIXME: It might be worth trying to detect if the unpack-feeding
14228       // shuffles will both be pshufb, in which case we shouldn't bother with
14229       // this.
14230       if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14231               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14232         return Unpack;
14233 
14234       // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
14235       if (Subtarget.hasVBMI())
14236         return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget,
14237                                      DAG);
14238 
14239       // If we have XOP we can use one VPPERM instead of multiple PSHUFBs.
14240       if (Subtarget.hasXOP()) {
14241         SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
14242         return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);
14243       }
14244 
14245       // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
14246       // PALIGNR will be cheaper than the second PSHUFB+OR.
14247       if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14248               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14249         return V;
14250     }
14251 
14252     return PSHUFB;
14253   }
14254 
14255   // There are special ways we can lower some single-element blends.
14256   if (NumV2Elements == 1)
14257     if (SDValue V = lowerShuffleAsElementInsertion(
14258             DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14259       return V;
14260 
14261   if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14262     return Blend;
14263 
14264   // Check whether a compaction lowering can be done. This handles shuffles
14265   // which take every Nth element for some even N. See the helper function for
14266   // details.
14267   //
14268   // We special case these as they can be particularly efficiently handled with
14269   // the PACKUSB instruction on x86 and they show up in common patterns of
14270   // rearranging bytes to truncate wide elements.
14271   if (NumEvenDrops) {
14272     // NumEvenDrops is the power of two stride of the elements. Another way of
14273     // thinking about it is that we need to drop the even elements this many
14274     // times to get the original input.
14275 
14276     // First we need to zero all the dropped bytes.
14277     assert(NumEvenDrops <= 3 &&
14278            "No support for dropping even elements more than 3 times.");
14279     SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
14280     for (unsigned i = 0; i != 8; i += 1 << (NumEvenDrops - 1))
14281       WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
14282     SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
14283     V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
14284                      WordClearMask);
14285     if (!IsSingleInput)
14286       V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
14287                        WordClearMask);
14288 
14289     // Now pack things back together.
14290     SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14291                                  IsSingleInput ? V1 : V2);
14292     for (int i = 1; i < NumEvenDrops; ++i) {
14293       Result = DAG.getBitcast(MVT::v8i16, Result);
14294       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
14295     }
14296     return Result;
14297   }
14298 
14299   int NumOddDrops = canLowerByDroppingElements(Mask, false, IsSingleInput);
14300   if (NumOddDrops == 1) {
14301     V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14302                      DAG.getBitcast(MVT::v8i16, V1),
14303                      DAG.getTargetConstant(8, DL, MVT::i8));
14304     if (!IsSingleInput)
14305       V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14306                        DAG.getBitcast(MVT::v8i16, V2),
14307                        DAG.getTargetConstant(8, DL, MVT::i8));
14308     return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14309                        IsSingleInput ? V1 : V2);
14310   }
14311 
14312   // Handle multi-input cases by blending/unpacking single-input shuffles.
14313   if (NumV2Elements > 0)
14314     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,
14315                                                 Subtarget, DAG);
14316 
14317   // The fallback path for single-input shuffles widens this into two v8i16
14318   // vectors with unpacks, shuffles those, and then pulls them back together
14319   // with a pack.
14320   SDValue V = V1;
14321 
14322   std::array<int, 8> LoBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14323   std::array<int, 8> HiBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14324   for (int i = 0; i < 16; ++i)
14325     if (Mask[i] >= 0)
14326       (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i];
14327 
14328   SDValue VLoHalf, VHiHalf;
14329   // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
14330   // them out and avoid using UNPCK{L,H} to extract the elements of V as
14331   // i16s.
14332   if (none_of(LoBlendMask, [](int M) { return M >= 0 && M % 2 == 1; }) &&
14333       none_of(HiBlendMask, [](int M) { return M >= 0 && M % 2 == 1; })) {
14334     // Use a mask to drop the high bytes.
14335     VLoHalf = DAG.getBitcast(MVT::v8i16, V);
14336     VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
14337                           DAG.getConstant(0x00FF, DL, MVT::v8i16));
14338 
14339     // This will be a single vector shuffle instead of a blend so nuke VHiHalf.
14340     VHiHalf = DAG.getUNDEF(MVT::v8i16);
14341 
14342     // Squash the masks to point directly into VLoHalf.
14343     for (int &M : LoBlendMask)
14344       if (M >= 0)
14345         M /= 2;
14346     for (int &M : HiBlendMask)
14347       if (M >= 0)
14348         M /= 2;
14349   } else {
14350     // Otherwise just unpack the low half of V into VLoHalf and the high half into
14351     // VHiHalf so that we can blend them as i16s.
14352     SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14353 
14354     VLoHalf = DAG.getBitcast(
14355         MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
14356     VHiHalf = DAG.getBitcast(
14357         MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
14358   }
14359 
14360   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14361   SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14362 
14363   return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
14364 }
14365 
14366 /// Dispatching routine to lower various 128-bit x86 vector shuffles.
14367 ///
14368 /// This routine breaks down the specific type of 128-bit shuffle and
14369 /// dispatches to the lowering routines accordingly.
14370 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14371                                   MVT VT, SDValue V1, SDValue V2,
14372                                   const APInt &Zeroable,
14373                                   const X86Subtarget &Subtarget,
14374                                   SelectionDAG &DAG) {
14375   if (VT == MVT::v8bf16) {
14376     V1 = DAG.getBitcast(MVT::v8i16, V1);
14377     V2 = DAG.getBitcast(MVT::v8i16, V2);
14378     return DAG.getBitcast(VT,
14379                           DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14380   }
14381 
14382   switch (VT.SimpleTy) {
14383   case MVT::v2i64:
14384     return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14385   case MVT::v2f64:
14386     return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14387   case MVT::v4i32:
14388     return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14389   case MVT::v4f32:
14390     return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14391   case MVT::v8i16:
14392     return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14393   case MVT::v8f16:
14394     return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14395   case MVT::v16i8:
14396     return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14397 
14398   default:
14399     llvm_unreachable("Unimplemented!");
14400   }
14401 }
14402 
14403 /// Generic routine to split vector shuffle into half-sized shuffles.
14404 ///
14405 /// This routine just extracts two subvectors, shuffles them independently, and
14406 /// then concatenates them back together. This should work effectively with all
14407 /// AVX vector shuffle types.
14408 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14409                                     SDValue V2, ArrayRef<int> Mask,
14410                                     SelectionDAG &DAG, bool SimpleOnly) {
14411   assert(VT.getSizeInBits() >= 256 &&
14412          "Only for 256-bit or wider vector shuffles!");
14413   assert(V1.getSimpleValueType() == VT && "Bad operand type!");
14414   assert(V2.getSimpleValueType() == VT && "Bad operand type!");
14415 
14416   ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
14417   ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
14418 
14419   int NumElements = VT.getVectorNumElements();
14420   int SplitNumElements = NumElements / 2;
14421   MVT ScalarVT = VT.getVectorElementType();
14422   MVT SplitVT = MVT::getVectorVT(ScalarVT, SplitNumElements);
14423 
14424   // Use splitVector/extractSubVector so that split build-vectors just build two
14425   // narrower build vectors. This helps shuffling with splats and zeros.
14426   auto SplitVector = [&](SDValue V) {
14427     SDValue LoV, HiV;
14428     std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL);
14429     return std::make_pair(DAG.getBitcast(SplitVT, LoV),
14430                           DAG.getBitcast(SplitVT, HiV));
14431   };
14432 
14433   SDValue LoV1, HiV1, LoV2, HiV2;
14434   std::tie(LoV1, HiV1) = SplitVector(V1);
14435   std::tie(LoV2, HiV2) = SplitVector(V2);
14436 
14437   // Now create two 4-way blends of these half-width vectors.
14438   auto GetHalfBlendPiecesReq = [&](const ArrayRef<int> &HalfMask, bool &UseLoV1,
14439                                    bool &UseHiV1, bool &UseLoV2,
14440                                    bool &UseHiV2) {
14441     UseLoV1 = UseHiV1 = UseLoV2 = UseHiV2 = false;
14442     for (int i = 0; i < SplitNumElements; ++i) {
14443       int M = HalfMask[i];
14444       if (M >= NumElements) {
14445         if (M >= NumElements + SplitNumElements)
14446           UseHiV2 = true;
14447         else
14448           UseLoV2 = true;
14449       } else if (M >= 0) {
14450         if (M >= SplitNumElements)
14451           UseHiV1 = true;
14452         else
14453           UseLoV1 = true;
14454       }
14455     }
14456   };
14457 
14458   auto CheckHalfBlendUsable = [&](const ArrayRef<int> &HalfMask) -> bool {
14459     if (!SimpleOnly)
14460       return true;
14461 
14462     bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14463     GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14464 
14465     return !(UseHiV1 || UseHiV2);
14466   };
14467 
14468   auto HalfBlend = [&](ArrayRef<int> HalfMask) {
14469     SmallVector<int, 32> V1BlendMask((unsigned)SplitNumElements, -1);
14470     SmallVector<int, 32> V2BlendMask((unsigned)SplitNumElements, -1);
14471     SmallVector<int, 32> BlendMask((unsigned)SplitNumElements, -1);
14472     for (int i = 0; i < SplitNumElements; ++i) {
14473       int M = HalfMask[i];
14474       if (M >= NumElements) {
14475         V2BlendMask[i] = M - NumElements;
14476         BlendMask[i] = SplitNumElements + i;
14477       } else if (M >= 0) {
14478         V1BlendMask[i] = M;
14479         BlendMask[i] = i;
14480       }
14481     }
14482 
14483     bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14484     GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14485 
14486     // Because the lowering happens after all combining takes place, we need to
14487     // manually combine these blend masks as much as possible so that we create
14488     // a minimal number of high-level vector shuffle nodes.
14489     assert((!SimpleOnly || (!UseHiV1 && !UseHiV2)) && "Shuffle isn't simple");
14490 
14491     // First try just blending the halves of V1 or V2.
14492     if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
14493       return DAG.getUNDEF(SplitVT);
14494     if (!UseLoV2 && !UseHiV2)
14495       return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14496     if (!UseLoV1 && !UseHiV1)
14497       return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14498 
14499     SDValue V1Blend, V2Blend;
14500     if (UseLoV1 && UseHiV1) {
14501       V1Blend = DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14502     } else {
14503       // We only use half of V1 so map the usage down into the final blend mask.
14504       V1Blend = UseLoV1 ? LoV1 : HiV1;
14505       for (int i = 0; i < SplitNumElements; ++i)
14506         if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
14507           BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
14508     }
14509     if (UseLoV2 && UseHiV2) {
14510       V2Blend = DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14511     } else {
14512       // We only use half of V2 so map the usage down into the final blend mask.
14513       V2Blend = UseLoV2 ? LoV2 : HiV2;
14514       for (int i = 0; i < SplitNumElements; ++i)
14515         if (BlendMask[i] >= SplitNumElements)
14516           BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
14517     }
14518     return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
14519   };
14520 
14521   if (!CheckHalfBlendUsable(LoMask) || !CheckHalfBlendUsable(HiMask))
14522     return SDValue();
14523 
14524   SDValue Lo = HalfBlend(LoMask);
14525   SDValue Hi = HalfBlend(HiMask);
14526   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
14527 }
14528 
14529 /// Either split a vector in halves or decompose the shuffles and the
14530 /// blend/unpack.
14531 ///
14532 /// This is provided as a good fallback for many lowerings of non-single-input
14533 /// shuffles with more than one 128-bit lane. In those cases, we want to select
14534 /// between splitting the shuffle into 128-bit components and stitching those
14535 /// back together vs. extracting the single-input shuffles and blending those
14536 /// results.
14537 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14538                                           SDValue V2, ArrayRef<int> Mask,
14539                                           const X86Subtarget &Subtarget,
14540                                           SelectionDAG &DAG) {
14541   assert(!V2.isUndef() && "This routine must not be used to lower single-input "
14542          "shuffles as it could then recurse on itself.");
14543   int Size = Mask.size();
14544 
14545   // If this can be modeled as a broadcast of two elements followed by a blend,
14546   // prefer that lowering. This is especially important because broadcasts can
14547   // often fold with memory operands.
14548   auto DoBothBroadcast = [&] {
14549     int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
14550     for (int M : Mask)
14551       if (M >= Size) {
14552         if (V2BroadcastIdx < 0)
14553           V2BroadcastIdx = M - Size;
14554         else if (M - Size != V2BroadcastIdx)
14555           return false;
14556       } else if (M >= 0) {
14557         if (V1BroadcastIdx < 0)
14558           V1BroadcastIdx = M;
14559         else if (M != V1BroadcastIdx)
14560           return false;
14561       }
14562     return true;
14563   };
14564   if (DoBothBroadcast())
14565     return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
14566                                                 DAG);
14567 
14568   // If the inputs all stem from a single 128-bit lane of each input, then we
14569   // split them rather than blending because the split will decompose to
14570   // unusually few instructions.
14571   int LaneCount = VT.getSizeInBits() / 128;
14572   int LaneSize = Size / LaneCount;
14573   SmallBitVector LaneInputs[2];
14574   LaneInputs[0].resize(LaneCount, false);
14575   LaneInputs[1].resize(LaneCount, false);
14576   for (int i = 0; i < Size; ++i)
14577     if (Mask[i] >= 0)
14578       LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
14579   if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
14580     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14581                                 /*SimpleOnly*/ false);
14582 
14583   // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This
14584   // requires that the decomposed single-input shuffles don't end up here.
14585   return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
14586                                               DAG);
14587 }
14588 
14589 // Lower as SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14590 // TODO: Extend to support v8f32 (+ 512-bit shuffles).
14591 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
14592                                                  SDValue V1, SDValue V2,
14593                                                  ArrayRef<int> Mask,
14594                                                  SelectionDAG &DAG) {
14595   assert(VT == MVT::v4f64 && "Only for v4f64 shuffles");
14596 
14597   int LHSMask[4] = {-1, -1, -1, -1};
14598   int RHSMask[4] = {-1, -1, -1, -1};
14599   unsigned SHUFPMask = 0;
14600 
14601   // As SHUFPD uses a single LHS/RHS element per lane, we can always
14602   // perform the shuffle once the lanes have been shuffled in place.
14603   for (int i = 0; i != 4; ++i) {
14604     int M = Mask[i];
14605     if (M < 0)
14606       continue;
14607     int LaneBase = i & ~1;
14608     auto &LaneMask = (i & 1) ? RHSMask : LHSMask;
14609     LaneMask[LaneBase + (M & 1)] = M;
14610     SHUFPMask |= (M & 1) << i;
14611   }
14612 
14613   SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
14614   SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
14615   return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS,
14616                      DAG.getTargetConstant(SHUFPMask, DL, MVT::i8));
14617 }
14618 
14619 /// Lower a vector shuffle crossing multiple 128-bit lanes as
14620 /// a lane permutation followed by a per-lane permutation.
14621 ///
14622 /// This is mainly for cases where we can have non-repeating permutes
14623 /// in each lane.
14624 ///
14625 /// TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask,
14626 /// we should investigate merging them.
14627 static SDValue lowerShuffleAsLanePermuteAndPermute(
14628     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14629     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14630   int NumElts = VT.getVectorNumElements();
14631   int NumLanes = VT.getSizeInBits() / 128;
14632   int NumEltsPerLane = NumElts / NumLanes;
14633   bool CanUseSublanes = Subtarget.hasAVX2() && V2.isUndef();
14634 
14635   /// Attempts to find a sublane permute with the given size
14636   /// that gets all elements into their target lanes.
14637   ///
14638   /// If successful, fills CrossLaneMask and InLaneMask and returns true.
14639   /// If unsuccessful, returns false and may overwrite InLaneMask.
14640   auto getSublanePermute = [&](int NumSublanes) -> SDValue {
14641     int NumSublanesPerLane = NumSublanes / NumLanes;
14642     int NumEltsPerSublane = NumElts / NumSublanes;
14643 
14644     SmallVector<int, 16> CrossLaneMask;
14645     SmallVector<int, 16> InLaneMask(NumElts, SM_SentinelUndef);
14646     // CrossLaneMask but one entry == one sublane.
14647     SmallVector<int, 16> CrossLaneMaskLarge(NumSublanes, SM_SentinelUndef);
14648 
14649     for (int i = 0; i != NumElts; ++i) {
14650       int M = Mask[i];
14651       if (M < 0)
14652         continue;
14653 
14654       int SrcSublane = M / NumEltsPerSublane;
14655       int DstLane = i / NumEltsPerLane;
14656 
14657       // We only need to get the elements into the right lane, not sublane.
14658       // So search all sublanes that make up the destination lane.
14659       bool Found = false;
14660       int DstSubStart = DstLane * NumSublanesPerLane;
14661       int DstSubEnd = DstSubStart + NumSublanesPerLane;
14662       for (int DstSublane = DstSubStart; DstSublane < DstSubEnd; ++DstSublane) {
14663         if (!isUndefOrEqual(CrossLaneMaskLarge[DstSublane], SrcSublane))
14664           continue;
14665 
14666         Found = true;
14667         CrossLaneMaskLarge[DstSublane] = SrcSublane;
14668         int DstSublaneOffset = DstSublane * NumEltsPerSublane;
14669         InLaneMask[i] = DstSublaneOffset + M % NumEltsPerSublane;
14670         break;
14671       }
14672       if (!Found)
14673         return SDValue();
14674     }
14675 
14676     // Fill CrossLaneMask using CrossLaneMaskLarge.
14677     narrowShuffleMaskElts(NumEltsPerSublane, CrossLaneMaskLarge, CrossLaneMask);
14678 
14679     if (!CanUseSublanes) {
14680       // If we're only shuffling a single lowest lane and the rest are identity
14681       // then don't bother.
14682       // TODO - isShuffleMaskInputInPlace could be extended to something like
14683       // this.
14684       int NumIdentityLanes = 0;
14685       bool OnlyShuffleLowestLane = true;
14686       for (int i = 0; i != NumLanes; ++i) {
14687         int LaneOffset = i * NumEltsPerLane;
14688         if (isSequentialOrUndefInRange(InLaneMask, LaneOffset, NumEltsPerLane,
14689                                        i * NumEltsPerLane))
14690           NumIdentityLanes++;
14691         else if (CrossLaneMask[LaneOffset] != 0)
14692           OnlyShuffleLowestLane = false;
14693       }
14694       if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1))
14695         return SDValue();
14696     }
14697 
14698     // Avoid returning the same shuffle operation. For example,
14699     // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5,
14700     //                             undef:v16i16
14701     if (CrossLaneMask == Mask || InLaneMask == Mask)
14702       return SDValue();
14703 
14704     SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
14705     return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT),
14706                                 InLaneMask);
14707   };
14708 
14709   // First attempt a solution with full lanes.
14710   if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes))
14711     return V;
14712 
14713   // The rest of the solutions use sublanes.
14714   if (!CanUseSublanes)
14715     return SDValue();
14716 
14717   // Then attempt a solution with 64-bit sublanes (vpermq).
14718   if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes * 2))
14719     return V;
14720 
14721   // If that doesn't work and we have fast variable cross-lane shuffle,
14722   // attempt 32-bit sublanes (vpermd).
14723   if (!Subtarget.hasFastVariableCrossLaneShuffle())
14724     return SDValue();
14725 
14726   return getSublanePermute(/*NumSublanes=*/NumLanes * 4);
14727 }
14728 
14729 /// Helper to get compute inlane shuffle mask for a complete shuffle mask.
14730 static void computeInLaneShuffleMask(const ArrayRef<int> &Mask, int LaneSize,
14731                                      SmallVector<int> &InLaneMask) {
14732   int Size = Mask.size();
14733   InLaneMask.assign(Mask.begin(), Mask.end());
14734   for (int i = 0; i < Size; ++i) {
14735     int &M = InLaneMask[i];
14736     if (M < 0)
14737       continue;
14738     if (((M % Size) / LaneSize) != (i / LaneSize))
14739       M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
14740   }
14741 }
14742 
14743 /// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one
14744 /// source with a lane permutation.
14745 ///
14746 /// This lowering strategy results in four instructions in the worst case for a
14747 /// single-input cross lane shuffle which is lower than any other fully general
14748 /// cross-lane shuffle strategy I'm aware of. Special cases for each particular
14749 /// shuffle pattern should be handled prior to trying this lowering.
14750 static SDValue lowerShuffleAsLanePermuteAndShuffle(
14751     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14752     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14753   // FIXME: This should probably be generalized for 512-bit vectors as well.
14754   assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
14755   int Size = Mask.size();
14756   int LaneSize = Size / 2;
14757 
14758   // Fold to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14759   // Only do this if the elements aren't all from the lower lane,
14760   // otherwise we're (probably) better off doing a split.
14761   if (VT == MVT::v4f64 &&
14762       !all_of(Mask, [LaneSize](int M) { return M < LaneSize; }))
14763     return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG);
14764 
14765   // If there are only inputs from one 128-bit lane, splitting will in fact be
14766   // less expensive. The flags track whether the given lane contains an element
14767   // that crosses to another lane.
14768   bool AllLanes;
14769   if (!Subtarget.hasAVX2()) {
14770     bool LaneCrossing[2] = {false, false};
14771     for (int i = 0; i < Size; ++i)
14772       if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
14773         LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
14774     AllLanes = LaneCrossing[0] && LaneCrossing[1];
14775   } else {
14776     bool LaneUsed[2] = {false, false};
14777     for (int i = 0; i < Size; ++i)
14778       if (Mask[i] >= 0)
14779         LaneUsed[(Mask[i] % Size) / LaneSize] = true;
14780     AllLanes = LaneUsed[0] && LaneUsed[1];
14781   }
14782 
14783   // TODO - we could support shuffling V2 in the Flipped input.
14784   assert(V2.isUndef() &&
14785          "This last part of this routine only works on single input shuffles");
14786 
14787   SmallVector<int> InLaneMask;
14788   computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
14789 
14790   assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&
14791          "In-lane shuffle mask expected");
14792 
14793   // If we're not using both lanes in each lane and the inlane mask is not
14794   // repeating, then we're better off splitting.
14795   if (!AllLanes && !is128BitLaneRepeatedShuffleMask(VT, InLaneMask))
14796     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14797                                 /*SimpleOnly*/ false);
14798 
14799   // Flip the lanes, and shuffle the results which should now be in-lane.
14800   MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
14801   SDValue Flipped = DAG.getBitcast(PVT, V1);
14802   Flipped =
14803       DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
14804   Flipped = DAG.getBitcast(VT, Flipped);
14805   return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
14806 }
14807 
14808 /// Handle lowering 2-lane 128-bit shuffles.
14809 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
14810                                   SDValue V2, ArrayRef<int> Mask,
14811                                   const APInt &Zeroable,
14812                                   const X86Subtarget &Subtarget,
14813                                   SelectionDAG &DAG) {
14814   if (V2.isUndef()) {
14815     // Attempt to match VBROADCAST*128 subvector broadcast load.
14816     bool SplatLo = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1);
14817     bool SplatHi = isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1);
14818     if ((SplatLo || SplatHi) && !Subtarget.hasAVX512() && V1.hasOneUse() &&
14819         X86::mayFoldLoad(peekThroughOneUseBitcasts(V1), Subtarget)) {
14820       MVT MemVT = VT.getHalfNumVectorElementsVT();
14821       unsigned Ofs = SplatLo ? 0 : MemVT.getStoreSize();
14822       auto *Ld = cast<LoadSDNode>(peekThroughOneUseBitcasts(V1));
14823       if (SDValue BcstLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, DL,
14824                                              VT, MemVT, Ld, Ofs, DAG))
14825         return BcstLd;
14826     }
14827 
14828     // With AVX2, use VPERMQ/VPERMPD for unary shuffles to allow memory folding.
14829     if (Subtarget.hasAVX2())
14830       return SDValue();
14831   }
14832 
14833   bool V2IsZero = !V2.isUndef() && ISD::isBuildVectorAllZeros(V2.getNode());
14834 
14835   SmallVector<int, 4> WidenedMask;
14836   if (!canWidenShuffleElements(Mask, Zeroable, V2IsZero, WidenedMask))
14837     return SDValue();
14838 
14839   bool IsLowZero = (Zeroable & 0x3) == 0x3;
14840   bool IsHighZero = (Zeroable & 0xc) == 0xc;
14841 
14842   // Try to use an insert into a zero vector.
14843   if (WidenedMask[0] == 0 && IsHighZero) {
14844     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
14845     SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
14846                               DAG.getIntPtrConstant(0, DL));
14847     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
14848                        getZeroVector(VT, Subtarget, DAG, DL), LoV,
14849                        DAG.getIntPtrConstant(0, DL));
14850   }
14851 
14852   // TODO: If minimizing size and one of the inputs is a zero vector and the
14853   // the zero vector has only one use, we could use a VPERM2X128 to save the
14854   // instruction bytes needed to explicitly generate the zero vector.
14855 
14856   // Blends are faster and handle all the non-lane-crossing cases.
14857   if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
14858                                           Subtarget, DAG))
14859     return Blend;
14860 
14861   // If either input operand is a zero vector, use VPERM2X128 because its mask
14862   // allows us to replace the zero input with an implicit zero.
14863   if (!IsLowZero && !IsHighZero) {
14864     // Check for patterns which can be matched with a single insert of a 128-bit
14865     // subvector.
14866     bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2);
14867     if (OnlyUsesV1 || isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2)) {
14868 
14869       // With AVX1, use vperm2f128 (below) to allow load folding. Otherwise,
14870       // this will likely become vinsertf128 which can't fold a 256-bit memop.
14871       if (!isa<LoadSDNode>(peekThroughBitcasts(V1))) {
14872         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
14873         SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
14874                                      OnlyUsesV1 ? V1 : V2,
14875                                      DAG.getIntPtrConstant(0, DL));
14876         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
14877                            DAG.getIntPtrConstant(2, DL));
14878       }
14879     }
14880 
14881     // Try to use SHUF128 if possible.
14882     if (Subtarget.hasVLX()) {
14883       if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
14884         unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
14885                             ((WidenedMask[1] % 2) << 1);
14886         return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
14887                            DAG.getTargetConstant(PermMask, DL, MVT::i8));
14888       }
14889     }
14890   }
14891 
14892   // Otherwise form a 128-bit permutation. After accounting for undefs,
14893   // convert the 64-bit shuffle mask selection values into 128-bit
14894   // selection bits by dividing the indexes by 2 and shifting into positions
14895   // defined by a vperm2*128 instruction's immediate control byte.
14896 
14897   // The immediate permute control byte looks like this:
14898   //    [1:0] - select 128 bits from sources for low half of destination
14899   //    [2]   - ignore
14900   //    [3]   - zero low half of destination
14901   //    [5:4] - select 128 bits from sources for high half of destination
14902   //    [6]   - ignore
14903   //    [7]   - zero high half of destination
14904 
14905   assert((WidenedMask[0] >= 0 || IsLowZero) &&
14906          (WidenedMask[1] >= 0 || IsHighZero) && "Undef half?");
14907 
14908   unsigned PermMask = 0;
14909   PermMask |= IsLowZero  ? 0x08 : (WidenedMask[0] << 0);
14910   PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
14911 
14912   // Check the immediate mask and replace unused sources with undef.
14913   if ((PermMask & 0x0a) != 0x00 && (PermMask & 0xa0) != 0x00)
14914     V1 = DAG.getUNDEF(VT);
14915   if ((PermMask & 0x0a) != 0x02 && (PermMask & 0xa0) != 0x20)
14916     V2 = DAG.getUNDEF(VT);
14917 
14918   return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
14919                      DAG.getTargetConstant(PermMask, DL, MVT::i8));
14920 }
14921 
14922 /// Lower a vector shuffle by first fixing the 128-bit lanes and then
14923 /// shuffling each lane.
14924 ///
14925 /// This attempts to create a repeated lane shuffle where each lane uses one
14926 /// or two of the lanes of the inputs. The lanes of the input vectors are
14927 /// shuffled in one or two independent shuffles to get the lanes into the
14928 /// position needed by the final shuffle.
14929 static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
14930     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14931     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
14932   assert(!V2.isUndef() && "This is only useful with multiple inputs.");
14933 
14934   if (is128BitLaneRepeatedShuffleMask(VT, Mask))
14935     return SDValue();
14936 
14937   int NumElts = Mask.size();
14938   int NumLanes = VT.getSizeInBits() / 128;
14939   int NumLaneElts = 128 / VT.getScalarSizeInBits();
14940   SmallVector<int, 16> RepeatMask(NumLaneElts, -1);
14941   SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}});
14942 
14943   // First pass will try to fill in the RepeatMask from lanes that need two
14944   // sources.
14945   for (int Lane = 0; Lane != NumLanes; ++Lane) {
14946     int Srcs[2] = {-1, -1};
14947     SmallVector<int, 16> InLaneMask(NumLaneElts, -1);
14948     for (int i = 0; i != NumLaneElts; ++i) {
14949       int M = Mask[(Lane * NumLaneElts) + i];
14950       if (M < 0)
14951         continue;
14952       // Determine which of the possible input lanes (NumLanes from each source)
14953       // this element comes from. Assign that as one of the sources for this
14954       // lane. We can assign up to 2 sources for this lane. If we run out
14955       // sources we can't do anything.
14956       int LaneSrc = M / NumLaneElts;
14957       int Src;
14958       if (Srcs[0] < 0 || Srcs[0] == LaneSrc)
14959         Src = 0;
14960       else if (Srcs[1] < 0 || Srcs[1] == LaneSrc)
14961         Src = 1;
14962       else
14963         return SDValue();
14964 
14965       Srcs[Src] = LaneSrc;
14966       InLaneMask[i] = (M % NumLaneElts) + Src * NumElts;
14967     }
14968 
14969     // If this lane has two sources, see if it fits with the repeat mask so far.
14970     if (Srcs[1] < 0)
14971       continue;
14972 
14973     LaneSrcs[Lane][0] = Srcs[0];
14974     LaneSrcs[Lane][1] = Srcs[1];
14975 
14976     auto MatchMasks = [](ArrayRef<int> M1, ArrayRef<int> M2) {
14977       assert(M1.size() == M2.size() && "Unexpected mask size");
14978       for (int i = 0, e = M1.size(); i != e; ++i)
14979         if (M1[i] >= 0 && M2[i] >= 0 && M1[i] != M2[i])
14980           return false;
14981       return true;
14982     };
14983 
14984     auto MergeMasks = [](ArrayRef<int> Mask, MutableArrayRef<int> MergedMask) {
14985       assert(Mask.size() == MergedMask.size() && "Unexpected mask size");
14986       for (int i = 0, e = MergedMask.size(); i != e; ++i) {
14987         int M = Mask[i];
14988         if (M < 0)
14989           continue;
14990         assert((MergedMask[i] < 0 || MergedMask[i] == M) &&
14991                "Unexpected mask element");
14992         MergedMask[i] = M;
14993       }
14994     };
14995 
14996     if (MatchMasks(InLaneMask, RepeatMask)) {
14997       // Merge this lane mask into the final repeat mask.
14998       MergeMasks(InLaneMask, RepeatMask);
14999       continue;
15000     }
15001 
15002     // Didn't find a match. Swap the operands and try again.
15003     std::swap(LaneSrcs[Lane][0], LaneSrcs[Lane][1]);
15004     ShuffleVectorSDNode::commuteMask(InLaneMask);
15005 
15006     if (MatchMasks(InLaneMask, RepeatMask)) {
15007       // Merge this lane mask into the final repeat mask.
15008       MergeMasks(InLaneMask, RepeatMask);
15009       continue;
15010     }
15011 
15012     // Couldn't find a match with the operands in either order.
15013     return SDValue();
15014   }
15015 
15016   // Now handle any lanes with only one source.
15017   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15018     // If this lane has already been processed, skip it.
15019     if (LaneSrcs[Lane][0] >= 0)
15020       continue;
15021 
15022     for (int i = 0; i != NumLaneElts; ++i) {
15023       int M = Mask[(Lane * NumLaneElts) + i];
15024       if (M < 0)
15025         continue;
15026 
15027       // If RepeatMask isn't defined yet we can define it ourself.
15028       if (RepeatMask[i] < 0)
15029         RepeatMask[i] = M % NumLaneElts;
15030 
15031       if (RepeatMask[i] < NumElts) {
15032         if (RepeatMask[i] != M % NumLaneElts)
15033           return SDValue();
15034         LaneSrcs[Lane][0] = M / NumLaneElts;
15035       } else {
15036         if (RepeatMask[i] != ((M % NumLaneElts) + NumElts))
15037           return SDValue();
15038         LaneSrcs[Lane][1] = M / NumLaneElts;
15039       }
15040     }
15041 
15042     if (LaneSrcs[Lane][0] < 0 && LaneSrcs[Lane][1] < 0)
15043       return SDValue();
15044   }
15045 
15046   SmallVector<int, 16> NewMask(NumElts, -1);
15047   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15048     int Src = LaneSrcs[Lane][0];
15049     for (int i = 0; i != NumLaneElts; ++i) {
15050       int M = -1;
15051       if (Src >= 0)
15052         M = Src * NumLaneElts + i;
15053       NewMask[Lane * NumLaneElts + i] = M;
15054     }
15055   }
15056   SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15057   // Ensure we didn't get back the shuffle we started with.
15058   // FIXME: This is a hack to make up for some splat handling code in
15059   // getVectorShuffle.
15060   if (isa<ShuffleVectorSDNode>(NewV1) &&
15061       cast<ShuffleVectorSDNode>(NewV1)->getMask() == Mask)
15062     return SDValue();
15063 
15064   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15065     int Src = LaneSrcs[Lane][1];
15066     for (int i = 0; i != NumLaneElts; ++i) {
15067       int M = -1;
15068       if (Src >= 0)
15069         M = Src * NumLaneElts + i;
15070       NewMask[Lane * NumLaneElts + i] = M;
15071     }
15072   }
15073   SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15074   // Ensure we didn't get back the shuffle we started with.
15075   // FIXME: This is a hack to make up for some splat handling code in
15076   // getVectorShuffle.
15077   if (isa<ShuffleVectorSDNode>(NewV2) &&
15078       cast<ShuffleVectorSDNode>(NewV2)->getMask() == Mask)
15079     return SDValue();
15080 
15081   for (int i = 0; i != NumElts; ++i) {
15082     if (Mask[i] < 0) {
15083       NewMask[i] = -1;
15084       continue;
15085     }
15086     NewMask[i] = RepeatMask[i % NumLaneElts];
15087     if (NewMask[i] < 0)
15088       continue;
15089 
15090     NewMask[i] += (i / NumLaneElts) * NumLaneElts;
15091   }
15092   return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15093 }
15094 
15095 /// If the input shuffle mask results in a vector that is undefined in all upper
15096 /// or lower half elements and that mask accesses only 2 halves of the
15097 /// shuffle's operands, return true. A mask of half the width with mask indexes
15098 /// adjusted to access the extracted halves of the original shuffle operands is
15099 /// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
15100 /// lower half of each input operand is accessed.
15101 static bool
15102 getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
15103                    int &HalfIdx1, int &HalfIdx2) {
15104   assert((Mask.size() == HalfMask.size() * 2) &&
15105          "Expected input mask to be twice as long as output");
15106 
15107   // Exactly one half of the result must be undef to allow narrowing.
15108   bool UndefLower = isUndefLowerHalf(Mask);
15109   bool UndefUpper = isUndefUpperHalf(Mask);
15110   if (UndefLower == UndefUpper)
15111     return false;
15112 
15113   unsigned HalfNumElts = HalfMask.size();
15114   unsigned MaskIndexOffset = UndefLower ? HalfNumElts : 0;
15115   HalfIdx1 = -1;
15116   HalfIdx2 = -1;
15117   for (unsigned i = 0; i != HalfNumElts; ++i) {
15118     int M = Mask[i + MaskIndexOffset];
15119     if (M < 0) {
15120       HalfMask[i] = M;
15121       continue;
15122     }
15123 
15124     // Determine which of the 4 half vectors this element is from.
15125     // i.e. 0 = Lower V1, 1 = Upper V1, 2 = Lower V2, 3 = Upper V2.
15126     int HalfIdx = M / HalfNumElts;
15127 
15128     // Determine the element index into its half vector source.
15129     int HalfElt = M % HalfNumElts;
15130 
15131     // We can shuffle with up to 2 half vectors, set the new 'half'
15132     // shuffle mask accordingly.
15133     if (HalfIdx1 < 0 || HalfIdx1 == HalfIdx) {
15134       HalfMask[i] = HalfElt;
15135       HalfIdx1 = HalfIdx;
15136       continue;
15137     }
15138     if (HalfIdx2 < 0 || HalfIdx2 == HalfIdx) {
15139       HalfMask[i] = HalfElt + HalfNumElts;
15140       HalfIdx2 = HalfIdx;
15141       continue;
15142     }
15143 
15144     // Too many half vectors referenced.
15145     return false;
15146   }
15147 
15148   return true;
15149 }
15150 
15151 /// Given the output values from getHalfShuffleMask(), create a half width
15152 /// shuffle of extracted vectors followed by an insert back to full width.
15153 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15154                                      ArrayRef<int> HalfMask, int HalfIdx1,
15155                                      int HalfIdx2, bool UndefLower,
15156                                      SelectionDAG &DAG, bool UseConcat = false) {
15157   assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
15158   assert(V1.getValueType().isSimple() && "Expecting only simple types");
15159 
15160   MVT VT = V1.getSimpleValueType();
15161   MVT HalfVT = VT.getHalfNumVectorElementsVT();
15162   unsigned HalfNumElts = HalfVT.getVectorNumElements();
15163 
15164   auto getHalfVector = [&](int HalfIdx) {
15165     if (HalfIdx < 0)
15166       return DAG.getUNDEF(HalfVT);
15167     SDValue V = (HalfIdx < 2 ? V1 : V2);
15168     HalfIdx = (HalfIdx % 2) * HalfNumElts;
15169     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15170                        DAG.getIntPtrConstant(HalfIdx, DL));
15171   };
15172 
15173   // ins undef, (shuf (ext V1, HalfIdx1), (ext V2, HalfIdx2), HalfMask), Offset
15174   SDValue Half1 = getHalfVector(HalfIdx1);
15175   SDValue Half2 = getHalfVector(HalfIdx2);
15176   SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15177   if (UseConcat) {
15178     SDValue Op0 = V;
15179     SDValue Op1 = DAG.getUNDEF(HalfVT);
15180     if (UndefLower)
15181       std::swap(Op0, Op1);
15182     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15183   }
15184 
15185   unsigned Offset = UndefLower ? HalfNumElts : 0;
15186   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15187                      DAG.getIntPtrConstant(Offset, DL));
15188 }
15189 
15190 /// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
15191 /// This allows for fast cases such as subvector extraction/insertion
15192 /// or shuffling smaller vector types which can lower more efficiently.
15193 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15194                                          SDValue V2, ArrayRef<int> Mask,
15195                                          const X86Subtarget &Subtarget,
15196                                          SelectionDAG &DAG) {
15197   assert((VT.is256BitVector() || VT.is512BitVector()) &&
15198          "Expected 256-bit or 512-bit vector");
15199 
15200   bool UndefLower = isUndefLowerHalf(Mask);
15201   if (!UndefLower && !isUndefUpperHalf(Mask))
15202     return SDValue();
15203 
15204   assert((!UndefLower || !isUndefUpperHalf(Mask)) &&
15205          "Completely undef shuffle mask should have been simplified already");
15206 
15207   // Upper half is undef and lower half is whole upper subvector.
15208   // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
15209   MVT HalfVT = VT.getHalfNumVectorElementsVT();
15210   unsigned HalfNumElts = HalfVT.getVectorNumElements();
15211   if (!UndefLower &&
15212       isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
15213     SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15214                              DAG.getIntPtrConstant(HalfNumElts, DL));
15215     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15216                        DAG.getIntPtrConstant(0, DL));
15217   }
15218 
15219   // Lower half is undef and upper half is whole lower subvector.
15220   // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
15221   if (UndefLower &&
15222       isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
15223     SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15224                              DAG.getIntPtrConstant(0, DL));
15225     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15226                        DAG.getIntPtrConstant(HalfNumElts, DL));
15227   }
15228 
15229   int HalfIdx1, HalfIdx2;
15230   SmallVector<int, 8> HalfMask(HalfNumElts);
15231   if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2))
15232     return SDValue();
15233 
15234   assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length");
15235 
15236   // Only shuffle the halves of the inputs when useful.
15237   unsigned NumLowerHalves =
15238       (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
15239   unsigned NumUpperHalves =
15240       (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
15241   assert(NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed");
15242 
15243   // Determine the larger pattern of undef/halves, then decide if it's worth
15244   // splitting the shuffle based on subtarget capabilities and types.
15245   unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
15246   if (!UndefLower) {
15247     // XXXXuuuu: no insert is needed.
15248     // Always extract lowers when setting lower - these are all free subreg ops.
15249     if (NumUpperHalves == 0)
15250       return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15251                                    UndefLower, DAG);
15252 
15253     if (NumUpperHalves == 1) {
15254       // AVX2 has efficient 32/64-bit element cross-lane shuffles.
15255       if (Subtarget.hasAVX2()) {
15256         // extract128 + vunpckhps/vshufps, is better than vblend + vpermps.
15257         if (EltWidth == 32 && NumLowerHalves && HalfVT.is128BitVector() &&
15258             !is128BitUnpackShuffleMask(HalfMask, DAG) &&
15259             (!isSingleSHUFPSMask(HalfMask) ||
15260              Subtarget.hasFastVariableCrossLaneShuffle()))
15261           return SDValue();
15262         // If this is a unary shuffle (assume that the 2nd operand is
15263         // canonicalized to undef), then we can use vpermpd. Otherwise, we
15264         // are better off extracting the upper half of 1 operand and using a
15265         // narrow shuffle.
15266         if (EltWidth == 64 && V2.isUndef())
15267           return SDValue();
15268       }
15269       // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15270       if (Subtarget.hasAVX512() && VT.is512BitVector())
15271         return SDValue();
15272       // Extract + narrow shuffle is better than the wide alternative.
15273       return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15274                                    UndefLower, DAG);
15275     }
15276 
15277     // Don't extract both uppers, instead shuffle and then extract.
15278     assert(NumUpperHalves == 2 && "Half vector count went wrong");
15279     return SDValue();
15280   }
15281 
15282   // UndefLower - uuuuXXXX: an insert to high half is required if we split this.
15283   if (NumUpperHalves == 0) {
15284     // AVX2 has efficient 64-bit element cross-lane shuffles.
15285     // TODO: Refine to account for unary shuffle, splat, and other masks?
15286     if (Subtarget.hasAVX2() && EltWidth == 64)
15287       return SDValue();
15288     // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15289     if (Subtarget.hasAVX512() && VT.is512BitVector())
15290       return SDValue();
15291     // Narrow shuffle + insert is better than the wide alternative.
15292     return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15293                                  UndefLower, DAG);
15294   }
15295 
15296   // NumUpperHalves != 0: don't bother with extract, shuffle, and then insert.
15297   return SDValue();
15298 }
15299 
15300 /// Handle case where shuffle sources are coming from the same 128-bit lane and
15301 /// every lane can be represented as the same repeating mask - allowing us to
15302 /// shuffle the sources with the repeating shuffle and then permute the result
15303 /// to the destination lanes.
15304 static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15305     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15306     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15307   int NumElts = VT.getVectorNumElements();
15308   int NumLanes = VT.getSizeInBits() / 128;
15309   int NumLaneElts = NumElts / NumLanes;
15310 
15311   // On AVX2 we may be able to just shuffle the lowest elements and then
15312   // broadcast the result.
15313   if (Subtarget.hasAVX2()) {
15314     for (unsigned BroadcastSize : {16, 32, 64}) {
15315       if (BroadcastSize <= VT.getScalarSizeInBits())
15316         continue;
15317       int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits();
15318 
15319       // Attempt to match a repeating pattern every NumBroadcastElts,
15320       // accounting for UNDEFs but only references the lowest 128-bit
15321       // lane of the inputs.
15322       auto FindRepeatingBroadcastMask = [&](SmallVectorImpl<int> &RepeatMask) {
15323         for (int i = 0; i != NumElts; i += NumBroadcastElts)
15324           for (int j = 0; j != NumBroadcastElts; ++j) {
15325             int M = Mask[i + j];
15326             if (M < 0)
15327               continue;
15328             int &R = RepeatMask[j];
15329             if (0 != ((M % NumElts) / NumLaneElts))
15330               return false;
15331             if (0 <= R && R != M)
15332               return false;
15333             R = M;
15334           }
15335         return true;
15336       };
15337 
15338       SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1);
15339       if (!FindRepeatingBroadcastMask(RepeatMask))
15340         continue;
15341 
15342       // Shuffle the (lowest) repeated elements in place for broadcast.
15343       SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15344 
15345       // Shuffle the actual broadcast.
15346       SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1);
15347       for (int i = 0; i != NumElts; i += NumBroadcastElts)
15348         for (int j = 0; j != NumBroadcastElts; ++j)
15349           BroadcastMask[i + j] = j;
15350 
15351       // Avoid returning the same shuffle operation. For example,
15352       // v8i32 = vector_shuffle<0,1,0,1,0,1,0,1> t5, undef:v8i32
15353       if (BroadcastMask == Mask)
15354         return SDValue();
15355 
15356       return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
15357                                   BroadcastMask);
15358     }
15359   }
15360 
15361   // Bail if the shuffle mask doesn't cross 128-bit lanes.
15362   if (!is128BitLaneCrossingShuffleMask(VT, Mask))
15363     return SDValue();
15364 
15365   // Bail if we already have a repeated lane shuffle mask.
15366   if (is128BitLaneRepeatedShuffleMask(VT, Mask))
15367     return SDValue();
15368 
15369   // Helper to look for repeated mask in each split sublane, and that those
15370   // sublanes can then be permuted into place.
15371   auto ShuffleSubLanes = [&](int SubLaneScale) {
15372     int NumSubLanes = NumLanes * SubLaneScale;
15373     int NumSubLaneElts = NumLaneElts / SubLaneScale;
15374 
15375     // Check that all the sources are coming from the same lane and see if we
15376     // can form a repeating shuffle mask (local to each sub-lane). At the same
15377     // time, determine the source sub-lane for each destination sub-lane.
15378     int TopSrcSubLane = -1;
15379     SmallVector<int, 8> Dst2SrcSubLanes((unsigned)NumSubLanes, -1);
15380     SmallVector<SmallVector<int, 8>> RepeatedSubLaneMasks(
15381         SubLaneScale,
15382         SmallVector<int, 8>((unsigned)NumSubLaneElts, SM_SentinelUndef));
15383 
15384     for (int DstSubLane = 0; DstSubLane != NumSubLanes; ++DstSubLane) {
15385       // Extract the sub-lane mask, check that it all comes from the same lane
15386       // and normalize the mask entries to come from the first lane.
15387       int SrcLane = -1;
15388       SmallVector<int, 8> SubLaneMask((unsigned)NumSubLaneElts, -1);
15389       for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15390         int M = Mask[(DstSubLane * NumSubLaneElts) + Elt];
15391         if (M < 0)
15392           continue;
15393         int Lane = (M % NumElts) / NumLaneElts;
15394         if ((0 <= SrcLane) && (SrcLane != Lane))
15395           return SDValue();
15396         SrcLane = Lane;
15397         int LocalM = (M % NumLaneElts) + (M < NumElts ? 0 : NumElts);
15398         SubLaneMask[Elt] = LocalM;
15399       }
15400 
15401       // Whole sub-lane is UNDEF.
15402       if (SrcLane < 0)
15403         continue;
15404 
15405       // Attempt to match against the candidate repeated sub-lane masks.
15406       for (int SubLane = 0; SubLane != SubLaneScale; ++SubLane) {
15407         auto MatchMasks = [NumSubLaneElts](ArrayRef<int> M1, ArrayRef<int> M2) {
15408           for (int i = 0; i != NumSubLaneElts; ++i) {
15409             if (M1[i] < 0 || M2[i] < 0)
15410               continue;
15411             if (M1[i] != M2[i])
15412               return false;
15413           }
15414           return true;
15415         };
15416 
15417         auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane];
15418         if (!MatchMasks(SubLaneMask, RepeatedSubLaneMask))
15419           continue;
15420 
15421         // Merge the sub-lane mask into the matching repeated sub-lane mask.
15422         for (int i = 0; i != NumSubLaneElts; ++i) {
15423           int M = SubLaneMask[i];
15424           if (M < 0)
15425             continue;
15426           assert((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) &&
15427                  "Unexpected mask element");
15428           RepeatedSubLaneMask[i] = M;
15429         }
15430 
15431         // Track the top most source sub-lane - by setting the remaining to
15432         // UNDEF we can greatly simplify shuffle matching.
15433         int SrcSubLane = (SrcLane * SubLaneScale) + SubLane;
15434         TopSrcSubLane = std::max(TopSrcSubLane, SrcSubLane);
15435         Dst2SrcSubLanes[DstSubLane] = SrcSubLane;
15436         break;
15437       }
15438 
15439       // Bail if we failed to find a matching repeated sub-lane mask.
15440       if (Dst2SrcSubLanes[DstSubLane] < 0)
15441         return SDValue();
15442     }
15443     assert(0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes &&
15444            "Unexpected source lane");
15445 
15446     // Create a repeating shuffle mask for the entire vector.
15447     SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1);
15448     for (int SubLane = 0; SubLane <= TopSrcSubLane; ++SubLane) {
15449       int Lane = SubLane / SubLaneScale;
15450       auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane % SubLaneScale];
15451       for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15452         int M = RepeatedSubLaneMask[Elt];
15453         if (M < 0)
15454           continue;
15455         int Idx = (SubLane * NumSubLaneElts) + Elt;
15456         RepeatedMask[Idx] = M + (Lane * NumLaneElts);
15457       }
15458     }
15459 
15460     // Shuffle each source sub-lane to its destination.
15461     SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1);
15462     for (int i = 0; i != NumElts; i += NumSubLaneElts) {
15463       int SrcSubLane = Dst2SrcSubLanes[i / NumSubLaneElts];
15464       if (SrcSubLane < 0)
15465         continue;
15466       for (int j = 0; j != NumSubLaneElts; ++j)
15467         SubLaneMask[i + j] = j + (SrcSubLane * NumSubLaneElts);
15468     }
15469 
15470     // Avoid returning the same shuffle operation.
15471     // v8i32 = vector_shuffle<0,1,4,5,2,3,6,7> t5, undef:v8i32
15472     if (RepeatedMask == Mask || SubLaneMask == Mask)
15473       return SDValue();
15474 
15475     SDValue RepeatedShuffle =
15476         DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15477 
15478     return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
15479                                 SubLaneMask);
15480   };
15481 
15482   // On AVX2 targets we can permute 256-bit vectors as 64-bit sub-lanes
15483   // (with PERMQ/PERMPD). On AVX2/AVX512BW targets, permuting 32-bit sub-lanes,
15484   // even with a variable shuffle, can be worth it for v32i8/v64i8 vectors.
15485   // Otherwise we can only permute whole 128-bit lanes.
15486   int MinSubLaneScale = 1, MaxSubLaneScale = 1;
15487   if (Subtarget.hasAVX2() && VT.is256BitVector()) {
15488     bool OnlyLowestElts = isUndefOrInRange(Mask, 0, NumLaneElts);
15489     MinSubLaneScale = 2;
15490     MaxSubLaneScale =
15491         (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2;
15492   }
15493   if (Subtarget.hasBWI() && VT == MVT::v64i8)
15494     MinSubLaneScale = MaxSubLaneScale = 4;
15495 
15496   for (int Scale = MinSubLaneScale; Scale <= MaxSubLaneScale; Scale *= 2)
15497     if (SDValue Shuffle = ShuffleSubLanes(Scale))
15498       return Shuffle;
15499 
15500   return SDValue();
15501 }
15502 
15503 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15504                                    bool &ForceV1Zero, bool &ForceV2Zero,
15505                                    unsigned &ShuffleImm, ArrayRef<int> Mask,
15506                                    const APInt &Zeroable) {
15507   int NumElts = VT.getVectorNumElements();
15508   assert(VT.getScalarSizeInBits() == 64 &&
15509          (NumElts == 2 || NumElts == 4 || NumElts == 8) &&
15510          "Unexpected data type for VSHUFPD");
15511   assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
15512          "Illegal shuffle mask");
15513 
15514   bool ZeroLane[2] = { true, true };
15515   for (int i = 0; i < NumElts; ++i)
15516     ZeroLane[i & 1] &= Zeroable[i];
15517 
15518   // Mask for V8F64: 0/1,  8/9,  2/3,  10/11, 4/5, ..
15519   // Mask for V4F64; 0/1,  4/5,  2/3,  6/7..
15520   ShuffleImm = 0;
15521   bool ShufpdMask = true;
15522   bool CommutableMask = true;
15523   for (int i = 0; i < NumElts; ++i) {
15524     if (Mask[i] == SM_SentinelUndef || ZeroLane[i & 1])
15525       continue;
15526     if (Mask[i] < 0)
15527       return false;
15528     int Val = (i & 6) + NumElts * (i & 1);
15529     int CommutVal = (i & 0xe) + NumElts * ((i & 1) ^ 1);
15530     if (Mask[i] < Val || Mask[i] > Val + 1)
15531       ShufpdMask = false;
15532     if (Mask[i] < CommutVal || Mask[i] > CommutVal + 1)
15533       CommutableMask = false;
15534     ShuffleImm |= (Mask[i] % 2) << i;
15535   }
15536 
15537   if (!ShufpdMask && !CommutableMask)
15538     return false;
15539 
15540   if (!ShufpdMask && CommutableMask)
15541     std::swap(V1, V2);
15542 
15543   ForceV1Zero = ZeroLane[0];
15544   ForceV2Zero = ZeroLane[1];
15545   return true;
15546 }
15547 
15548 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15549                                       SDValue V2, ArrayRef<int> Mask,
15550                                       const APInt &Zeroable,
15551                                       const X86Subtarget &Subtarget,
15552                                       SelectionDAG &DAG) {
15553   assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
15554          "Unexpected data type for VSHUFPD");
15555 
15556   unsigned Immediate = 0;
15557   bool ForceV1Zero = false, ForceV2Zero = false;
15558   if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate,
15559                               Mask, Zeroable))
15560     return SDValue();
15561 
15562   // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
15563   if (ForceV1Zero)
15564     V1 = getZeroVector(VT, Subtarget, DAG, DL);
15565   if (ForceV2Zero)
15566     V2 = getZeroVector(VT, Subtarget, DAG, DL);
15567 
15568   return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
15569                      DAG.getTargetConstant(Immediate, DL, MVT::i8));
15570 }
15571 
15572 // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
15573 // by zeroable elements in the remaining 24 elements. Turn this into two
15574 // vmovqb instructions shuffled together.
15575 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
15576                                              SDValue V1, SDValue V2,
15577                                              ArrayRef<int> Mask,
15578                                              const APInt &Zeroable,
15579                                              SelectionDAG &DAG) {
15580   assert(VT == MVT::v32i8 && "Unexpected type!");
15581 
15582   // The first 8 indices should be every 8th element.
15583   if (!isSequentialOrUndefInRange(Mask, 0, 8, 0, 8))
15584     return SDValue();
15585 
15586   // Remaining elements need to be zeroable.
15587   if (Zeroable.countl_one() < (Mask.size() - 8))
15588     return SDValue();
15589 
15590   V1 = DAG.getBitcast(MVT::v4i64, V1);
15591   V2 = DAG.getBitcast(MVT::v4i64, V2);
15592 
15593   V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
15594   V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
15595 
15596   // The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
15597   // the upper bits of the result using an unpckldq.
15598   SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
15599                                         { 0, 1, 2, 3, 16, 17, 18, 19,
15600                                           4, 5, 6, 7, 20, 21, 22, 23 });
15601   // Insert the unpckldq into a zero vector to widen to v32i8.
15602   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
15603                      DAG.getConstant(0, DL, MVT::v32i8), Unpack,
15604                      DAG.getIntPtrConstant(0, DL));
15605 }
15606 
15607 // a = shuffle v1, v2, mask1    ; interleaving lower lanes of v1 and v2
15608 // b = shuffle v1, v2, mask2    ; interleaving higher lanes of v1 and v2
15609 //     =>
15610 // ul = unpckl v1, v2
15611 // uh = unpckh v1, v2
15612 // a = vperm ul, uh
15613 // b = vperm ul, uh
15614 //
15615 // Pattern-match interleave(256b v1, 256b v2) -> 512b v3 and lower it into unpck
15616 // and permute. We cannot directly match v3 because it is split into two
15617 // 256-bit vectors in earlier isel stages. Therefore, this function matches a
15618 // pair of 256-bit shuffles and makes sure the masks are consecutive.
15619 //
15620 // Once unpck and permute nodes are created, the permute corresponding to this
15621 // shuffle is returned, while the other permute replaces the other half of the
15622 // shuffle in the selection dag.
15623 static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
15624                                                  SDValue V1, SDValue V2,
15625                                                  ArrayRef<int> Mask,
15626                                                  SelectionDAG &DAG) {
15627   if (VT != MVT::v8f32 && VT != MVT::v8i32 && VT != MVT::v16i16 &&
15628       VT != MVT::v32i8)
15629     return SDValue();
15630   // <B0, B1, B0+1, B1+1, ..., >
15631   auto IsInterleavingPattern = [&](ArrayRef<int> Mask, unsigned Begin0,
15632                                    unsigned Begin1) {
15633     size_t Size = Mask.size();
15634     assert(Size % 2 == 0 && "Expected even mask size");
15635     for (unsigned I = 0; I < Size; I += 2) {
15636       if (Mask[I] != (int)(Begin0 + I / 2) ||
15637           Mask[I + 1] != (int)(Begin1 + I / 2))
15638         return false;
15639     }
15640     return true;
15641   };
15642   // Check which half is this shuffle node
15643   int NumElts = VT.getVectorNumElements();
15644   size_t FirstQtr = NumElts / 2;
15645   size_t ThirdQtr = NumElts + NumElts / 2;
15646   bool IsFirstHalf = IsInterleavingPattern(Mask, 0, NumElts);
15647   bool IsSecondHalf = IsInterleavingPattern(Mask, FirstQtr, ThirdQtr);
15648   if (!IsFirstHalf && !IsSecondHalf)
15649     return SDValue();
15650 
15651   // Find the intersection between shuffle users of V1 and V2.
15652   SmallVector<SDNode *, 2> Shuffles;
15653   for (SDNode *User : V1->uses())
15654     if (User->getOpcode() == ISD::VECTOR_SHUFFLE && User->getOperand(0) == V1 &&
15655         User->getOperand(1) == V2)
15656       Shuffles.push_back(User);
15657   // Limit user size to two for now.
15658   if (Shuffles.size() != 2)
15659     return SDValue();
15660   // Find out which half of the 512-bit shuffles is each smaller shuffle
15661   auto *SVN1 = cast<ShuffleVectorSDNode>(Shuffles[0]);
15662   auto *SVN2 = cast<ShuffleVectorSDNode>(Shuffles[1]);
15663   SDNode *FirstHalf;
15664   SDNode *SecondHalf;
15665   if (IsInterleavingPattern(SVN1->getMask(), 0, NumElts) &&
15666       IsInterleavingPattern(SVN2->getMask(), FirstQtr, ThirdQtr)) {
15667     FirstHalf = Shuffles[0];
15668     SecondHalf = Shuffles[1];
15669   } else if (IsInterleavingPattern(SVN1->getMask(), FirstQtr, ThirdQtr) &&
15670              IsInterleavingPattern(SVN2->getMask(), 0, NumElts)) {
15671     FirstHalf = Shuffles[1];
15672     SecondHalf = Shuffles[0];
15673   } else {
15674     return SDValue();
15675   }
15676   // Lower into unpck and perm. Return the perm of this shuffle and replace
15677   // the other.
15678   SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
15679   SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
15680   SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15681                               DAG.getTargetConstant(0x20, DL, MVT::i8));
15682   SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15683                               DAG.getTargetConstant(0x31, DL, MVT::i8));
15684   if (IsFirstHalf) {
15685     DAG.ReplaceAllUsesWith(SecondHalf, &Perm2);
15686     return Perm1;
15687   }
15688   DAG.ReplaceAllUsesWith(FirstHalf, &Perm1);
15689   return Perm2;
15690 }
15691 
15692 /// Handle lowering of 4-lane 64-bit floating point shuffles.
15693 ///
15694 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
15695 /// isn't available.
15696 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15697                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15698                                  const X86Subtarget &Subtarget,
15699                                  SelectionDAG &DAG) {
15700   assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15701   assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15702   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15703 
15704   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15705                                      Subtarget, DAG))
15706     return V;
15707 
15708   if (V2.isUndef()) {
15709     // Check for being able to broadcast a single element.
15710     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
15711                                                     Mask, Subtarget, DAG))
15712       return Broadcast;
15713 
15714     // Use low duplicate instructions for masks that match their pattern.
15715     if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
15716       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
15717 
15718     if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
15719       // Non-half-crossing single input shuffles can be lowered with an
15720       // interleaved permutation.
15721       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
15722                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
15723       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
15724                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
15725     }
15726 
15727     // With AVX2 we have direct support for this permutation.
15728     if (Subtarget.hasAVX2())
15729       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
15730                          getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15731 
15732     // Try to create an in-lane repeating shuffle mask and then shuffle the
15733     // results into the target lanes.
15734     if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15735             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15736       return V;
15737 
15738     // Try to permute the lanes and then use a per-lane permute.
15739     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
15740                                                         Mask, DAG, Subtarget))
15741       return V;
15742 
15743     // Otherwise, fall back.
15744     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,
15745                                                DAG, Subtarget);
15746   }
15747 
15748   // Use dedicated unpack instructions for masks that match their pattern.
15749   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
15750     return V;
15751 
15752   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
15753                                           Zeroable, Subtarget, DAG))
15754     return Blend;
15755 
15756   // Check if the blend happens to exactly fit that of SHUFPD.
15757   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
15758                                           Zeroable, Subtarget, DAG))
15759     return Op;
15760 
15761   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
15762   bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
15763 
15764   // If we have lane crossing shuffles AND they don't all come from the lower
15765   // lane elements, lower to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
15766   // TODO: Handle BUILD_VECTOR sources which getVectorShuffle currently
15767   // canonicalize to a blend of splat which isn't necessary for this combine.
15768   if (is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask) &&
15769       !all_of(Mask, [](int M) { return M < 2 || (4 <= M && M < 6); }) &&
15770       (V1.getOpcode() != ISD::BUILD_VECTOR) &&
15771       (V2.getOpcode() != ISD::BUILD_VECTOR))
15772     return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);
15773 
15774   // If we have one input in place, then we can permute the other input and
15775   // blend the result.
15776   if (V1IsInPlace || V2IsInPlace)
15777     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15778                                                 Subtarget, DAG);
15779 
15780   // Try to create an in-lane repeating shuffle mask and then shuffle the
15781   // results into the target lanes.
15782   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15783           DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15784     return V;
15785 
15786   // Try to simplify this by merging 128-bit lanes to enable a lane-based
15787   // shuffle. However, if we have AVX2 and either inputs are already in place,
15788   // we will be able to shuffle even across lanes the other input in a single
15789   // instruction so skip this pattern.
15790   if (!(Subtarget.hasAVX2() && (V1IsInPlace || V2IsInPlace)))
15791     if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
15792             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15793       return V;
15794 
15795   // If we have VLX support, we can use VEXPAND.
15796   if (Subtarget.hasVLX())
15797     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
15798                                          DAG, Subtarget))
15799       return V;
15800 
15801   // If we have AVX2 then we always want to lower with a blend because an v4 we
15802   // can fully permute the elements.
15803   if (Subtarget.hasAVX2())
15804     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15805                                                 Subtarget, DAG);
15806 
15807   // Otherwise fall back on generic lowering.
15808   return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
15809                                     Subtarget, DAG);
15810 }
15811 
15812 /// Handle lowering of 4-lane 64-bit integer shuffles.
15813 ///
15814 /// This routine is only called when we have AVX2 and thus a reasonable
15815 /// instruction set for v4i64 shuffling..
15816 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15817                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15818                                  const X86Subtarget &Subtarget,
15819                                  SelectionDAG &DAG) {
15820   assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15821   assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15822   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15823   assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!");
15824 
15825   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15826                                      Subtarget, DAG))
15827     return V;
15828 
15829   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
15830                                           Zeroable, Subtarget, DAG))
15831     return Blend;
15832 
15833   // Check for being able to broadcast a single element.
15834   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
15835                                                   Subtarget, DAG))
15836     return Broadcast;
15837 
15838   // Try to use shift instructions if fast.
15839   if (Subtarget.preferLowerShuffleAsShift())
15840     if (SDValue Shift =
15841             lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15842                                 Subtarget, DAG, /*BitwiseOnly*/ true))
15843       return Shift;
15844 
15845   if (V2.isUndef()) {
15846     // When the shuffle is mirrored between the 128-bit lanes of the unit, we
15847     // can use lower latency instructions that will operate on both lanes.
15848     SmallVector<int, 2> RepeatedMask;
15849     if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
15850       SmallVector<int, 4> PSHUFDMask;
15851       narrowShuffleMaskElts(2, RepeatedMask, PSHUFDMask);
15852       return DAG.getBitcast(
15853           MVT::v4i64,
15854           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
15855                       DAG.getBitcast(MVT::v8i32, V1),
15856                       getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
15857     }
15858 
15859     // AVX2 provides a direct instruction for permuting a single input across
15860     // lanes.
15861     return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
15862                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15863   }
15864 
15865   // Try to use shift instructions.
15866   if (SDValue Shift =
15867           lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable, Subtarget,
15868                               DAG, /*BitwiseOnly*/ false))
15869     return Shift;
15870 
15871   // If we have VLX support, we can use VALIGN or VEXPAND.
15872   if (Subtarget.hasVLX()) {
15873     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
15874                                               Zeroable, Subtarget, DAG))
15875       return Rotate;
15876 
15877     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
15878                                          DAG, Subtarget))
15879       return V;
15880   }
15881 
15882   // Try to use PALIGNR.
15883   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
15884                                                 Subtarget, DAG))
15885     return Rotate;
15886 
15887   // Use dedicated unpack instructions for masks that match their pattern.
15888   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
15889     return V;
15890 
15891   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
15892   bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
15893 
15894   // If we have one input in place, then we can permute the other input and
15895   // blend the result.
15896   if (V1IsInPlace || V2IsInPlace)
15897     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
15898                                                 Subtarget, DAG);
15899 
15900   // Try to create an in-lane repeating shuffle mask and then shuffle the
15901   // results into the target lanes.
15902   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15903           DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15904     return V;
15905 
15906   // Try to lower to PERMQ(BLENDD(V1,V2)).
15907   if (SDValue V =
15908           lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG))
15909     return V;
15910 
15911   // Try to simplify this by merging 128-bit lanes to enable a lane-based
15912   // shuffle. However, if we have AVX2 and either inputs are already in place,
15913   // we will be able to shuffle even across lanes the other input in a single
15914   // instruction so skip this pattern.
15915   if (!V1IsInPlace && !V2IsInPlace)
15916     if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15917             DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15918       return Result;
15919 
15920   // Otherwise fall back on generic blend lowering.
15921   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
15922                                               Subtarget, DAG);
15923 }
15924 
15925 /// Handle lowering of 8-lane 32-bit floating point shuffles.
15926 ///
15927 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
15928 /// isn't available.
15929 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15930                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15931                                  const X86Subtarget &Subtarget,
15932                                  SelectionDAG &DAG) {
15933   assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
15934   assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
15935   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
15936 
15937   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
15938                                           Zeroable, Subtarget, DAG))
15939     return Blend;
15940 
15941   // Check for being able to broadcast a single element.
15942   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
15943                                                   Subtarget, DAG))
15944     return Broadcast;
15945 
15946   if (!Subtarget.hasAVX2()) {
15947     SmallVector<int> InLaneMask;
15948     computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
15949 
15950     if (!is128BitLaneRepeatedShuffleMask(MVT::v8f32, InLaneMask))
15951       if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG,
15952                                            /*SimpleOnly*/ true))
15953         return R;
15954   }
15955   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
15956                                                    Zeroable, Subtarget, DAG))
15957     return DAG.getBitcast(MVT::v8f32, ZExt);
15958 
15959   // If the shuffle mask is repeated in each 128-bit lane, we have many more
15960   // options to efficiently lower the shuffle.
15961   SmallVector<int, 4> RepeatedMask;
15962   if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
15963     assert(RepeatedMask.size() == 4 &&
15964            "Repeated masks must be half the mask width!");
15965 
15966     // Use even/odd duplicate instructions for masks that match their pattern.
15967     if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
15968       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
15969     if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
15970       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
15971 
15972     if (V2.isUndef())
15973       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
15974                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
15975 
15976     // Use dedicated unpack instructions for masks that match their pattern.
15977     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
15978       return V;
15979 
15980     // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
15981     // have already handled any direct blends.
15982     return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
15983   }
15984 
15985   // Try to create an in-lane repeating shuffle mask and then shuffle the
15986   // results into the target lanes.
15987   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15988           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
15989     return V;
15990 
15991   // If we have a single input shuffle with different shuffle patterns in the
15992   // two 128-bit lanes use the variable mask to VPERMILPS.
15993   if (V2.isUndef()) {
15994     if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) {
15995       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
15996       return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
15997     }
15998     if (Subtarget.hasAVX2()) {
15999       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16000       return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16001     }
16002     // Otherwise, fall back.
16003     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,
16004                                                DAG, Subtarget);
16005   }
16006 
16007   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16008   // shuffle.
16009   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16010           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16011     return Result;
16012 
16013   // If we have VLX support, we can use VEXPAND.
16014   if (Subtarget.hasVLX())
16015     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16016                                          DAG, Subtarget))
16017       return V;
16018 
16019   // Try to match an interleave of two v8f32s and lower them as unpck and
16020   // permutes using ymms. This needs to go before we try to split the vectors.
16021   //
16022   // TODO: Expand this to AVX1. Currently v8i32 is casted to v8f32 and hits
16023   // this path inadvertently.
16024   if (Subtarget.hasAVX2() && !Subtarget.hasAVX512())
16025     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8f32, V1, V2,
16026                                                       Mask, DAG))
16027       return V;
16028 
16029   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16030   // since after split we get a more efficient code using vpunpcklwd and
16031   // vpunpckhwd instrs than vblend.
16032   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
16033     return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget,
16034                                       DAG);
16035 
16036   // If we have AVX2 then we always want to lower with a blend because at v8 we
16037   // can fully permute the elements.
16038   if (Subtarget.hasAVX2())
16039     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,
16040                                                 Subtarget, DAG);
16041 
16042   // Otherwise fall back on generic lowering.
16043   return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16044                                     Subtarget, DAG);
16045 }
16046 
16047 /// Handle lowering of 8-lane 32-bit integer shuffles.
16048 ///
16049 /// This routine is only called when we have AVX2 and thus a reasonable
16050 /// instruction set for v8i32 shuffling..
16051 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16052                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16053                                  const X86Subtarget &Subtarget,
16054                                  SelectionDAG &DAG) {
16055   assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16056   assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16057   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16058   assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!");
16059 
16060   int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
16061 
16062   // Whenever we can lower this as a zext, that instruction is strictly faster
16063   // than any alternative. It also allows us to fold memory operands into the
16064   // shuffle in many cases.
16065   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16066                                                    Zeroable, Subtarget, DAG))
16067     return ZExt;
16068 
16069   // Try to match an interleave of two v8i32s and lower them as unpck and
16070   // permutes using ymms. This needs to go before we try to split the vectors.
16071   if (!Subtarget.hasAVX512())
16072     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8i32, V1, V2,
16073                                                       Mask, DAG))
16074       return V;
16075 
16076   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16077   // since after split we get a more efficient code than vblend by using
16078   // vpunpcklwd and vpunpckhwd instrs.
16079   if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
16080       !Subtarget.hasAVX512())
16081     return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget,
16082                                       DAG);
16083 
16084   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16085                                           Zeroable, Subtarget, DAG))
16086     return Blend;
16087 
16088   // Check for being able to broadcast a single element.
16089   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16090                                                   Subtarget, DAG))
16091     return Broadcast;
16092 
16093   // Try to use shift instructions if fast.
16094   if (Subtarget.preferLowerShuffleAsShift()) {
16095     if (SDValue Shift =
16096             lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16097                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16098       return Shift;
16099     if (NumV2Elements == 0)
16100       if (SDValue Rotate =
16101               lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16102         return Rotate;
16103   }
16104 
16105   // If the shuffle mask is repeated in each 128-bit lane we can use more
16106   // efficient instructions that mirror the shuffles across the two 128-bit
16107   // lanes.
16108   SmallVector<int, 4> RepeatedMask;
16109   bool Is128BitLaneRepeatedShuffle =
16110       is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask);
16111   if (Is128BitLaneRepeatedShuffle) {
16112     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16113     if (V2.isUndef())
16114       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16115                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16116 
16117     // Use dedicated unpack instructions for masks that match their pattern.
16118     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16119       return V;
16120   }
16121 
16122   // Try to use shift instructions.
16123   if (SDValue Shift =
16124           lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget,
16125                               DAG, /*BitwiseOnly*/ false))
16126     return Shift;
16127 
16128   if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements == 0)
16129     if (SDValue Rotate =
16130             lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16131       return Rotate;
16132 
16133   // If we have VLX support, we can use VALIGN or EXPAND.
16134   if (Subtarget.hasVLX()) {
16135     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16136                                               Zeroable, Subtarget, DAG))
16137       return Rotate;
16138 
16139     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16140                                          DAG, Subtarget))
16141       return V;
16142   }
16143 
16144   // Try to use byte rotation instructions.
16145   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16146                                                 Subtarget, DAG))
16147     return Rotate;
16148 
16149   // Try to create an in-lane repeating shuffle mask and then shuffle the
16150   // results into the target lanes.
16151   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16152           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16153     return V;
16154 
16155   if (V2.isUndef()) {
16156     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16157     // because that should be faster than the variable permute alternatives.
16158     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
16159       return V;
16160 
16161     // If the shuffle patterns aren't repeated but it's a single input, directly
16162     // generate a cross-lane VPERMD instruction.
16163     SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16164     return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16165   }
16166 
16167   // Assume that a single SHUFPS is faster than an alternative sequence of
16168   // multiple instructions (even if the CPU has a domain penalty).
16169   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16170   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16171     SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16172     SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16173     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16174                                             CastV1, CastV2, DAG);
16175     return DAG.getBitcast(MVT::v8i32, ShufPS);
16176   }
16177 
16178   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16179   // shuffle.
16180   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16181           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16182     return Result;
16183 
16184   // Otherwise fall back on generic blend lowering.
16185   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,
16186                                               Subtarget, DAG);
16187 }
16188 
16189 /// Handle lowering of 16-lane 16-bit integer shuffles.
16190 ///
16191 /// This routine is only called when we have AVX2 and thus a reasonable
16192 /// instruction set for v16i16 shuffling..
16193 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16194                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16195                                   const X86Subtarget &Subtarget,
16196                                   SelectionDAG &DAG) {
16197   assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16198   assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16199   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16200   assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!");
16201 
16202   // Whenever we can lower this as a zext, that instruction is strictly faster
16203   // than any alternative. It also allows us to fold memory operands into the
16204   // shuffle in many cases.
16205   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16206           DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16207     return ZExt;
16208 
16209   // Check for being able to broadcast a single element.
16210   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16211                                                   Subtarget, DAG))
16212     return Broadcast;
16213 
16214   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16215                                           Zeroable, Subtarget, DAG))
16216     return Blend;
16217 
16218   // Use dedicated unpack instructions for masks that match their pattern.
16219   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16220     return V;
16221 
16222   // Use dedicated pack instructions for masks that match their pattern.
16223   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16224                                        Subtarget))
16225     return V;
16226 
16227   // Try to use lower using a truncation.
16228   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16229                                        Subtarget, DAG))
16230     return V;
16231 
16232   // Try to use shift instructions.
16233   if (SDValue Shift =
16234           lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16235                               Subtarget, DAG, /*BitwiseOnly*/ false))
16236     return Shift;
16237 
16238   // Try to use byte rotation instructions.
16239   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16240                                                 Subtarget, DAG))
16241     return Rotate;
16242 
16243   // Try to create an in-lane repeating shuffle mask and then shuffle the
16244   // results into the target lanes.
16245   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16246           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16247     return V;
16248 
16249   if (V2.isUndef()) {
16250     // Try to use bit rotation instructions.
16251     if (SDValue Rotate =
16252             lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))
16253       return Rotate;
16254 
16255     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16256     // because that should be faster than the variable permute alternatives.
16257     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
16258       return V;
16259 
16260     // There are no generalized cross-lane shuffle operations available on i16
16261     // element types.
16262     if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {
16263       if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16264               DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16265         return V;
16266 
16267       return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,
16268                                                  DAG, Subtarget);
16269     }
16270 
16271     SmallVector<int, 8> RepeatedMask;
16272     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
16273       // As this is a single-input shuffle, the repeated mask should be
16274       // a strictly valid v8i16 mask that we can pass through to the v8i16
16275       // lowering to handle even the v16 case.
16276       return lowerV8I16GeneralSingleInputShuffle(
16277           DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16278     }
16279   }
16280 
16281   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16282                                               Zeroable, Subtarget, DAG))
16283     return PSHUFB;
16284 
16285   // AVX512BW can lower to VPERMW (non-VLX will pad to v32i16).
16286   if (Subtarget.hasBWI())
16287     return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG);
16288 
16289   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16290   // shuffle.
16291   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16292           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16293     return Result;
16294 
16295   // Try to permute the lanes and then use a per-lane permute.
16296   if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16297           DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16298     return V;
16299 
16300   // Try to match an interleave of two v16i16s and lower them as unpck and
16301   // permutes using ymms.
16302   if (!Subtarget.hasAVX512())
16303     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v16i16, V1, V2,
16304                                                       Mask, DAG))
16305       return V;
16306 
16307   // Otherwise fall back on generic lowering.
16308   return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
16309                                     Subtarget, DAG);
16310 }
16311 
16312 /// Handle lowering of 32-lane 8-bit integer shuffles.
16313 ///
16314 /// This routine is only called when we have AVX2 and thus a reasonable
16315 /// instruction set for v32i8 shuffling..
16316 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16317                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16318                                  const X86Subtarget &Subtarget,
16319                                  SelectionDAG &DAG) {
16320   assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16321   assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16322   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
16323   assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!");
16324 
16325   // Whenever we can lower this as a zext, that instruction is strictly faster
16326   // than any alternative. It also allows us to fold memory operands into the
16327   // shuffle in many cases.
16328   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16329                                                    Zeroable, Subtarget, DAG))
16330     return ZExt;
16331 
16332   // Check for being able to broadcast a single element.
16333   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16334                                                   Subtarget, DAG))
16335     return Broadcast;
16336 
16337   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16338                                           Zeroable, Subtarget, DAG))
16339     return Blend;
16340 
16341   // Use dedicated unpack instructions for masks that match their pattern.
16342   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16343     return V;
16344 
16345   // Use dedicated pack instructions for masks that match their pattern.
16346   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16347                                        Subtarget))
16348     return V;
16349 
16350   // Try to use lower using a truncation.
16351   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16352                                        Subtarget, DAG))
16353     return V;
16354 
16355   // Try to use shift instructions.
16356   if (SDValue Shift =
16357           lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget,
16358                               DAG, /*BitwiseOnly*/ false))
16359     return Shift;
16360 
16361   // Try to use byte rotation instructions.
16362   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16363                                                 Subtarget, DAG))
16364     return Rotate;
16365 
16366   // Try to use bit rotation instructions.
16367   if (V2.isUndef())
16368     if (SDValue Rotate =
16369             lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))
16370       return Rotate;
16371 
16372   // Try to create an in-lane repeating shuffle mask and then shuffle the
16373   // results into the target lanes.
16374   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16375           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16376     return V;
16377 
16378   // There are no generalized cross-lane shuffle operations available on i8
16379   // element types.
16380   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
16381     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16382     // because that should be faster than the variable permute alternatives.
16383     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
16384       return V;
16385 
16386     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16387             DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16388       return V;
16389 
16390     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
16391                                                DAG, Subtarget);
16392   }
16393 
16394   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16395                                               Zeroable, Subtarget, DAG))
16396     return PSHUFB;
16397 
16398   // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
16399   if (Subtarget.hasVBMI())
16400     return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG);
16401 
16402   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16403   // shuffle.
16404   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16405           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16406     return Result;
16407 
16408   // Try to permute the lanes and then use a per-lane permute.
16409   if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16410           DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16411     return V;
16412 
16413   // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
16414   // by zeroable elements in the remaining 24 elements. Turn this into two
16415   // vmovqb instructions shuffled together.
16416   if (Subtarget.hasVLX())
16417     if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16418                                                   Mask, Zeroable, DAG))
16419       return V;
16420 
16421   // Try to match an interleave of two v32i8s and lower them as unpck and
16422   // permutes using ymms.
16423   if (!Subtarget.hasAVX512())
16424     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v32i8, V1, V2,
16425                                                       Mask, DAG))
16426       return V;
16427 
16428   // Otherwise fall back on generic lowering.
16429   return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
16430                                     Subtarget, DAG);
16431 }
16432 
16433 /// High-level routine to lower various 256-bit x86 vector shuffles.
16434 ///
16435 /// This routine either breaks down the specific type of a 256-bit x86 vector
16436 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
16437 /// together based on the available instructions.
16438 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16439                                   SDValue V1, SDValue V2, const APInt &Zeroable,
16440                                   const X86Subtarget &Subtarget,
16441                                   SelectionDAG &DAG) {
16442   // If we have a single input to the zero element, insert that into V1 if we
16443   // can do so cheaply.
16444   int NumElts = VT.getVectorNumElements();
16445   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
16446 
16447   if (NumV2Elements == 1 && Mask[0] >= NumElts)
16448     if (SDValue Insertion = lowerShuffleAsElementInsertion(
16449             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16450       return Insertion;
16451 
16452   // Handle special cases where the lower or upper half is UNDEF.
16453   if (SDValue V =
16454           lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16455     return V;
16456 
16457   // There is a really nice hard cut-over between AVX1 and AVX2 that means we
16458   // can check for those subtargets here and avoid much of the subtarget
16459   // querying in the per-vector-type lowering routines. With AVX1 we have
16460   // essentially *zero* ability to manipulate a 256-bit vector with integer
16461   // types. Since we'll use floating point types there eventually, just
16462   // immediately cast everything to a float and operate entirely in that domain.
16463   if (VT.isInteger() && !Subtarget.hasAVX2()) {
16464     int ElementBits = VT.getScalarSizeInBits();
16465     if (ElementBits < 32) {
16466       // No floating point type available, if we can't use the bit operations
16467       // for masking/blending then decompose into 128-bit vectors.
16468       if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16469                                             Subtarget, DAG))
16470         return V;
16471       if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16472         return V;
16473       return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
16474     }
16475 
16476     MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
16477                                 VT.getVectorNumElements());
16478     V1 = DAG.getBitcast(FpVT, V1);
16479     V2 = DAG.getBitcast(FpVT, V2);
16480     return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16481   }
16482 
16483   if (VT == MVT::v16f16 || VT == MVT::v16bf16) {
16484     V1 = DAG.getBitcast(MVT::v16i16, V1);
16485     V2 = DAG.getBitcast(MVT::v16i16, V2);
16486     return DAG.getBitcast(VT,
16487                           DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask));
16488   }
16489 
16490   switch (VT.SimpleTy) {
16491   case MVT::v4f64:
16492     return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16493   case MVT::v4i64:
16494     return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16495   case MVT::v8f32:
16496     return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16497   case MVT::v8i32:
16498     return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16499   case MVT::v16i16:
16500     return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16501   case MVT::v32i8:
16502     return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16503 
16504   default:
16505     llvm_unreachable("Not a valid 256-bit x86 vector type!");
16506   }
16507 }
16508 
16509 /// Try to lower a vector shuffle as a 128-bit shuffles.
16510 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16511                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16512                                   const X86Subtarget &Subtarget,
16513                                   SelectionDAG &DAG) {
16514   assert(VT.getScalarSizeInBits() == 64 &&
16515          "Unexpected element type size for 128bit shuffle.");
16516 
16517   // To handle 256 bit vector requires VLX and most probably
16518   // function lowerV2X128VectorShuffle() is better solution.
16519   assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.");
16520 
16521   // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
16522   SmallVector<int, 4> Widened128Mask;
16523   if (!canWidenShuffleElements(Mask, Widened128Mask))
16524     return SDValue();
16525   assert(Widened128Mask.size() == 4 && "Shuffle widening mismatch");
16526 
16527   // Try to use an insert into a zero vector.
16528   if (Widened128Mask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
16529       (Widened128Mask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
16530     unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
16531     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
16532     SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16533                               DAG.getIntPtrConstant(0, DL));
16534     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
16535                        getZeroVector(VT, Subtarget, DAG, DL), LoV,
16536                        DAG.getIntPtrConstant(0, DL));
16537   }
16538 
16539   // Check for patterns which can be matched with a single insert of a 256-bit
16540   // subvector.
16541   bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}, V1, V2);
16542   if (OnlyUsesV1 ||
16543       isShuffleEquivalent(Mask, {0, 1, 2, 3, 8, 9, 10, 11}, V1, V2)) {
16544     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);
16545     SDValue SubVec =
16546         DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
16547                     DAG.getIntPtrConstant(0, DL));
16548     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
16549                        DAG.getIntPtrConstant(4, DL));
16550   }
16551 
16552   // See if this is an insertion of the lower 128-bits of V2 into V1.
16553   bool IsInsert = true;
16554   int V2Index = -1;
16555   for (int i = 0; i < 4; ++i) {
16556     assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16557     if (Widened128Mask[i] < 0)
16558       continue;
16559 
16560     // Make sure all V1 subvectors are in place.
16561     if (Widened128Mask[i] < 4) {
16562       if (Widened128Mask[i] != i) {
16563         IsInsert = false;
16564         break;
16565       }
16566     } else {
16567       // Make sure we only have a single V2 index and its the lowest 128-bits.
16568       if (V2Index >= 0 || Widened128Mask[i] != 4) {
16569         IsInsert = false;
16570         break;
16571       }
16572       V2Index = i;
16573     }
16574   }
16575   if (IsInsert && V2Index >= 0) {
16576     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
16577     SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
16578                                  DAG.getIntPtrConstant(0, DL));
16579     return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
16580   }
16581 
16582   // See if we can widen to a 256-bit lane shuffle, we're going to lose 128-lane
16583   // UNDEF info by lowering to X86ISD::SHUF128 anyway, so by widening where
16584   // possible we at least ensure the lanes stay sequential to help later
16585   // combines.
16586   SmallVector<int, 2> Widened256Mask;
16587   if (canWidenShuffleElements(Widened128Mask, Widened256Mask)) {
16588     Widened128Mask.clear();
16589     narrowShuffleMaskElts(2, Widened256Mask, Widened128Mask);
16590   }
16591 
16592   // Try to lower to vshuf64x2/vshuf32x4.
16593   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
16594   int PermMask[4] = {-1, -1, -1, -1};
16595   // Ensure elements came from the same Op.
16596   for (int i = 0; i < 4; ++i) {
16597     assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16598     if (Widened128Mask[i] < 0)
16599       continue;
16600 
16601     SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
16602     unsigned OpIndex = i / 2;
16603     if (Ops[OpIndex].isUndef())
16604       Ops[OpIndex] = Op;
16605     else if (Ops[OpIndex] != Op)
16606       return SDValue();
16607 
16608     PermMask[i] = Widened128Mask[i] % 4;
16609   }
16610 
16611   return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
16612                      getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
16613 }
16614 
16615 /// Handle lowering of 8-lane 64-bit floating point shuffles.
16616 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16617                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16618                                  const X86Subtarget &Subtarget,
16619                                  SelectionDAG &DAG) {
16620   assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16621   assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16622   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16623 
16624   if (V2.isUndef()) {
16625     // Use low duplicate instructions for masks that match their pattern.
16626     if (isShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1, V2))
16627       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
16628 
16629     if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) {
16630       // Non-half-crossing single input shuffles can be lowered with an
16631       // interleaved permutation.
16632       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
16633                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3) |
16634                               ((Mask[4] == 5) << 4) | ((Mask[5] == 5) << 5) |
16635                               ((Mask[6] == 7) << 6) | ((Mask[7] == 7) << 7);
16636       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
16637                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16638     }
16639 
16640     SmallVector<int, 4> RepeatedMask;
16641     if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask))
16642       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
16643                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16644   }
16645 
16646   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
16647                                            V2, Subtarget, DAG))
16648     return Shuf128;
16649 
16650   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
16651     return Unpck;
16652 
16653   // Check if the blend happens to exactly fit that of SHUFPD.
16654   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
16655                                           Zeroable, Subtarget, DAG))
16656     return Op;
16657 
16658   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
16659                                        DAG, Subtarget))
16660     return V;
16661 
16662   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
16663                                           Zeroable, Subtarget, DAG))
16664     return Blend;
16665 
16666   return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);
16667 }
16668 
16669 /// Handle lowering of 16-lane 32-bit floating point shuffles.
16670 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16671                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16672                                   const X86Subtarget &Subtarget,
16673                                   SelectionDAG &DAG) {
16674   assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16675   assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16676   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16677 
16678   // If the shuffle mask is repeated in each 128-bit lane, we have many more
16679   // options to efficiently lower the shuffle.
16680   SmallVector<int, 4> RepeatedMask;
16681   if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) {
16682     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16683 
16684     // Use even/odd duplicate instructions for masks that match their pattern.
16685     if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
16686       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
16687     if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
16688       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
16689 
16690     if (V2.isUndef())
16691       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
16692                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16693 
16694     // Use dedicated unpack instructions for masks that match their pattern.
16695     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
16696       return V;
16697 
16698     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16699                                             Zeroable, Subtarget, DAG))
16700       return Blend;
16701 
16702     // Otherwise, fall back to a SHUFPS sequence.
16703     return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
16704   }
16705 
16706   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16707                                           Zeroable, Subtarget, DAG))
16708     return Blend;
16709 
16710   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16711           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16712     return DAG.getBitcast(MVT::v16f32, ZExt);
16713 
16714   // Try to create an in-lane repeating shuffle mask and then shuffle the
16715   // results into the target lanes.
16716   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16717           DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
16718     return V;
16719 
16720   // If we have a single input shuffle with different shuffle patterns in the
16721   // 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
16722   if (V2.isUndef() &&
16723       !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
16724     SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
16725     return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
16726   }
16727 
16728   // If we have AVX512F support, we can use VEXPAND.
16729   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
16730                                              V1, V2, DAG, Subtarget))
16731     return V;
16732 
16733   return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);
16734 }
16735 
16736 /// Handle lowering of 8-lane 64-bit integer shuffles.
16737 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16738                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16739                                  const X86Subtarget &Subtarget,
16740                                  SelectionDAG &DAG) {
16741   assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16742   assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16743   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16744 
16745   // Try to use shift instructions if fast.
16746   if (Subtarget.preferLowerShuffleAsShift())
16747     if (SDValue Shift =
16748             lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
16749                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16750       return Shift;
16751 
16752   if (V2.isUndef()) {
16753     // When the shuffle is mirrored between the 128-bit lanes of the unit, we
16754     // can use lower latency instructions that will operate on all four
16755     // 128-bit lanes.
16756     SmallVector<int, 2> Repeated128Mask;
16757     if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {
16758       SmallVector<int, 4> PSHUFDMask;
16759       narrowShuffleMaskElts(2, Repeated128Mask, PSHUFDMask);
16760       return DAG.getBitcast(
16761           MVT::v8i64,
16762           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
16763                       DAG.getBitcast(MVT::v16i32, V1),
16764                       getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16765     }
16766 
16767     SmallVector<int, 4> Repeated256Mask;
16768     if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask))
16769       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
16770                          getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
16771   }
16772 
16773   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
16774                                            V2, Subtarget, DAG))
16775     return Shuf128;
16776 
16777   // Try to use shift instructions.
16778   if (SDValue Shift =
16779           lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable, Subtarget,
16780                               DAG, /*BitwiseOnly*/ false))
16781     return Shift;
16782 
16783   // Try to use VALIGN.
16784   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
16785                                             Zeroable, Subtarget, DAG))
16786     return Rotate;
16787 
16788   // Try to use PALIGNR.
16789   if (Subtarget.hasBWI())
16790     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
16791                                                   Subtarget, DAG))
16792       return Rotate;
16793 
16794   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
16795     return Unpck;
16796 
16797   // If we have AVX512F support, we can use VEXPAND.
16798   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
16799                                        DAG, Subtarget))
16800     return V;
16801 
16802   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
16803                                           Zeroable, Subtarget, DAG))
16804     return Blend;
16805 
16806   return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG);
16807 }
16808 
16809 /// Handle lowering of 16-lane 32-bit integer shuffles.
16810 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16811                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16812                                   const X86Subtarget &Subtarget,
16813                                   SelectionDAG &DAG) {
16814   assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16815   assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16816   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16817 
16818   int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
16819 
16820   // Whenever we can lower this as a zext, that instruction is strictly faster
16821   // than any alternative. It also allows us to fold memory operands into the
16822   // shuffle in many cases.
16823   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16824           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16825     return ZExt;
16826 
16827   // Try to use shift instructions if fast.
16828   if (Subtarget.preferLowerShuffleAsShift()) {
16829     if (SDValue Shift =
16830             lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16831                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16832       return Shift;
16833     if (NumV2Elements == 0)
16834       if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask,
16835                                                    Subtarget, DAG))
16836         return Rotate;
16837   }
16838 
16839   // If the shuffle mask is repeated in each 128-bit lane we can use more
16840   // efficient instructions that mirror the shuffles across the four 128-bit
16841   // lanes.
16842   SmallVector<int, 4> RepeatedMask;
16843   bool Is128BitLaneRepeatedShuffle =
16844       is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask);
16845   if (Is128BitLaneRepeatedShuffle) {
16846     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16847     if (V2.isUndef())
16848       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
16849                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16850 
16851     // Use dedicated unpack instructions for masks that match their pattern.
16852     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
16853       return V;
16854   }
16855 
16856   // Try to use shift instructions.
16857   if (SDValue Shift =
16858           lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16859                               Subtarget, DAG, /*BitwiseOnly*/ false))
16860     return Shift;
16861 
16862   if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements != 0)
16863     if (SDValue Rotate =
16864             lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask, Subtarget, DAG))
16865       return Rotate;
16866 
16867   // Try to use VALIGN.
16868   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
16869                                             Zeroable, Subtarget, DAG))
16870     return Rotate;
16871 
16872   // Try to use byte rotation instructions.
16873   if (Subtarget.hasBWI())
16874     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
16875                                                   Subtarget, DAG))
16876       return Rotate;
16877 
16878   // Assume that a single SHUFPS is faster than using a permv shuffle.
16879   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16880   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16881     SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
16882     SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
16883     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
16884                                             CastV1, CastV2, DAG);
16885     return DAG.getBitcast(MVT::v16i32, ShufPS);
16886   }
16887 
16888   // Try to create an in-lane repeating shuffle mask and then shuffle the
16889   // results into the target lanes.
16890   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16891           DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
16892     return V;
16893 
16894   // If we have AVX512F support, we can use VEXPAND.
16895   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
16896                                        DAG, Subtarget))
16897     return V;
16898 
16899   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
16900                                           Zeroable, Subtarget, DAG))
16901     return Blend;
16902 
16903   return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG);
16904 }
16905 
16906 /// Handle lowering of 32-lane 16-bit integer shuffles.
16907 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16908                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16909                                   const X86Subtarget &Subtarget,
16910                                   SelectionDAG &DAG) {
16911   assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
16912   assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
16913   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
16914   assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!");
16915 
16916   // Whenever we can lower this as a zext, that instruction is strictly faster
16917   // than any alternative. It also allows us to fold memory operands into the
16918   // shuffle in many cases.
16919   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16920           DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16921     return ZExt;
16922 
16923   // Use dedicated unpack instructions for masks that match their pattern.
16924   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
16925     return V;
16926 
16927   // Use dedicated pack instructions for masks that match their pattern.
16928   if (SDValue V =
16929           lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
16930     return V;
16931 
16932   // Try to use shift instructions.
16933   if (SDValue Shift =
16934           lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask, Zeroable,
16935                               Subtarget, DAG, /*BitwiseOnly*/ false))
16936     return Shift;
16937 
16938   // Try to use byte rotation instructions.
16939   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
16940                                                 Subtarget, DAG))
16941     return Rotate;
16942 
16943   if (V2.isUndef()) {
16944     // Try to use bit rotation instructions.
16945     if (SDValue Rotate =
16946             lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))
16947       return Rotate;
16948 
16949     SmallVector<int, 8> RepeatedMask;
16950     if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {
16951       // As this is a single-input shuffle, the repeated mask should be
16952       // a strictly valid v8i16 mask that we can pass through to the v8i16
16953       // lowering to handle even the v32 case.
16954       return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,
16955                                                  RepeatedMask, Subtarget, DAG);
16956     }
16957   }
16958 
16959   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
16960                                           Zeroable, Subtarget, DAG))
16961     return Blend;
16962 
16963   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
16964                                               Zeroable, Subtarget, DAG))
16965     return PSHUFB;
16966 
16967   return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);
16968 }
16969 
16970 /// Handle lowering of 64-lane 8-bit integer shuffles.
16971 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16972                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16973                                  const X86Subtarget &Subtarget,
16974                                  SelectionDAG &DAG) {
16975   assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
16976   assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
16977   assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
16978   assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!");
16979 
16980   // Whenever we can lower this as a zext, that instruction is strictly faster
16981   // than any alternative. It also allows us to fold memory operands into the
16982   // shuffle in many cases.
16983   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16984           DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
16985     return ZExt;
16986 
16987   // Use dedicated unpack instructions for masks that match their pattern.
16988   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
16989     return V;
16990 
16991   // Use dedicated pack instructions for masks that match their pattern.
16992   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
16993                                        Subtarget))
16994     return V;
16995 
16996   // Try to use shift instructions.
16997   if (SDValue Shift =
16998           lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget,
16999                               DAG, /*BitwiseOnly*/ false))
17000     return Shift;
17001 
17002   // Try to use byte rotation instructions.
17003   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17004                                                 Subtarget, DAG))
17005     return Rotate;
17006 
17007   // Try to use bit rotation instructions.
17008   if (V2.isUndef())
17009     if (SDValue Rotate =
17010             lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
17011       return Rotate;
17012 
17013   // Lower as AND if possible.
17014   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17015                                              Zeroable, Subtarget, DAG))
17016     return Masked;
17017 
17018   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17019                                               Zeroable, Subtarget, DAG))
17020     return PSHUFB;
17021 
17022   // Try to create an in-lane repeating shuffle mask and then shuffle the
17023   // results into the target lanes.
17024   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17025           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17026     return V;
17027 
17028   if (SDValue Result = lowerShuffleAsLanePermuteAndPermute(
17029           DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget))
17030     return Result;
17031 
17032   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17033                                           Zeroable, Subtarget, DAG))
17034     return Blend;
17035 
17036   if (!is128BitLaneCrossingShuffleMask(MVT::v64i8, Mask)) {
17037     // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
17038     // PALIGNR will be cheaper than the second PSHUFB+OR.
17039     if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v64i8, V1, V2,
17040                                                        Mask, Subtarget, DAG))
17041       return V;
17042 
17043     // If we can't directly blend but can use PSHUFB, that will be better as it
17044     // can both shuffle and set up the inefficient blend.
17045     bool V1InUse, V2InUse;
17046     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v64i8, V1, V2, Mask, Zeroable,
17047                                         DAG, V1InUse, V2InUse);
17048   }
17049 
17050   // Try to simplify this by merging 128-bit lanes to enable a lane-based
17051   // shuffle.
17052   if (!V2.isUndef())
17053     if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17054             DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17055       return Result;
17056 
17057   // VBMI can use VPERMV/VPERMV3 byte shuffles.
17058   if (Subtarget.hasVBMI())
17059     return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG);
17060 
17061   return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17062 }
17063 
17064 /// High-level routine to lower various 512-bit x86 vector shuffles.
17065 ///
17066 /// This routine either breaks down the specific type of a 512-bit x86 vector
17067 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
17068 /// together based on the available instructions.
17069 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17070                                   MVT VT, SDValue V1, SDValue V2,
17071                                   const APInt &Zeroable,
17072                                   const X86Subtarget &Subtarget,
17073                                   SelectionDAG &DAG) {
17074   assert(Subtarget.hasAVX512() &&
17075          "Cannot lower 512-bit vectors w/ basic ISA!");
17076 
17077   // If we have a single input to the zero element, insert that into V1 if we
17078   // can do so cheaply.
17079   int NumElts = Mask.size();
17080   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17081 
17082   if (NumV2Elements == 1 && Mask[0] >= NumElts)
17083     if (SDValue Insertion = lowerShuffleAsElementInsertion(
17084             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17085       return Insertion;
17086 
17087   // Handle special cases where the lower or upper half is UNDEF.
17088   if (SDValue V =
17089           lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
17090     return V;
17091 
17092   // Check for being able to broadcast a single element.
17093   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17094                                                   Subtarget, DAG))
17095     return Broadcast;
17096 
17097   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
17098     // Try using bit ops for masking and blending before falling back to
17099     // splitting.
17100     if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17101                                           Subtarget, DAG))
17102       return V;
17103     if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17104       return V;
17105 
17106     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17107   }
17108 
17109   if (VT == MVT::v32f16 || VT == MVT::v32bf16) {
17110     if (!Subtarget.hasBWI())
17111       return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
17112                                   /*SimpleOnly*/ false);
17113 
17114     V1 = DAG.getBitcast(MVT::v32i16, V1);
17115     V2 = DAG.getBitcast(MVT::v32i16, V2);
17116     return DAG.getBitcast(VT,
17117                           DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask));
17118   }
17119 
17120   // Dispatch to each element type for lowering. If we don't have support for
17121   // specific element type shuffles at 512 bits, immediately split them and
17122   // lower them. Each lowering routine of a given type is allowed to assume that
17123   // the requisite ISA extensions for that element type are available.
17124   switch (VT.SimpleTy) {
17125   case MVT::v8f64:
17126     return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17127   case MVT::v16f32:
17128     return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17129   case MVT::v8i64:
17130     return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17131   case MVT::v16i32:
17132     return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17133   case MVT::v32i16:
17134     return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17135   case MVT::v64i8:
17136     return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17137 
17138   default:
17139     llvm_unreachable("Not a valid 512-bit x86 vector type!");
17140   }
17141 }
17142 
17143 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17144                                          MVT VT, SDValue V1, SDValue V2,
17145                                          const X86Subtarget &Subtarget,
17146                                          SelectionDAG &DAG) {
17147   // Shuffle should be unary.
17148   if (!V2.isUndef())
17149     return SDValue();
17150 
17151   int ShiftAmt = -1;
17152   int NumElts = Mask.size();
17153   for (int i = 0; i != NumElts; ++i) {
17154     int M = Mask[i];
17155     assert((M == SM_SentinelUndef || (0 <= M && M < NumElts)) &&
17156            "Unexpected mask index.");
17157     if (M < 0)
17158       continue;
17159 
17160     // The first non-undef element determines our shift amount.
17161     if (ShiftAmt < 0) {
17162       ShiftAmt = M - i;
17163       // Need to be shifting right.
17164       if (ShiftAmt <= 0)
17165         return SDValue();
17166     }
17167     // All non-undef elements must shift by the same amount.
17168     if (ShiftAmt != M - i)
17169       return SDValue();
17170   }
17171   assert(ShiftAmt >= 0 && "All undef?");
17172 
17173   // Great we found a shift right.
17174   SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL);
17175   Res = DAG.getNode(X86ISD::KSHIFTR, DL, Res.getValueType(), Res,
17176                     DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17177   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17178                      DAG.getIntPtrConstant(0, DL));
17179 }
17180 
17181 // Determine if this shuffle can be implemented with a KSHIFT instruction.
17182 // Returns the shift amount if possible or -1 if not. This is a simplified
17183 // version of matchShuffleAsShift.
17184 static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
17185                                     int MaskOffset, const APInt &Zeroable) {
17186   int Size = Mask.size();
17187 
17188   auto CheckZeros = [&](int Shift, bool Left) {
17189     for (int j = 0; j < Shift; ++j)
17190       if (!Zeroable[j + (Left ? 0 : (Size - Shift))])
17191         return false;
17192 
17193     return true;
17194   };
17195 
17196   auto MatchShift = [&](int Shift, bool Left) {
17197     unsigned Pos = Left ? Shift : 0;
17198     unsigned Low = Left ? 0 : Shift;
17199     unsigned Len = Size - Shift;
17200     return isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset);
17201   };
17202 
17203   for (int Shift = 1; Shift != Size; ++Shift)
17204     for (bool Left : {true, false})
17205       if (CheckZeros(Shift, Left) && MatchShift(Shift, Left)) {
17206         Opcode = Left ? X86ISD::KSHIFTL : X86ISD::KSHIFTR;
17207         return Shift;
17208       }
17209 
17210   return -1;
17211 }
17212 
17213 
17214 // Lower vXi1 vector shuffles.
17215 // There is no a dedicated instruction on AVX-512 that shuffles the masks.
17216 // The only way to shuffle bits is to sign-extend the mask vector to SIMD
17217 // vector, shuffle and then truncate it back.
17218 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17219                                 MVT VT, SDValue V1, SDValue V2,
17220                                 const APInt &Zeroable,
17221                                 const X86Subtarget &Subtarget,
17222                                 SelectionDAG &DAG) {
17223   assert(Subtarget.hasAVX512() &&
17224          "Cannot lower 512-bit vectors w/o basic ISA!");
17225 
17226   int NumElts = Mask.size();
17227   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17228 
17229   // Try to recognize shuffles that are just padding a subvector with zeros.
17230   int SubvecElts = 0;
17231   int Src = -1;
17232   for (int i = 0; i != NumElts; ++i) {
17233     if (Mask[i] >= 0) {
17234       // Grab the source from the first valid mask. All subsequent elements need
17235       // to use this same source.
17236       if (Src < 0)
17237         Src = Mask[i] / NumElts;
17238       if (Src != (Mask[i] / NumElts) || (Mask[i] % NumElts) != i)
17239         break;
17240     }
17241 
17242     ++SubvecElts;
17243   }
17244   assert(SubvecElts != NumElts && "Identity shuffle?");
17245 
17246   // Clip to a power 2.
17247   SubvecElts = llvm::bit_floor<uint32_t>(SubvecElts);
17248 
17249   // Make sure the number of zeroable bits in the top at least covers the bits
17250   // not covered by the subvector.
17251   if ((int)Zeroable.countl_one() >= (NumElts - SubvecElts)) {
17252     assert(Src >= 0 && "Expected a source!");
17253     MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts);
17254     SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17255                                   Src == 0 ? V1 : V2,
17256                                   DAG.getIntPtrConstant(0, DL));
17257     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17258                        DAG.getConstant(0, DL, VT),
17259                        Extract, DAG.getIntPtrConstant(0, DL));
17260   }
17261 
17262   // Try a simple shift right with undef elements. Later we'll try with zeros.
17263   if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17264                                                 DAG))
17265     return Shift;
17266 
17267   // Try to match KSHIFTs.
17268   unsigned Offset = 0;
17269   for (SDValue V : { V1, V2 }) {
17270     unsigned Opcode;
17271     int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable);
17272     if (ShiftAmt >= 0) {
17273       SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL);
17274       MVT WideVT = Res.getSimpleValueType();
17275       // Widened right shifts need two shifts to ensure we shift in zeroes.
17276       if (Opcode == X86ISD::KSHIFTR && WideVT != VT) {
17277         int WideElts = WideVT.getVectorNumElements();
17278         // Shift left to put the original vector in the MSBs of the new size.
17279         Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17280                           DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17281         // Increase the shift amount to account for the left shift.
17282         ShiftAmt += WideElts - NumElts;
17283       }
17284 
17285       Res = DAG.getNode(Opcode, DL, WideVT, Res,
17286                         DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17287       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17288                          DAG.getIntPtrConstant(0, DL));
17289     }
17290     Offset += NumElts; // Increment for next iteration.
17291   }
17292 
17293   // If we're performing an unary shuffle on a SETCC result, try to shuffle the
17294   // ops instead.
17295   // TODO: What other unary shuffles would benefit from this?
17296   if (NumV2Elements == 0 && V1.getOpcode() == ISD::SETCC && V1->hasOneUse()) {
17297     SDValue Op0 = V1.getOperand(0);
17298     SDValue Op1 = V1.getOperand(1);
17299     ISD::CondCode CC = cast<CondCodeSDNode>(V1.getOperand(2))->get();
17300     EVT OpVT = Op0.getValueType();
17301     if (OpVT.getScalarSizeInBits() >= 32 || isBroadcastShuffleMask(Mask))
17302       return DAG.getSetCC(
17303           DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask),
17304           DAG.getVectorShuffle(OpVT, DL, Op1, DAG.getUNDEF(OpVT), Mask), CC);
17305   }
17306 
17307   MVT ExtVT;
17308   switch (VT.SimpleTy) {
17309   default:
17310     llvm_unreachable("Expected a vector of i1 elements");
17311   case MVT::v2i1:
17312     ExtVT = MVT::v2i64;
17313     break;
17314   case MVT::v4i1:
17315     ExtVT = MVT::v4i32;
17316     break;
17317   case MVT::v8i1:
17318     // Take 512-bit type, more shuffles on KNL. If we have VLX use a 256-bit
17319     // shuffle.
17320     ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64;
17321     break;
17322   case MVT::v16i1:
17323     // Take 512-bit type, unless we are avoiding 512-bit types and have the
17324     // 256-bit operation available.
17325     ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16;
17326     break;
17327   case MVT::v32i1:
17328     // Take 512-bit type, unless we are avoiding 512-bit types and have the
17329     // 256-bit operation available.
17330     assert(Subtarget.hasBWI() && "Expected AVX512BW support");
17331     ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8;
17332     break;
17333   case MVT::v64i1:
17334     // Fall back to scalarization. FIXME: We can do better if the shuffle
17335     // can be partitioned cleanly.
17336     if (!Subtarget.useBWIRegs())
17337       return SDValue();
17338     ExtVT = MVT::v64i8;
17339     break;
17340   }
17341 
17342   V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17343   V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17344 
17345   SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17346   // i1 was sign extended we can use X86ISD::CVT2MASK.
17347   int NumElems = VT.getVectorNumElements();
17348   if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
17349       (Subtarget.hasDQI() && (NumElems < 32)))
17350     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17351                        Shuffle, ISD::SETGT);
17352 
17353   return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17354 }
17355 
17356 /// Helper function that returns true if the shuffle mask should be
17357 /// commuted to improve canonicalization.
17358 static bool canonicalizeShuffleMaskWithCommute(ArrayRef<int> Mask) {
17359   int NumElements = Mask.size();
17360 
17361   int NumV1Elements = 0, NumV2Elements = 0;
17362   for (int M : Mask)
17363     if (M < 0)
17364       continue;
17365     else if (M < NumElements)
17366       ++NumV1Elements;
17367     else
17368       ++NumV2Elements;
17369 
17370   // Commute the shuffle as needed such that more elements come from V1 than
17371   // V2. This allows us to match the shuffle pattern strictly on how many
17372   // elements come from V1 without handling the symmetric cases.
17373   if (NumV2Elements > NumV1Elements)
17374     return true;
17375 
17376   assert(NumV1Elements > 0 && "No V1 indices");
17377 
17378   if (NumV2Elements == 0)
17379     return false;
17380 
17381   // When the number of V1 and V2 elements are the same, try to minimize the
17382   // number of uses of V2 in the low half of the vector. When that is tied,
17383   // ensure that the sum of indices for V1 is equal to or lower than the sum
17384   // indices for V2. When those are equal, try to ensure that the number of odd
17385   // indices for V1 is lower than the number of odd indices for V2.
17386   if (NumV1Elements == NumV2Elements) {
17387     int LowV1Elements = 0, LowV2Elements = 0;
17388     for (int M : Mask.slice(0, NumElements / 2))
17389       if (M >= NumElements)
17390         ++LowV2Elements;
17391       else if (M >= 0)
17392         ++LowV1Elements;
17393     if (LowV2Elements > LowV1Elements)
17394       return true;
17395     if (LowV2Elements == LowV1Elements) {
17396       int SumV1Indices = 0, SumV2Indices = 0;
17397       for (int i = 0, Size = Mask.size(); i < Size; ++i)
17398         if (Mask[i] >= NumElements)
17399           SumV2Indices += i;
17400         else if (Mask[i] >= 0)
17401           SumV1Indices += i;
17402       if (SumV2Indices < SumV1Indices)
17403         return true;
17404       if (SumV2Indices == SumV1Indices) {
17405         int NumV1OddIndices = 0, NumV2OddIndices = 0;
17406         for (int i = 0, Size = Mask.size(); i < Size; ++i)
17407           if (Mask[i] >= NumElements)
17408             NumV2OddIndices += i % 2;
17409           else if (Mask[i] >= 0)
17410             NumV1OddIndices += i % 2;
17411         if (NumV2OddIndices < NumV1OddIndices)
17412           return true;
17413       }
17414     }
17415   }
17416 
17417   return false;
17418 }
17419 
17420 static bool canCombineAsMaskOperation(SDValue V,
17421                                       const X86Subtarget &Subtarget) {
17422   if (!Subtarget.hasAVX512())
17423     return false;
17424 
17425   if (!V.getValueType().isSimple())
17426     return false;
17427 
17428   MVT VT = V.getSimpleValueType().getScalarType();
17429   if ((VT == MVT::i16 || VT == MVT::i8) && !Subtarget.hasBWI())
17430     return false;
17431 
17432   // If vec width < 512, widen i8/i16 even with BWI as blendd/blendps/blendpd
17433   // are preferable to blendw/blendvb/masked-mov.
17434   if ((VT == MVT::i16 || VT == MVT::i8) &&
17435       V.getSimpleValueType().getSizeInBits() < 512)
17436     return false;
17437 
17438   auto HasMaskOperation = [&](SDValue V) {
17439     // TODO: Currently we only check limited opcode. We probably extend
17440     // it to all binary operation by checking TLI.isBinOp().
17441     switch (V->getOpcode()) {
17442     default:
17443       return false;
17444     case ISD::ADD:
17445     case ISD::SUB:
17446     case ISD::AND:
17447     case ISD::XOR:
17448     case ISD::OR:
17449     case ISD::SMAX:
17450     case ISD::SMIN:
17451     case ISD::UMAX:
17452     case ISD::UMIN:
17453     case ISD::ABS:
17454     case ISD::SHL:
17455     case ISD::SRL:
17456     case ISD::SRA:
17457     case ISD::MUL:
17458       break;
17459     }
17460     if (!V->hasOneUse())
17461       return false;
17462 
17463     return true;
17464   };
17465 
17466   if (HasMaskOperation(V))
17467     return true;
17468 
17469   return false;
17470 }
17471 
17472 // Forward declaration.
17473 static SDValue canonicalizeShuffleMaskWithHorizOp(
17474     MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
17475     unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
17476     const X86Subtarget &Subtarget);
17477 
17478     /// Top-level lowering for x86 vector shuffles.
17479 ///
17480 /// This handles decomposition, canonicalization, and lowering of all x86
17481 /// vector shuffles. Most of the specific lowering strategies are encapsulated
17482 /// above in helper routines. The canonicalization attempts to widen shuffles
17483 /// to involve fewer lanes of wider elements, consolidate symmetric patterns
17484 /// s.t. only one of the two inputs needs to be tested, etc.
17485 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17486                                    SelectionDAG &DAG) {
17487   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
17488   ArrayRef<int> OrigMask = SVOp->getMask();
17489   SDValue V1 = Op.getOperand(0);
17490   SDValue V2 = Op.getOperand(1);
17491   MVT VT = Op.getSimpleValueType();
17492   int NumElements = VT.getVectorNumElements();
17493   SDLoc DL(Op);
17494   bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);
17495 
17496   assert((VT.getSizeInBits() != 64 || Is1BitVector) &&
17497          "Can't lower MMX shuffles");
17498 
17499   bool V1IsUndef = V1.isUndef();
17500   bool V2IsUndef = V2.isUndef();
17501   if (V1IsUndef && V2IsUndef)
17502     return DAG.getUNDEF(VT);
17503 
17504   // When we create a shuffle node we put the UNDEF node to second operand,
17505   // but in some cases the first operand may be transformed to UNDEF.
17506   // In this case we should just commute the node.
17507   if (V1IsUndef)
17508     return DAG.getCommutedVectorShuffle(*SVOp);
17509 
17510   // Check for non-undef masks pointing at an undef vector and make the masks
17511   // undef as well. This makes it easier to match the shuffle based solely on
17512   // the mask.
17513   if (V2IsUndef &&
17514       any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {
17515     SmallVector<int, 8> NewMask(OrigMask);
17516     for (int &M : NewMask)
17517       if (M >= NumElements)
17518         M = -1;
17519     return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17520   }
17521 
17522   // Check for illegal shuffle mask element index values.
17523   int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
17524   (void)MaskUpperLimit;
17525   assert(llvm::all_of(OrigMask,
17526                       [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
17527          "Out of bounds shuffle index");
17528 
17529   // We actually see shuffles that are entirely re-arrangements of a set of
17530   // zero inputs. This mostly happens while decomposing complex shuffles into
17531   // simple ones. Directly lower these as a buildvector of zeros.
17532   APInt KnownUndef, KnownZero;
17533   computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero);
17534 
17535   APInt Zeroable = KnownUndef | KnownZero;
17536   if (Zeroable.isAllOnes())
17537     return getZeroVector(VT, Subtarget, DAG, DL);
17538 
17539   bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode());
17540 
17541   // Try to collapse shuffles into using a vector type with fewer elements but
17542   // wider element types. We cap this to not form integers or floating point
17543   // elements wider than 64 bits. It does not seem beneficial to form i128
17544   // integers to handle flipping the low and high halves of AVX 256-bit vectors.
17545   SmallVector<int, 16> WidenedMask;
17546   if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&
17547       !canCombineAsMaskOperation(V1, Subtarget) &&
17548       !canCombineAsMaskOperation(V2, Subtarget) &&
17549       canWidenShuffleElements(OrigMask, Zeroable, V2IsZero, WidenedMask)) {
17550     // Shuffle mask widening should not interfere with a broadcast opportunity
17551     // by obfuscating the operands with bitcasts.
17552     // TODO: Avoid lowering directly from this top-level function: make this
17553     // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
17554     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17555                                                     Subtarget, DAG))
17556       return Broadcast;
17557 
17558     MVT NewEltVT = VT.isFloatingPoint()
17559                        ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
17560                        : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
17561     int NewNumElts = NumElements / 2;
17562     MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);
17563     // Make sure that the new vector type is legal. For example, v2f64 isn't
17564     // legal on SSE1.
17565     if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
17566       if (V2IsZero) {
17567         // Modify the new Mask to take all zeros from the all-zero vector.
17568         // Choose indices that are blend-friendly.
17569         bool UsedZeroVector = false;
17570         assert(is_contained(WidenedMask, SM_SentinelZero) &&
17571                "V2's non-undef elements are used?!");
17572         for (int i = 0; i != NewNumElts; ++i)
17573           if (WidenedMask[i] == SM_SentinelZero) {
17574             WidenedMask[i] = i + NewNumElts;
17575             UsedZeroVector = true;
17576           }
17577         // Ensure all elements of V2 are zero - isBuildVectorAllZeros permits
17578         // some elements to be undef.
17579         if (UsedZeroVector)
17580           V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
17581       }
17582       V1 = DAG.getBitcast(NewVT, V1);
17583       V2 = DAG.getBitcast(NewVT, V2);
17584       return DAG.getBitcast(
17585           VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
17586     }
17587   }
17588 
17589   SmallVector<SDValue> Ops = {V1, V2};
17590   SmallVector<int> Mask(OrigMask);
17591 
17592   // Canonicalize the shuffle with any horizontal ops inputs.
17593   // NOTE: This may update Ops and Mask.
17594   if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
17595           Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget))
17596     return DAG.getBitcast(VT, HOp);
17597 
17598   V1 = DAG.getBitcast(VT, Ops[0]);
17599   V2 = DAG.getBitcast(VT, Ops[1]);
17600   assert(NumElements == (int)Mask.size() &&
17601          "canonicalizeShuffleMaskWithHorizOp "
17602          "shouldn't alter the shuffle mask size");
17603 
17604   // Commute the shuffle if it will improve canonicalization.
17605   if (canonicalizeShuffleMaskWithCommute(Mask)) {
17606     ShuffleVectorSDNode::commuteMask(Mask);
17607     std::swap(V1, V2);
17608   }
17609 
17610   // For each vector width, delegate to a specialized lowering routine.
17611   if (VT.is128BitVector())
17612     return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17613 
17614   if (VT.is256BitVector())
17615     return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17616 
17617   if (VT.is512BitVector())
17618     return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17619 
17620   if (Is1BitVector)
17621     return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17622 
17623   llvm_unreachable("Unimplemented!");
17624 }
17625 
17626 /// Try to lower a VSELECT instruction to a vector shuffle.
17627 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17628                                            const X86Subtarget &Subtarget,
17629                                            SelectionDAG &DAG) {
17630   SDValue Cond = Op.getOperand(0);
17631   SDValue LHS = Op.getOperand(1);
17632   SDValue RHS = Op.getOperand(2);
17633   MVT VT = Op.getSimpleValueType();
17634 
17635   // Only non-legal VSELECTs reach this lowering, convert those into generic
17636   // shuffles and re-use the shuffle lowering path for blends.
17637   if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) {
17638     SmallVector<int, 32> Mask;
17639     if (createShuffleMaskFromVSELECT(Mask, Cond))
17640       return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);
17641   }
17642 
17643   return SDValue();
17644 }
17645 
17646 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17647   SDValue Cond = Op.getOperand(0);
17648   SDValue LHS = Op.getOperand(1);
17649   SDValue RHS = Op.getOperand(2);
17650 
17651   SDLoc dl(Op);
17652   MVT VT = Op.getSimpleValueType();
17653   if (isSoftF16(VT, Subtarget)) {
17654     MVT NVT = VT.changeVectorElementTypeToInteger();
17655     return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond,
17656                                           DAG.getBitcast(NVT, LHS),
17657                                           DAG.getBitcast(NVT, RHS)));
17658   }
17659 
17660   // A vselect where all conditions and data are constants can be optimized into
17661   // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
17662   if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()) &&
17663       ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
17664       ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
17665     return SDValue();
17666 
17667   // Try to lower this to a blend-style vector shuffle. This can handle all
17668   // constant condition cases.
17669   if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
17670     return BlendOp;
17671 
17672   // If this VSELECT has a vector if i1 as a mask, it will be directly matched
17673   // with patterns on the mask registers on AVX-512.
17674   MVT CondVT = Cond.getSimpleValueType();
17675   unsigned CondEltSize = Cond.getScalarValueSizeInBits();
17676   if (CondEltSize == 1)
17677     return Op;
17678 
17679   // Variable blends are only legal from SSE4.1 onward.
17680   if (!Subtarget.hasSSE41())
17681     return SDValue();
17682 
17683   unsigned EltSize = VT.getScalarSizeInBits();
17684   unsigned NumElts = VT.getVectorNumElements();
17685 
17686   // Expand v32i16/v64i8 without BWI.
17687   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
17688     return SDValue();
17689 
17690   // If the VSELECT is on a 512-bit type, we have to convert a non-i1 condition
17691   // into an i1 condition so that we can use the mask-based 512-bit blend
17692   // instructions.
17693   if (VT.getSizeInBits() == 512) {
17694     // Build a mask by testing the condition against zero.
17695     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
17696     SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
17697                                 DAG.getConstant(0, dl, CondVT),
17698                                 ISD::SETNE);
17699     // Now return a new VSELECT using the mask.
17700     return DAG.getSelect(dl, VT, Mask, LHS, RHS);
17701   }
17702 
17703   // SEXT/TRUNC cases where the mask doesn't match the destination size.
17704   if (CondEltSize != EltSize) {
17705     // If we don't have a sign splat, rely on the expansion.
17706     if (CondEltSize != DAG.ComputeNumSignBits(Cond))
17707       return SDValue();
17708 
17709     MVT NewCondSVT = MVT::getIntegerVT(EltSize);
17710     MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts);
17711     Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT);
17712     return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);
17713   }
17714 
17715   // Only some types will be legal on some subtargets. If we can emit a legal
17716   // VSELECT-matching blend, return Op, and but if we need to expand, return
17717   // a null value.
17718   switch (VT.SimpleTy) {
17719   default:
17720     // Most of the vector types have blends past SSE4.1.
17721     return Op;
17722 
17723   case MVT::v32i8:
17724     // The byte blends for AVX vectors were introduced only in AVX2.
17725     if (Subtarget.hasAVX2())
17726       return Op;
17727 
17728     return SDValue();
17729 
17730   case MVT::v8i16:
17731   case MVT::v16i16: {
17732     // Bitcast everything to the vXi8 type and use a vXi8 vselect.
17733     MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);
17734     Cond = DAG.getBitcast(CastVT, Cond);
17735     LHS = DAG.getBitcast(CastVT, LHS);
17736     RHS = DAG.getBitcast(CastVT, RHS);
17737     SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
17738     return DAG.getBitcast(VT, Select);
17739   }
17740   }
17741 }
17742 
17743 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17744   MVT VT = Op.getSimpleValueType();
17745   SDValue Vec = Op.getOperand(0);
17746   SDValue Idx = Op.getOperand(1);
17747   assert(isa<ConstantSDNode>(Idx) && "Constant index expected");
17748   SDLoc dl(Op);
17749 
17750   if (!Vec.getSimpleValueType().is128BitVector())
17751     return SDValue();
17752 
17753   if (VT.getSizeInBits() == 8) {
17754     // If IdxVal is 0, it's cheaper to do a move instead of a pextrb, unless
17755     // we're going to zero extend the register or fold the store.
17756     if (llvm::isNullConstant(Idx) && !X86::mayFoldIntoZeroExtend(Op) &&
17757         !X86::mayFoldIntoStore(Op))
17758       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
17759                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17760                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));
17761 
17762     unsigned IdxVal = Idx->getAsZExtVal();
17763     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,
17764                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17765     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17766   }
17767 
17768   if (VT == MVT::f32) {
17769     // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
17770     // the result back to FR32 register. It's only worth matching if the
17771     // result has a single use which is a store or a bitcast to i32.  And in
17772     // the case of a store, it's not worth it if the index is a constant 0,
17773     // because a MOVSSmr can be used instead, which is smaller and faster.
17774     if (!Op.hasOneUse())
17775       return SDValue();
17776     SDNode *User = *Op.getNode()->use_begin();
17777     if ((User->getOpcode() != ISD::STORE || isNullConstant(Idx)) &&
17778         (User->getOpcode() != ISD::BITCAST ||
17779          User->getValueType(0) != MVT::i32))
17780       return SDValue();
17781     SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17782                                   DAG.getBitcast(MVT::v4i32, Vec), Idx);
17783     return DAG.getBitcast(MVT::f32, Extract);
17784   }
17785 
17786   if (VT == MVT::i32 || VT == MVT::i64)
17787       return Op;
17788 
17789   return SDValue();
17790 }
17791 
17792 /// Extract one bit from mask vector, like v16i1 or v8i1.
17793 /// AVX-512 feature.
17794 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17795                                         const X86Subtarget &Subtarget) {
17796   SDValue Vec = Op.getOperand(0);
17797   SDLoc dl(Vec);
17798   MVT VecVT = Vec.getSimpleValueType();
17799   SDValue Idx = Op.getOperand(1);
17800   auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
17801   MVT EltVT = Op.getSimpleValueType();
17802 
17803   assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&
17804          "Unexpected vector type in ExtractBitFromMaskVector");
17805 
17806   // variable index can't be handled in mask registers,
17807   // extend vector to VR512/128
17808   if (!IdxC) {
17809     unsigned NumElts = VecVT.getVectorNumElements();
17810     // Extending v8i1/v16i1 to 512-bit get better performance on KNL
17811     // than extending to 128/256bit.
17812     if (NumElts == 1) {
17813       Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17814       MVT IntVT = MVT::getIntegerVT(Vec.getValueType().getVectorNumElements());
17815       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
17816     }
17817     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
17818     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
17819     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
17820     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
17821     return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
17822   }
17823 
17824   unsigned IdxVal = IdxC->getZExtValue();
17825   if (IdxVal == 0) // the operation is legal
17826     return Op;
17827 
17828   // Extend to natively supported kshift.
17829   Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17830 
17831   // Use kshiftr instruction to move to the lower element.
17832   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
17833                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17834 
17835   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17836                      DAG.getIntPtrConstant(0, dl));
17837 }
17838 
17839 // Helper to find all the extracted elements from a vector.
17840 static APInt getExtractedDemandedElts(SDNode *N) {
17841   MVT VT = N->getSimpleValueType(0);
17842   unsigned NumElts = VT.getVectorNumElements();
17843   APInt DemandedElts = APInt::getZero(NumElts);
17844   for (SDNode *User : N->uses()) {
17845     switch (User->getOpcode()) {
17846     case X86ISD::PEXTRB:
17847     case X86ISD::PEXTRW:
17848     case ISD::EXTRACT_VECTOR_ELT:
17849       if (!isa<ConstantSDNode>(User->getOperand(1))) {
17850         DemandedElts.setAllBits();
17851         return DemandedElts;
17852       }
17853       DemandedElts.setBit(User->getConstantOperandVal(1));
17854       break;
17855     case ISD::BITCAST: {
17856       if (!User->getValueType(0).isSimple() ||
17857           !User->getValueType(0).isVector()) {
17858         DemandedElts.setAllBits();
17859         return DemandedElts;
17860       }
17861       APInt DemandedSrcElts = getExtractedDemandedElts(User);
17862       DemandedElts |= APIntOps::ScaleBitMask(DemandedSrcElts, NumElts);
17863       break;
17864     }
17865     default:
17866       DemandedElts.setAllBits();
17867       return DemandedElts;
17868     }
17869   }
17870   return DemandedElts;
17871 }
17872 
17873 SDValue
17874 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
17875                                            SelectionDAG &DAG) const {
17876   SDLoc dl(Op);
17877   SDValue Vec = Op.getOperand(0);
17878   MVT VecVT = Vec.getSimpleValueType();
17879   SDValue Idx = Op.getOperand(1);
17880   auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
17881 
17882   if (VecVT.getVectorElementType() == MVT::i1)
17883     return ExtractBitFromMaskVector(Op, DAG, Subtarget);
17884 
17885   if (!IdxC) {
17886     // Its more profitable to go through memory (1 cycles throughput)
17887     // than using VMOVD + VPERMV/PSHUFB sequence (2/3 cycles throughput)
17888     // IACA tool was used to get performance estimation
17889     // (https://software.intel.com/en-us/articles/intel-architecture-code-analyzer)
17890     //
17891     // example : extractelement <16 x i8> %a, i32 %i
17892     //
17893     // Block Throughput: 3.00 Cycles
17894     // Throughput Bottleneck: Port5
17895     //
17896     // | Num Of |   Ports pressure in cycles  |    |
17897     // |  Uops  |  0  - DV  |  5  |  6  |  7  |    |
17898     // ---------------------------------------------
17899     // |   1    |           | 1.0 |     |     | CP | vmovd xmm1, edi
17900     // |   1    |           | 1.0 |     |     | CP | vpshufb xmm0, xmm0, xmm1
17901     // |   2    | 1.0       | 1.0 |     |     | CP | vpextrb eax, xmm0, 0x0
17902     // Total Num Of Uops: 4
17903     //
17904     //
17905     // Block Throughput: 1.00 Cycles
17906     // Throughput Bottleneck: PORT2_AGU, PORT3_AGU, Port4
17907     //
17908     // |    |  Ports pressure in cycles   |  |
17909     // |Uops| 1 | 2 - D  |3 -  D  | 4 | 5 |  |
17910     // ---------------------------------------------------------
17911     // |2^  |   | 0.5    | 0.5    |1.0|   |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
17912     // |1   |0.5|        |        |   |0.5|  | lea rax, ptr [rsp-0x18]
17913     // |1   |   |0.5, 0.5|0.5, 0.5|   |   |CP| mov al, byte ptr [rdi+rax*1]
17914     // Total Num Of Uops: 4
17915 
17916     return SDValue();
17917   }
17918 
17919   unsigned IdxVal = IdxC->getZExtValue();
17920 
17921   // If this is a 256-bit vector result, first extract the 128-bit vector and
17922   // then extract the element from the 128-bit vector.
17923   if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
17924     // Get the 128-bit vector.
17925     Vec = extract128BitVector(Vec, IdxVal, DAG, dl);
17926     MVT EltVT = VecVT.getVectorElementType();
17927 
17928     unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
17929     assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
17930 
17931     // Find IdxVal modulo ElemsPerChunk. Since ElemsPerChunk is a power of 2
17932     // this can be done with a mask.
17933     IdxVal &= ElemsPerChunk - 1;
17934     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17935                        DAG.getIntPtrConstant(IdxVal, dl));
17936   }
17937 
17938   assert(VecVT.is128BitVector() && "Unexpected vector length");
17939 
17940   MVT VT = Op.getSimpleValueType();
17941 
17942   if (VT == MVT::i16) {
17943     // If IdxVal is 0, it's cheaper to do a move instead of a pextrw, unless
17944     // we're going to zero extend the register or fold the store (SSE41 only).
17945     if (IdxVal == 0 && !X86::mayFoldIntoZeroExtend(Op) &&
17946         !(Subtarget.hasSSE41() && X86::mayFoldIntoStore(Op))) {
17947       if (Subtarget.hasFP16())
17948         return Op;
17949 
17950       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
17951                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17952                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));
17953     }
17954 
17955     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
17956                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17957     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17958   }
17959 
17960   if (Subtarget.hasSSE41())
17961     if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
17962       return Res;
17963 
17964   // Only extract a single element from a v16i8 source - determine the common
17965   // DWORD/WORD that all extractions share, and extract the sub-byte.
17966   // TODO: Add QWORD MOVQ extraction?
17967   if (VT == MVT::i8) {
17968     APInt DemandedElts = getExtractedDemandedElts(Vec.getNode());
17969     assert(DemandedElts.getBitWidth() == 16 && "Vector width mismatch");
17970 
17971     // Extract either the lowest i32 or any i16, and extract the sub-byte.
17972     int DWordIdx = IdxVal / 4;
17973     if (DWordIdx == 0 && DemandedElts == (DemandedElts & 15)) {
17974       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17975                                 DAG.getBitcast(MVT::v4i32, Vec),
17976                                 DAG.getIntPtrConstant(DWordIdx, dl));
17977       int ShiftVal = (IdxVal % 4) * 8;
17978       if (ShiftVal != 0)
17979         Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
17980                           DAG.getConstant(ShiftVal, dl, MVT::i8));
17981       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17982     }
17983 
17984     int WordIdx = IdxVal / 2;
17985     if (DemandedElts == (DemandedElts & (3 << (WordIdx * 2)))) {
17986       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
17987                                 DAG.getBitcast(MVT::v8i16, Vec),
17988                                 DAG.getIntPtrConstant(WordIdx, dl));
17989       int ShiftVal = (IdxVal % 2) * 8;
17990       if (ShiftVal != 0)
17991         Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
17992                           DAG.getConstant(ShiftVal, dl, MVT::i8));
17993       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17994     }
17995   }
17996 
17997   if (VT == MVT::f16 || VT.getSizeInBits() == 32) {
17998     if (IdxVal == 0)
17999       return Op;
18000 
18001     // Shuffle the element to the lowest element, then movss or movsh.
18002     SmallVector<int, 8> Mask(VecVT.getVectorNumElements(), -1);
18003     Mask[0] = static_cast<int>(IdxVal);
18004     Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18005     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18006                        DAG.getIntPtrConstant(0, dl));
18007   }
18008 
18009   if (VT.getSizeInBits() == 64) {
18010     // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
18011     // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
18012     //        to match extract_elt for f64.
18013     if (IdxVal == 0)
18014       return Op;
18015 
18016     // UNPCKHPD the element to the lowest double word, then movsd.
18017     // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
18018     // to a f64mem, the whole operation is folded into a single MOVHPDmr.
18019     int Mask[2] = { 1, -1 };
18020     Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18021     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18022                        DAG.getIntPtrConstant(0, dl));
18023   }
18024 
18025   return SDValue();
18026 }
18027 
18028 /// Insert one bit to mask vector, like v16i1 or v8i1.
18029 /// AVX-512 feature.
18030 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18031                                      const X86Subtarget &Subtarget) {
18032   SDLoc dl(Op);
18033   SDValue Vec = Op.getOperand(0);
18034   SDValue Elt = Op.getOperand(1);
18035   SDValue Idx = Op.getOperand(2);
18036   MVT VecVT = Vec.getSimpleValueType();
18037 
18038   if (!isa<ConstantSDNode>(Idx)) {
18039     // Non constant index. Extend source and destination,
18040     // insert element and then truncate the result.
18041     unsigned NumElts = VecVT.getVectorNumElements();
18042     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
18043     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
18044     SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18045       DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
18046       DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
18047     return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
18048   }
18049 
18050   // Copy into a k-register, extract to v1i1 and insert_subvector.
18051   SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18052   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx);
18053 }
18054 
18055 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
18056                                                   SelectionDAG &DAG) const {
18057   MVT VT = Op.getSimpleValueType();
18058   MVT EltVT = VT.getVectorElementType();
18059   unsigned NumElts = VT.getVectorNumElements();
18060   unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
18061 
18062   if (EltVT == MVT::i1)
18063     return InsertBitToMaskVector(Op, DAG, Subtarget);
18064 
18065   SDLoc dl(Op);
18066   SDValue N0 = Op.getOperand(0);
18067   SDValue N1 = Op.getOperand(1);
18068   SDValue N2 = Op.getOperand(2);
18069   auto *N2C = dyn_cast<ConstantSDNode>(N2);
18070 
18071   if (EltVT == MVT::bf16) {
18072     MVT IVT = VT.changeVectorElementTypeToInteger();
18073     SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
18074                               DAG.getBitcast(IVT, N0),
18075                               DAG.getBitcast(MVT::i16, N1), N2);
18076     return DAG.getBitcast(VT, Res);
18077   }
18078 
18079   if (!N2C) {
18080     // Variable insertion indices, usually we're better off spilling to stack,
18081     // but AVX512 can use a variable compare+select by comparing against all
18082     // possible vector indices, and FP insertion has less gpr->simd traffic.
18083     if (!(Subtarget.hasBWI() ||
18084           (Subtarget.hasAVX512() && EltSizeInBits >= 32) ||
18085           (Subtarget.hasSSE41() && (EltVT == MVT::f32 || EltVT == MVT::f64))))
18086       return SDValue();
18087 
18088     MVT IdxSVT = MVT::getIntegerVT(EltSizeInBits);
18089     MVT IdxVT = MVT::getVectorVT(IdxSVT, NumElts);
18090     if (!isTypeLegal(IdxSVT) || !isTypeLegal(IdxVT))
18091       return SDValue();
18092 
18093     SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT);
18094     SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt);
18095     SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1);
18096 
18097     SmallVector<SDValue, 16> RawIndices;
18098     for (unsigned I = 0; I != NumElts; ++I)
18099       RawIndices.push_back(DAG.getConstant(I, dl, IdxSVT));
18100     SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices);
18101 
18102     // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
18103     return DAG.getSelectCC(dl, IdxSplat, Indices, EltSplat, N0,
18104                            ISD::CondCode::SETEQ);
18105   }
18106 
18107   if (N2C->getAPIntValue().uge(NumElts))
18108     return SDValue();
18109   uint64_t IdxVal = N2C->getZExtValue();
18110 
18111   bool IsZeroElt = X86::isZeroNode(N1);
18112   bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1);
18113 
18114   if (IsZeroElt || IsAllOnesElt) {
18115     // Lower insertion of v16i8/v32i8/v64i16 -1 elts as an 'OR' blend.
18116     // We don't deal with i8 0 since it appears to be handled elsewhere.
18117     if (IsAllOnesElt &&
18118         ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) ||
18119          ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) {
18120       SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType());
18121       SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType());
18122       SmallVector<SDValue, 8> CstVectorElts(NumElts, ZeroCst);
18123       CstVectorElts[IdxVal] = OnesCst;
18124       SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts);
18125       return DAG.getNode(ISD::OR, dl, VT, N0, CstVector);
18126     }
18127     // See if we can do this more efficiently with a blend shuffle with a
18128     // rematerializable vector.
18129     if (Subtarget.hasSSE41() &&
18130         (EltSizeInBits >= 16 || (IsZeroElt && !VT.is128BitVector()))) {
18131       SmallVector<int, 8> BlendMask;
18132       for (unsigned i = 0; i != NumElts; ++i)
18133         BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18134       SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18135                                     : getOnesVector(VT, DAG, dl);
18136       return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);
18137     }
18138   }
18139 
18140   // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
18141   // into that, and then insert the subvector back into the result.
18142   if (VT.is256BitVector() || VT.is512BitVector()) {
18143     // With a 256-bit vector, we can insert into the zero element efficiently
18144     // using a blend if we have AVX or AVX2 and the right data type.
18145     if (VT.is256BitVector() && IdxVal == 0) {
18146       // TODO: It is worthwhile to cast integer to floating point and back
18147       // and incur a domain crossing penalty if that's what we'll end up
18148       // doing anyway after extracting to a 128-bit vector.
18149       if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
18150           (Subtarget.hasAVX2() && (EltVT == MVT::i32 || EltVT == MVT::i64))) {
18151         SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18152         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
18153                            DAG.getTargetConstant(1, dl, MVT::i8));
18154       }
18155     }
18156 
18157     unsigned NumEltsIn128 = 128 / EltSizeInBits;
18158     assert(isPowerOf2_32(NumEltsIn128) &&
18159            "Vectors will always have power-of-two number of elements.");
18160 
18161     // If we are not inserting into the low 128-bit vector chunk,
18162     // then prefer the broadcast+blend sequence.
18163     // FIXME: relax the profitability check iff all N1 uses are insertions.
18164     if (IdxVal >= NumEltsIn128 &&
18165         ((Subtarget.hasAVX2() && EltSizeInBits != 8) ||
18166          (Subtarget.hasAVX() && (EltSizeInBits >= 32) &&
18167           X86::mayFoldLoad(N1, Subtarget)))) {
18168       SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1);
18169       SmallVector<int, 8> BlendMask;
18170       for (unsigned i = 0; i != NumElts; ++i)
18171         BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18172       return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask);
18173     }
18174 
18175     // Get the desired 128-bit vector chunk.
18176     SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18177 
18178     // Insert the element into the desired chunk.
18179     // Since NumEltsIn128 is a power of 2 we can use mask instead of modulo.
18180     unsigned IdxIn128 = IdxVal & (NumEltsIn128 - 1);
18181 
18182     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
18183                     DAG.getIntPtrConstant(IdxIn128, dl));
18184 
18185     // Insert the changed part back into the bigger vector
18186     return insert128BitVector(N0, V, IdxVal, DAG, dl);
18187   }
18188   assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");
18189 
18190   // This will be just movw/movd/movq/movsh/movss/movsd.
18191   if (IdxVal == 0 && ISD::isBuildVectorAllZeros(N0.getNode())) {
18192     if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||
18193         EltVT == MVT::f16 || EltVT == MVT::i64) {
18194       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18195       return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18196     }
18197 
18198     // We can't directly insert an i8 or i16 into a vector, so zero extend
18199     // it to i32 first.
18200     if (EltVT == MVT::i16 || EltVT == MVT::i8) {
18201       N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1);
18202       MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
18203       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1);
18204       N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18205       return DAG.getBitcast(VT, N1);
18206     }
18207   }
18208 
18209   // Transform it so it match pinsr{b,w} which expects a GR32 as its second
18210   // argument. SSE41 required for pinsrb.
18211   if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {
18212     unsigned Opc;
18213     if (VT == MVT::v8i16) {
18214       assert(Subtarget.hasSSE2() && "SSE2 required for PINSRW");
18215       Opc = X86ISD::PINSRW;
18216     } else {
18217       assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector");
18218       assert(Subtarget.hasSSE41() && "SSE41 required for PINSRB");
18219       Opc = X86ISD::PINSRB;
18220     }
18221 
18222     assert(N1.getValueType() != MVT::i32 && "Unexpected VT");
18223     N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
18224     N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8);
18225     return DAG.getNode(Opc, dl, VT, N0, N1, N2);
18226   }
18227 
18228   if (Subtarget.hasSSE41()) {
18229     if (EltVT == MVT::f32) {
18230       // Bits [7:6] of the constant are the source select. This will always be
18231       //   zero here. The DAG Combiner may combine an extract_elt index into
18232       //   these bits. For example (insert (extract, 3), 2) could be matched by
18233       //   putting the '3' into bits [7:6] of X86ISD::INSERTPS.
18234       // Bits [5:4] of the constant are the destination select. This is the
18235       //   value of the incoming immediate.
18236       // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
18237       //   combine either bitwise AND or insert of float 0.0 to set these bits.
18238 
18239       bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize();
18240       if (IdxVal == 0 && (!MinSize || !X86::mayFoldLoad(N1, Subtarget))) {
18241         // If this is an insertion of 32-bits into the low 32-bits of
18242         // a vector, we prefer to generate a blend with immediate rather
18243         // than an insertps. Blends are simpler operations in hardware and so
18244         // will always have equal or better performance than insertps.
18245         // But if optimizing for size and there's a load folding opportunity,
18246         // generate insertps because blendps does not have a 32-bit memory
18247         // operand form.
18248         N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18249         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
18250                            DAG.getTargetConstant(1, dl, MVT::i8));
18251       }
18252       // Create this as a scalar to vector..
18253       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18254       return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
18255                          DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
18256     }
18257 
18258     // PINSR* works with constant index.
18259     if (EltVT == MVT::i32 || EltVT == MVT::i64)
18260       return Op;
18261   }
18262 
18263   return SDValue();
18264 }
18265 
18266 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
18267                                      SelectionDAG &DAG) {
18268   SDLoc dl(Op);
18269   MVT OpVT = Op.getSimpleValueType();
18270 
18271   // It's always cheaper to replace a xor+movd with xorps and simplifies further
18272   // combines.
18273   if (X86::isZeroNode(Op.getOperand(0)))
18274     return getZeroVector(OpVT, Subtarget, DAG, dl);
18275 
18276   // If this is a 256-bit vector result, first insert into a 128-bit
18277   // vector and then insert into the 256-bit vector.
18278   if (!OpVT.is128BitVector()) {
18279     // Insert into a 128-bit vector.
18280     unsigned SizeFactor = OpVT.getSizeInBits() / 128;
18281     MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
18282                                  OpVT.getVectorNumElements() / SizeFactor);
18283 
18284     Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
18285 
18286     // Insert the 128-bit vector.
18287     return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
18288   }
18289   assert(OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 &&
18290          "Expected an SSE type!");
18291 
18292   // Pass through a v4i32 or V8i16 SCALAR_TO_VECTOR as that's what we use in
18293   // tblgen.
18294   if (OpVT == MVT::v4i32 || (OpVT == MVT::v8i16 && Subtarget.hasFP16()))
18295     return Op;
18296 
18297   SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18298   return DAG.getBitcast(
18299       OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
18300 }
18301 
18302 // Lower a node with an INSERT_SUBVECTOR opcode.  This may result in a
18303 // simple superregister reference or explicit instructions to insert
18304 // the upper bits of a vector.
18305 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18306                                      SelectionDAG &DAG) {
18307   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1);
18308 
18309   return insert1BitVector(Op, DAG, Subtarget);
18310 }
18311 
18312 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18313                                       SelectionDAG &DAG) {
18314   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&
18315          "Only vXi1 extract_subvectors need custom lowering");
18316 
18317   SDLoc dl(Op);
18318   SDValue Vec = Op.getOperand(0);
18319   uint64_t IdxVal = Op.getConstantOperandVal(1);
18320 
18321   if (IdxVal == 0) // the operation is legal
18322     return Op;
18323 
18324   // Extend to natively supported kshift.
18325   Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18326 
18327   // Shift to the LSB.
18328   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
18329                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18330 
18331   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
18332                      DAG.getIntPtrConstant(0, dl));
18333 }
18334 
18335 // Returns the appropriate wrapper opcode for a global reference.
18336 unsigned X86TargetLowering::getGlobalWrapperKind(
18337     const GlobalValue *GV, const unsigned char OpFlags) const {
18338   // References to absolute symbols are never PC-relative.
18339   if (GV && GV->isAbsoluteSymbolRef())
18340     return X86ISD::Wrapper;
18341 
18342   // The following OpFlags under RIP-rel PIC use RIP.
18343   if (Subtarget.isPICStyleRIPRel() &&
18344       (OpFlags == X86II::MO_NO_FLAG || OpFlags == X86II::MO_COFFSTUB ||
18345        OpFlags == X86II::MO_DLLIMPORT))
18346     return X86ISD::WrapperRIP;
18347 
18348   // GOTPCREL references must always use RIP.
18349   if (OpFlags == X86II::MO_GOTPCREL || OpFlags == X86II::MO_GOTPCREL_NORELAX)
18350     return X86ISD::WrapperRIP;
18351 
18352   return X86ISD::Wrapper;
18353 }
18354 
18355 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
18356 // their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
18357 // one of the above mentioned nodes. It has to be wrapped because otherwise
18358 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
18359 // be used to form addressing mode. These wrapped nodes will be selected
18360 // into MOV32ri.
18361 SDValue
18362 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18363   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
18364 
18365   // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18366   // global base reg.
18367   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18368 
18369   auto PtrVT = getPointerTy(DAG.getDataLayout());
18370   SDValue Result = DAG.getTargetConstantPool(
18371       CP->getConstVal(), PtrVT, CP->getAlign(), CP->getOffset(), OpFlag);
18372   SDLoc DL(CP);
18373   Result =
18374       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18375   // With PIC, the address is actually $g + Offset.
18376   if (OpFlag) {
18377     Result =
18378         DAG.getNode(ISD::ADD, DL, PtrVT,
18379                     DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18380   }
18381 
18382   return Result;
18383 }
18384 
18385 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18386   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
18387 
18388   // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18389   // global base reg.
18390   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18391 
18392   auto PtrVT = getPointerTy(DAG.getDataLayout());
18393   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18394   SDLoc DL(JT);
18395   Result =
18396       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18397 
18398   // With PIC, the address is actually $g + Offset.
18399   if (OpFlag)
18400     Result =
18401         DAG.getNode(ISD::ADD, DL, PtrVT,
18402                     DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18403 
18404   return Result;
18405 }
18406 
18407 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18408                                                SelectionDAG &DAG) const {
18409   return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18410 }
18411 
18412 SDValue
18413 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18414   // Create the TargetBlockAddressAddress node.
18415   unsigned char OpFlags =
18416     Subtarget.classifyBlockAddressReference();
18417   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
18418   int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
18419   SDLoc dl(Op);
18420   auto PtrVT = getPointerTy(DAG.getDataLayout());
18421   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18422   Result =
18423       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
18424 
18425   // With PIC, the address is actually $g + Offset.
18426   if (isGlobalRelativeToPICBase(OpFlags)) {
18427     Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18428                          DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18429   }
18430 
18431   return Result;
18432 }
18433 
18434 /// Creates target global address or external symbol nodes for calls or
18435 /// other uses.
18436 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18437                                                  bool ForCall) const {
18438   // Unpack the global address or external symbol.
18439   const SDLoc &dl = SDLoc(Op);
18440   const GlobalValue *GV = nullptr;
18441   int64_t Offset = 0;
18442   const char *ExternalSym = nullptr;
18443   if (const auto *G = dyn_cast<GlobalAddressSDNode>(Op)) {
18444     GV = G->getGlobal();
18445     Offset = G->getOffset();
18446   } else {
18447     const auto *ES = cast<ExternalSymbolSDNode>(Op);
18448     ExternalSym = ES->getSymbol();
18449   }
18450 
18451   // Calculate some flags for address lowering.
18452   const Module &Mod = *DAG.getMachineFunction().getFunction().getParent();
18453   unsigned char OpFlags;
18454   if (ForCall)
18455     OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod);
18456   else
18457     OpFlags = Subtarget.classifyGlobalReference(GV, Mod);
18458   bool HasPICReg = isGlobalRelativeToPICBase(OpFlags);
18459   bool NeedsLoad = isGlobalStubReference(OpFlags);
18460 
18461   CodeModel::Model M = DAG.getTarget().getCodeModel();
18462   auto PtrVT = getPointerTy(DAG.getDataLayout());
18463   SDValue Result;
18464 
18465   if (GV) {
18466     // Create a target global address if this is a global. If possible, fold the
18467     // offset into the global address reference. Otherwise, ADD it on later.
18468     // Suppress the folding if Offset is negative: movl foo-1, %eax is not
18469     // allowed because if the address of foo is 0, the ELF R_X86_64_32
18470     // relocation will compute to a negative value, which is invalid.
18471     int64_t GlobalOffset = 0;
18472     if (OpFlags == X86II::MO_NO_FLAG && Offset >= 0 &&
18473         X86::isOffsetSuitableForCodeModel(Offset, M, true)) {
18474       std::swap(GlobalOffset, Offset);
18475     }
18476     Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags);
18477   } else {
18478     // If this is not a global address, this must be an external symbol.
18479     Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags);
18480   }
18481 
18482   // If this is a direct call, avoid the wrapper if we don't need to do any
18483   // loads or adds. This allows SDAG ISel to match direct calls.
18484   if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0)
18485     return Result;
18486 
18487   Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
18488 
18489   // With PIC, the address is actually $g + Offset.
18490   if (HasPICReg) {
18491     Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18492                          DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18493   }
18494 
18495   // For globals that require a load from a stub to get the address, emit the
18496   // load.
18497   if (NeedsLoad)
18498     Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
18499                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18500 
18501   // If there was a non-zero offset that we didn't fold, create an explicit
18502   // addition for it.
18503   if (Offset != 0)
18504     Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
18505                          DAG.getConstant(Offset, dl, PtrVT));
18506 
18507   return Result;
18508 }
18509 
18510 SDValue
18511 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
18512   return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18513 }
18514 
18515 static SDValue
18516 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
18517            SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg,
18518            unsigned char OperandFlags, bool LocalDynamic = false) {
18519   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18520   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18521   SDLoc dl(GA);
18522   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18523                                            GA->getValueType(0),
18524                                            GA->getOffset(),
18525                                            OperandFlags);
18526 
18527   X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
18528                                            : X86ISD::TLSADDR;
18529 
18530   if (InGlue) {
18531     SDValue Ops[] = { Chain,  TGA, *InGlue };
18532     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18533   } else {
18534     SDValue Ops[]  = { Chain, TGA };
18535     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18536   }
18537 
18538   // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
18539   MFI.setAdjustsStack(true);
18540   MFI.setHasCalls(true);
18541 
18542   SDValue Glue = Chain.getValue(1);
18543   return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
18544 }
18545 
18546 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
18547 static SDValue
18548 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18549                                 const EVT PtrVT) {
18550   SDValue InGlue;
18551   SDLoc dl(GA);  // ? function entry point might be better
18552   SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18553                                    DAG.getNode(X86ISD::GlobalBaseReg,
18554                                                SDLoc(), PtrVT), InGlue);
18555   InGlue = Chain.getValue(1);
18556 
18557   return GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX, X86II::MO_TLSGD);
18558 }
18559 
18560 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
18561 static SDValue
18562 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18563                                 const EVT PtrVT) {
18564   return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18565                     X86::RAX, X86II::MO_TLSGD);
18566 }
18567 
18568 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
18569 static SDValue
18570 LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18571                                  const EVT PtrVT) {
18572   return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18573                     X86::EAX, X86II::MO_TLSGD);
18574 }
18575 
18576 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
18577                                            SelectionDAG &DAG, const EVT PtrVT,
18578                                            bool Is64Bit, bool Is64BitLP64) {
18579   SDLoc dl(GA);
18580 
18581   // Get the start address of the TLS block for this module.
18582   X86MachineFunctionInfo *MFI = DAG.getMachineFunction()
18583       .getInfo<X86MachineFunctionInfo>();
18584   MFI->incNumLocalDynamicTLSAccesses();
18585 
18586   SDValue Base;
18587   if (Is64Bit) {
18588     unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX;
18589     Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg,
18590                       X86II::MO_TLSLD, /*LocalDynamic=*/true);
18591   } else {
18592     SDValue InGlue;
18593     SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18594         DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InGlue);
18595     InGlue = Chain.getValue(1);
18596     Base = GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX,
18597                       X86II::MO_TLSLDM, /*LocalDynamic=*/true);
18598   }
18599 
18600   // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
18601   // of Base.
18602 
18603   // Build x@dtpoff.
18604   unsigned char OperandFlags = X86II::MO_DTPOFF;
18605   unsigned WrapperKind = X86ISD::Wrapper;
18606   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18607                                            GA->getValueType(0),
18608                                            GA->getOffset(), OperandFlags);
18609   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18610 
18611   // Add x@dtpoff with the base.
18612   return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
18613 }
18614 
18615 // Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
18616 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18617                                    const EVT PtrVT, TLSModel::Model model,
18618                                    bool is64Bit, bool isPIC) {
18619   SDLoc dl(GA);
18620 
18621   // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
18622   Value *Ptr = Constant::getNullValue(
18623       PointerType::get(*DAG.getContext(), is64Bit ? 257 : 256));
18624 
18625   SDValue ThreadPointer =
18626       DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
18627                   MachinePointerInfo(Ptr));
18628 
18629   unsigned char OperandFlags = 0;
18630   // Most TLS accesses are not RIP relative, even on x86-64.  One exception is
18631   // initialexec.
18632   unsigned WrapperKind = X86ISD::Wrapper;
18633   if (model == TLSModel::LocalExec) {
18634     OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
18635   } else if (model == TLSModel::InitialExec) {
18636     if (is64Bit) {
18637       OperandFlags = X86II::MO_GOTTPOFF;
18638       WrapperKind = X86ISD::WrapperRIP;
18639     } else {
18640       OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
18641     }
18642   } else {
18643     llvm_unreachable("Unexpected model");
18644   }
18645 
18646   // emit "addl x@ntpoff,%eax" (local exec)
18647   // or "addl x@indntpoff,%eax" (initial exec)
18648   // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
18649   SDValue TGA =
18650       DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
18651                                  GA->getOffset(), OperandFlags);
18652   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18653 
18654   if (model == TLSModel::InitialExec) {
18655     if (isPIC && !is64Bit) {
18656       Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
18657                            DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18658                            Offset);
18659     }
18660 
18661     Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
18662                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18663   }
18664 
18665   // The address of the thread local variable is the add of the thread
18666   // pointer with the offset of the variable.
18667   return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
18668 }
18669 
18670 SDValue
18671 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18672 
18673   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
18674 
18675   if (DAG.getTarget().useEmulatedTLS())
18676     return LowerToTLSEmulatedModel(GA, DAG);
18677 
18678   const GlobalValue *GV = GA->getGlobal();
18679   auto PtrVT = getPointerTy(DAG.getDataLayout());
18680   bool PositionIndependent = isPositionIndependent();
18681 
18682   if (Subtarget.isTargetELF()) {
18683     TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
18684     switch (model) {
18685       case TLSModel::GeneralDynamic:
18686         if (Subtarget.is64Bit()) {
18687           if (Subtarget.isTarget64BitLP64())
18688             return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
18689           return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
18690         }
18691         return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
18692       case TLSModel::LocalDynamic:
18693         return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
18694                                            Subtarget.isTarget64BitLP64());
18695       case TLSModel::InitialExec:
18696       case TLSModel::LocalExec:
18697         return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
18698                                    PositionIndependent);
18699     }
18700     llvm_unreachable("Unknown TLS model.");
18701   }
18702 
18703   if (Subtarget.isTargetDarwin()) {
18704     // Darwin only has one model of TLS.  Lower to that.
18705     unsigned char OpFlag = 0;
18706     unsigned WrapperKind = Subtarget.isPICStyleRIPRel() ?
18707                            X86ISD::WrapperRIP : X86ISD::Wrapper;
18708 
18709     // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18710     // global base reg.
18711     bool PIC32 = PositionIndependent && !Subtarget.is64Bit();
18712     if (PIC32)
18713       OpFlag = X86II::MO_TLVP_PIC_BASE;
18714     else
18715       OpFlag = X86II::MO_TLVP;
18716     SDLoc DL(Op);
18717     SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18718                                                 GA->getValueType(0),
18719                                                 GA->getOffset(), OpFlag);
18720     SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18721 
18722     // With PIC32, the address is actually $g + Offset.
18723     if (PIC32)
18724       Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
18725                            DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18726                            Offset);
18727 
18728     // Lowering the machine isd will make sure everything is in the right
18729     // location.
18730     SDValue Chain = DAG.getEntryNode();
18731     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18732     Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
18733     SDValue Args[] = { Chain, Offset };
18734     Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
18735     Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
18736 
18737     // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
18738     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18739     MFI.setAdjustsStack(true);
18740 
18741     // And our return value (tls address) is in the standard call return value
18742     // location.
18743     unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
18744     return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
18745   }
18746 
18747   if (Subtarget.isOSWindows()) {
18748     // Just use the implicit TLS architecture
18749     // Need to generate something similar to:
18750     //   mov     rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
18751     //                                  ; from TEB
18752     //   mov     ecx, dword [rel _tls_index]: Load index (from C runtime)
18753     //   mov     rcx, qword [rdx+rcx*8]
18754     //   mov     eax, .tls$:tlsvar
18755     //   [rax+rcx] contains the address
18756     // Windows 64bit: gs:0x58
18757     // Windows 32bit: fs:__tls_array
18758 
18759     SDLoc dl(GA);
18760     SDValue Chain = DAG.getEntryNode();
18761 
18762     // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
18763     // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
18764     // use its literal value of 0x2C.
18765     Value *Ptr = Constant::getNullValue(
18766         Subtarget.is64Bit() ? PointerType::get(*DAG.getContext(), 256)
18767                             : PointerType::get(*DAG.getContext(), 257));
18768 
18769     SDValue TlsArray = Subtarget.is64Bit()
18770                            ? DAG.getIntPtrConstant(0x58, dl)
18771                            : (Subtarget.isTargetWindowsGNU()
18772                                   ? DAG.getIntPtrConstant(0x2C, dl)
18773                                   : DAG.getExternalSymbol("_tls_array", PtrVT));
18774 
18775     SDValue ThreadPointer =
18776         DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr));
18777 
18778     SDValue res;
18779     if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) {
18780       res = ThreadPointer;
18781     } else {
18782       // Load the _tls_index variable
18783       SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
18784       if (Subtarget.is64Bit())
18785         IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
18786                              MachinePointerInfo(), MVT::i32);
18787       else
18788         IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo());
18789 
18790       const DataLayout &DL = DAG.getDataLayout();
18791       SDValue Scale =
18792           DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
18793       IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
18794 
18795       res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
18796     }
18797 
18798     res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
18799 
18800     // Get the offset of start of .tls section
18801     SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18802                                              GA->getValueType(0),
18803                                              GA->getOffset(), X86II::MO_SECREL);
18804     SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
18805 
18806     // The address of the thread local variable is the add of the thread
18807     // pointer with the offset of the variable.
18808     return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
18809   }
18810 
18811   llvm_unreachable("TLS not implemented for this target.");
18812 }
18813 
18814 /// Lower SRA_PARTS and friends, which return two i32 values
18815 /// and take a 2 x i32 value to shift plus a shift amount.
18816 /// TODO: Can this be moved to general expansion code?
18817 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
18818   SDValue Lo, Hi;
18819   DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG);
18820   return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
18821 }
18822 
18823 // Try to use a packed vector operation to handle i64 on 32-bit targets when
18824 // AVX512DQ is enabled.
18825 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
18826                                         const X86Subtarget &Subtarget) {
18827   assert((Op.getOpcode() == ISD::SINT_TO_FP ||
18828           Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
18829           Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
18830           Op.getOpcode() == ISD::UINT_TO_FP) &&
18831          "Unexpected opcode!");
18832   bool IsStrict = Op->isStrictFPOpcode();
18833   unsigned OpNo = IsStrict ? 1 : 0;
18834   SDValue Src = Op.getOperand(OpNo);
18835   MVT SrcVT = Src.getSimpleValueType();
18836   MVT VT = Op.getSimpleValueType();
18837 
18838    if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
18839        (VT != MVT::f32 && VT != MVT::f64))
18840     return SDValue();
18841 
18842   // Pack the i64 into a vector, do the operation and extract.
18843 
18844   // Using 256-bit to ensure result is 128-bits for f32 case.
18845   unsigned NumElts = Subtarget.hasVLX() ? 4 : 8;
18846   MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts);
18847   MVT VecVT = MVT::getVectorVT(VT, NumElts);
18848 
18849   SDLoc dl(Op);
18850   SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
18851   if (IsStrict) {
18852     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
18853                                  {Op.getOperand(0), InVec});
18854     SDValue Chain = CvtVec.getValue(1);
18855     SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18856                                 DAG.getIntPtrConstant(0, dl));
18857     return DAG.getMergeValues({Value, Chain}, dl);
18858   }
18859 
18860   SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
18861 
18862   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18863                      DAG.getIntPtrConstant(0, dl));
18864 }
18865 
18866 // Try to use a packed vector operation to handle i64 on 32-bit targets.
18867 static SDValue LowerI64IntToFP16(SDValue Op, SelectionDAG &DAG,
18868                                  const X86Subtarget &Subtarget) {
18869   assert((Op.getOpcode() == ISD::SINT_TO_FP ||
18870           Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
18871           Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
18872           Op.getOpcode() == ISD::UINT_TO_FP) &&
18873          "Unexpected opcode!");
18874   bool IsStrict = Op->isStrictFPOpcode();
18875   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
18876   MVT SrcVT = Src.getSimpleValueType();
18877   MVT VT = Op.getSimpleValueType();
18878 
18879   if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16)
18880     return SDValue();
18881 
18882   // Pack the i64 into a vector, do the operation and extract.
18883 
18884   assert(Subtarget.hasFP16() && "Expected FP16");
18885 
18886   SDLoc dl(Op);
18887   SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
18888   if (IsStrict) {
18889     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},
18890                                  {Op.getOperand(0), InVec});
18891     SDValue Chain = CvtVec.getValue(1);
18892     SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18893                                 DAG.getIntPtrConstant(0, dl));
18894     return DAG.getMergeValues({Value, Chain}, dl);
18895   }
18896 
18897   SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);
18898 
18899   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18900                      DAG.getIntPtrConstant(0, dl));
18901 }
18902 
18903 static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
18904                           const X86Subtarget &Subtarget) {
18905   switch (Opcode) {
18906     case ISD::SINT_TO_FP:
18907       // TODO: Handle wider types with AVX/AVX512.
18908       if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
18909         return false;
18910       // CVTDQ2PS or (V)CVTDQ2PD
18911       return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
18912 
18913     case ISD::UINT_TO_FP:
18914       // TODO: Handle wider types and i64 elements.
18915       if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
18916         return false;
18917       // VCVTUDQ2PS or VCVTUDQ2PD
18918       return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
18919 
18920     default:
18921       return false;
18922   }
18923 }
18924 
18925 /// Given a scalar cast operation that is extracted from a vector, try to
18926 /// vectorize the cast op followed by extraction. This will avoid an expensive
18927 /// round-trip between XMM and GPR.
18928 static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
18929                                       const X86Subtarget &Subtarget) {
18930   // TODO: This could be enhanced to handle smaller integer types by peeking
18931   // through an extend.
18932   SDValue Extract = Cast.getOperand(0);
18933   MVT DestVT = Cast.getSimpleValueType();
18934   if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18935       !isa<ConstantSDNode>(Extract.getOperand(1)))
18936     return SDValue();
18937 
18938   // See if we have a 128-bit vector cast op for this type of cast.
18939   SDValue VecOp = Extract.getOperand(0);
18940   MVT FromVT = VecOp.getSimpleValueType();
18941   unsigned NumEltsInXMM = 128 / FromVT.getScalarSizeInBits();
18942   MVT Vec128VT = MVT::getVectorVT(FromVT.getScalarType(), NumEltsInXMM);
18943   MVT ToVT = MVT::getVectorVT(DestVT, NumEltsInXMM);
18944   if (!useVectorCast(Cast.getOpcode(), Vec128VT, ToVT, Subtarget))
18945     return SDValue();
18946 
18947   // If we are extracting from a non-zero element, first shuffle the source
18948   // vector to allow extracting from element zero.
18949   SDLoc DL(Cast);
18950   if (!isNullConstant(Extract.getOperand(1))) {
18951     SmallVector<int, 16> Mask(FromVT.getVectorNumElements(), -1);
18952     Mask[0] = Extract.getConstantOperandVal(1);
18953     VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
18954   }
18955   // If the source vector is wider than 128-bits, extract the low part. Do not
18956   // create an unnecessarily wide vector cast op.
18957   if (FromVT != Vec128VT)
18958     VecOp = extract128BitVector(VecOp, 0, DAG, DL);
18959 
18960   // cast (extelt V, 0) --> extelt (cast (extract_subv V)), 0
18961   // cast (extelt V, C) --> extelt (cast (extract_subv (shuffle V, [C...]))), 0
18962   SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
18963   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
18964                      DAG.getIntPtrConstant(0, DL));
18965 }
18966 
18967 /// Given a scalar cast to FP with a cast to integer operand (almost an ftrunc),
18968 /// try to vectorize the cast ops. This will avoid an expensive round-trip
18969 /// between XMM and GPR.
18970 static SDValue lowerFPToIntToFP(SDValue CastToFP, SelectionDAG &DAG,
18971                                 const X86Subtarget &Subtarget) {
18972   // TODO: Allow FP_TO_UINT.
18973   SDValue CastToInt = CastToFP.getOperand(0);
18974   MVT VT = CastToFP.getSimpleValueType();
18975   if (CastToInt.getOpcode() != ISD::FP_TO_SINT || VT.isVector())
18976     return SDValue();
18977 
18978   MVT IntVT = CastToInt.getSimpleValueType();
18979   SDValue X = CastToInt.getOperand(0);
18980   MVT SrcVT = X.getSimpleValueType();
18981   if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
18982     return SDValue();
18983 
18984   // See if we have 128-bit vector cast instructions for this type of cast.
18985   // We need cvttps2dq/cvttpd2dq and cvtdq2ps/cvtdq2pd.
18986   if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) ||
18987       IntVT != MVT::i32)
18988     return SDValue();
18989 
18990   unsigned SrcSize = SrcVT.getSizeInBits();
18991   unsigned IntSize = IntVT.getSizeInBits();
18992   unsigned VTSize = VT.getSizeInBits();
18993   MVT VecSrcVT = MVT::getVectorVT(SrcVT, 128 / SrcSize);
18994   MVT VecIntVT = MVT::getVectorVT(IntVT, 128 / IntSize);
18995   MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize);
18996 
18997   // We need target-specific opcodes if this is v2f64 -> v4i32 -> v2f64.
18998   unsigned ToIntOpcode =
18999       SrcSize != IntSize ? X86ISD::CVTTP2SI : (unsigned)ISD::FP_TO_SINT;
19000   unsigned ToFPOpcode =
19001       IntSize != VTSize ? X86ISD::CVTSI2P : (unsigned)ISD::SINT_TO_FP;
19002 
19003   // sint_to_fp (fp_to_sint X) --> extelt (sint_to_fp (fp_to_sint (s2v X))), 0
19004   //
19005   // We are not defining the high elements (for example, zero them) because
19006   // that could nullify any performance advantage that we hoped to gain from
19007   // this vector op hack. We do not expect any adverse effects (like denorm
19008   // penalties) with cast ops.
19009   SDLoc DL(CastToFP);
19010   SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
19011   SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19012   SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19013   SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19014   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx);
19015 }
19016 
19017 static SDValue lowerINT_TO_FP_vXi64(SDValue Op, SelectionDAG &DAG,
19018                                     const X86Subtarget &Subtarget) {
19019   SDLoc DL(Op);
19020   bool IsStrict = Op->isStrictFPOpcode();
19021   MVT VT = Op->getSimpleValueType(0);
19022   SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
19023 
19024   if (Subtarget.hasDQI()) {
19025     assert(!Subtarget.hasVLX() && "Unexpected features");
19026 
19027     assert((Src.getSimpleValueType() == MVT::v2i64 ||
19028             Src.getSimpleValueType() == MVT::v4i64) &&
19029            "Unsupported custom type");
19030 
19031     // With AVX512DQ, but not VLX we need to widen to get a 512-bit result type.
19032     assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
19033            "Unexpected VT!");
19034     MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
19035 
19036     // Need to concat with zero vector for strict fp to avoid spurious
19037     // exceptions.
19038     SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19039                            : DAG.getUNDEF(MVT::v8i64);
19040     Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
19041                       DAG.getIntPtrConstant(0, DL));
19042     SDValue Res, Chain;
19043     if (IsStrict) {
19044       Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
19045                         {Op->getOperand(0), Src});
19046       Chain = Res.getValue(1);
19047     } else {
19048       Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
19049     }
19050 
19051     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19052                       DAG.getIntPtrConstant(0, DL));
19053 
19054     if (IsStrict)
19055       return DAG.getMergeValues({Res, Chain}, DL);
19056     return Res;
19057   }
19058 
19059   bool IsSigned = Op->getOpcode() == ISD::SINT_TO_FP ||
19060                   Op->getOpcode() == ISD::STRICT_SINT_TO_FP;
19061   if (VT != MVT::v4f32 || IsSigned)
19062     return SDValue();
19063 
19064   SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19065   SDValue One  = DAG.getConstant(1, DL, MVT::v4i64);
19066   SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19067                              DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
19068                              DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
19069   SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19070   SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19071   SmallVector<SDValue, 4> SignCvts(4);
19072   SmallVector<SDValue, 4> Chains(4);
19073   for (int i = 0; i != 4; ++i) {
19074     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19075                               DAG.getIntPtrConstant(i, DL));
19076     if (IsStrict) {
19077       SignCvts[i] =
19078           DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
19079                       {Op.getOperand(0), Elt});
19080       Chains[i] = SignCvts[i].getValue(1);
19081     } else {
19082       SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);
19083     }
19084   }
19085   SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19086 
19087   SDValue Slow, Chain;
19088   if (IsStrict) {
19089     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19090     Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
19091                        {Chain, SignCvt, SignCvt});
19092     Chain = Slow.getValue(1);
19093   } else {
19094     Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
19095   }
19096 
19097   IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
19098   SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19099 
19100   if (IsStrict)
19101     return DAG.getMergeValues({Cvt, Chain}, DL);
19102 
19103   return Cvt;
19104 }
19105 
19106 static SDValue promoteXINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
19107   bool IsStrict = Op->isStrictFPOpcode();
19108   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19109   SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19110   MVT VT = Op.getSimpleValueType();
19111   MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
19112   SDLoc dl(Op);
19113 
19114   SDValue Rnd = DAG.getIntPtrConstant(0, dl);
19115   if (IsStrict)
19116     return DAG.getNode(
19117         ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},
19118         {Chain,
19119          DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}),
19120          Rnd});
19121   return DAG.getNode(ISD::FP_ROUND, dl, VT,
19122                      DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd);
19123 }
19124 
19125 static bool isLegalConversion(MVT VT, bool IsSigned,
19126                               const X86Subtarget &Subtarget) {
19127   if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
19128     return true;
19129   if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)
19130     return true;
19131   if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32))
19132     return true;
19133   if (Subtarget.useAVX512Regs()) {
19134     if (VT == MVT::v16i32)
19135       return true;
19136     if (VT == MVT::v8i64 && Subtarget.hasDQI())
19137       return true;
19138   }
19139   if (Subtarget.hasDQI() && Subtarget.hasVLX() &&
19140       (VT == MVT::v2i64 || VT == MVT::v4i64))
19141     return true;
19142   return false;
19143 }
19144 
19145 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
19146                                            SelectionDAG &DAG) const {
19147   bool IsStrict = Op->isStrictFPOpcode();
19148   unsigned OpNo = IsStrict ? 1 : 0;
19149   SDValue Src = Op.getOperand(OpNo);
19150   SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19151   MVT SrcVT = Src.getSimpleValueType();
19152   MVT VT = Op.getSimpleValueType();
19153   SDLoc dl(Op);
19154 
19155   if (isSoftF16(VT, Subtarget))
19156     return promoteXINT_TO_FP(Op, DAG);
19157   else if (isLegalConversion(SrcVT, true, Subtarget))
19158     return Op;
19159 
19160   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19161     return LowerWin64_INT128_TO_FP(Op, DAG);
19162 
19163   if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19164     return Extract;
19165 
19166   if (SDValue R = lowerFPToIntToFP(Op, DAG, Subtarget))
19167     return R;
19168 
19169   if (SrcVT.isVector()) {
19170     if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
19171       // Note: Since v2f64 is a legal type. We don't need to zero extend the
19172       // source for strict FP.
19173       if (IsStrict)
19174         return DAG.getNode(
19175             X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
19176             {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19177                                 DAG.getUNDEF(SrcVT))});
19178       return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
19179                          DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19180                                      DAG.getUNDEF(SrcVT)));
19181     }
19182     if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)
19183       return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
19184 
19185     return SDValue();
19186   }
19187 
19188   assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
19189          "Unknown SINT_TO_FP to lower!");
19190 
19191   bool UseSSEReg = isScalarFPTypeInSSEReg(VT);
19192 
19193   // These are really Legal; return the operand so the caller accepts it as
19194   // Legal.
19195   if (SrcVT == MVT::i32 && UseSSEReg)
19196     return Op;
19197   if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit())
19198     return Op;
19199 
19200   if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19201     return V;
19202   if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget))
19203     return V;
19204 
19205   // SSE doesn't have an i16 conversion so we need to promote.
19206   if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) {
19207     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19208     if (IsStrict)
19209       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
19210                          {Chain, Ext});
19211 
19212     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext);
19213   }
19214 
19215   if (VT == MVT::f128 || !Subtarget.hasX87())
19216     return SDValue();
19217 
19218   SDValue ValueToStore = Src;
19219   if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit())
19220     // Bitcasting to f64 here allows us to do a single 64-bit store from
19221     // an SSE register, avoiding the store forwarding penalty that would come
19222     // with two 32-bit stores.
19223     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19224 
19225   unsigned Size = SrcVT.getStoreSize();
19226   Align Alignment(Size);
19227   MachineFunction &MF = DAG.getMachineFunction();
19228   auto PtrVT = getPointerTy(MF.getDataLayout());
19229   int SSFI = MF.getFrameInfo().CreateStackObject(Size, Alignment, false);
19230   MachinePointerInfo MPI =
19231       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19232   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19233   Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment);
19234   std::pair<SDValue, SDValue> Tmp =
19235       BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG);
19236 
19237   if (IsStrict)
19238     return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19239 
19240   return Tmp.first;
19241 }
19242 
19243 std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
19244     EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19245     MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const {
19246   // Build the FILD
19247   SDVTList Tys;
19248   bool useSSE = isScalarFPTypeInSSEReg(DstVT);
19249   if (useSSE)
19250     Tys = DAG.getVTList(MVT::f80, MVT::Other);
19251   else
19252     Tys = DAG.getVTList(DstVT, MVT::Other);
19253 
19254   SDValue FILDOps[] = {Chain, Pointer};
19255   SDValue Result =
19256       DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
19257                               Alignment, MachineMemOperand::MOLoad);
19258   Chain = Result.getValue(1);
19259 
19260   if (useSSE) {
19261     MachineFunction &MF = DAG.getMachineFunction();
19262     unsigned SSFISize = DstVT.getStoreSize();
19263     int SSFI =
19264         MF.getFrameInfo().CreateStackObject(SSFISize, Align(SSFISize), false);
19265     auto PtrVT = getPointerTy(MF.getDataLayout());
19266     SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19267     Tys = DAG.getVTList(MVT::Other);
19268     SDValue FSTOps[] = {Chain, Result, StackSlot};
19269     MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
19270         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
19271         MachineMemOperand::MOStore, SSFISize, Align(SSFISize));
19272 
19273     Chain =
19274         DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
19275     Result = DAG.getLoad(
19276         DstVT, DL, Chain, StackSlot,
19277         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
19278     Chain = Result.getValue(1);
19279   }
19280 
19281   return { Result, Chain };
19282 }
19283 
19284 /// Horizontal vector math instructions may be slower than normal math with
19285 /// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
19286 /// implementation, and likely shuffle complexity of the alternate sequence.
19287 static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
19288                                   const X86Subtarget &Subtarget) {
19289   bool IsOptimizingSize = DAG.shouldOptForSize();
19290   bool HasFastHOps = Subtarget.hasFastHorizontalOps();
19291   return !IsSingleSource || IsOptimizingSize || HasFastHOps;
19292 }
19293 
19294 /// 64-bit unsigned integer to double expansion.
19295 static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
19296                                    const X86Subtarget &Subtarget) {
19297   // We can't use this algorithm for strict fp. It produces -0.0 instead of +0.0
19298   // when converting 0 when rounding toward negative infinity. Caller will
19299   // fall back to Expand for when i64 or is legal or use FILD in 32-bit mode.
19300   assert(!Op->isStrictFPOpcode() && "Expected non-strict uint_to_fp!");
19301   // This algorithm is not obvious. Here it is what we're trying to output:
19302   /*
19303      movq       %rax,  %xmm0
19304      punpckldq  (c0),  %xmm0  // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
19305      subpd      (c1),  %xmm0  // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
19306      #ifdef __SSE3__
19307        haddpd   %xmm0, %xmm0
19308      #else
19309        pshufd   $0x4e, %xmm0, %xmm1
19310        addpd    %xmm1, %xmm0
19311      #endif
19312   */
19313 
19314   SDLoc dl(Op);
19315   LLVMContext *Context = DAG.getContext();
19316 
19317   // Build some magic constants.
19318   static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
19319   Constant *C0 = ConstantDataVector::get(*Context, CV0);
19320   auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19321   SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19322 
19323   SmallVector<Constant*,2> CV1;
19324   CV1.push_back(
19325     ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19326                                       APInt(64, 0x4330000000000000ULL))));
19327   CV1.push_back(
19328     ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19329                                       APInt(64, 0x4530000000000000ULL))));
19330   Constant *C1 = ConstantVector::get(CV1);
19331   SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19332 
19333   // Load the 64-bit value into an XMM register.
19334   SDValue XR1 =
19335       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0));
19336   SDValue CLod0 = DAG.getLoad(
19337       MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
19338       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19339   SDValue Unpck1 =
19340       getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
19341 
19342   SDValue CLod1 = DAG.getLoad(
19343       MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
19344       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19345   SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19346   // TODO: Are there any fast-math-flags to propagate here?
19347   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
19348   SDValue Result;
19349 
19350   if (Subtarget.hasSSE3() &&
19351       shouldUseHorizontalOp(true, DAG, Subtarget)) {
19352     Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
19353   } else {
19354     SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19355     Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
19356   }
19357   Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
19358                        DAG.getIntPtrConstant(0, dl));
19359   return Result;
19360 }
19361 
19362 /// 32-bit unsigned integer to float expansion.
19363 static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
19364                                    const X86Subtarget &Subtarget) {
19365   unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19366   SDLoc dl(Op);
19367   // FP constant to bias correct the final result.
19368   SDValue Bias = DAG.getConstantFP(
19369       llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::f64);
19370 
19371   // Load the 32-bit value into an XMM register.
19372   SDValue Load =
19373       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));
19374 
19375   // Zero out the upper parts of the register.
19376   Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
19377 
19378   // Or the load with the bias.
19379   SDValue Or = DAG.getNode(
19380       ISD::OR, dl, MVT::v2i64,
19381       DAG.getBitcast(MVT::v2i64, Load),
19382       DAG.getBitcast(MVT::v2i64,
19383                      DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
19384   Or =
19385       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
19386                   DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));
19387 
19388   if (Op.getNode()->isStrictFPOpcode()) {
19389     // Subtract the bias.
19390     // TODO: Are there any fast-math-flags to propagate here?
19391     SDValue Chain = Op.getOperand(0);
19392     SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19393                               {Chain, Or, Bias});
19394 
19395     if (Op.getValueType() == Sub.getValueType())
19396       return Sub;
19397 
19398     // Handle final rounding.
19399     std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19400         Sub, Sub.getValue(1), dl, Op.getSimpleValueType());
19401 
19402     return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl);
19403   }
19404 
19405   // Subtract the bias.
19406   // TODO: Are there any fast-math-flags to propagate here?
19407   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19408 
19409   // Handle final rounding.
19410   return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType());
19411 }
19412 
19413 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
19414                                      const X86Subtarget &Subtarget,
19415                                      const SDLoc &DL) {
19416   if (Op.getSimpleValueType() != MVT::v2f64)
19417     return SDValue();
19418 
19419   bool IsStrict = Op->isStrictFPOpcode();
19420 
19421   SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
19422   assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type");
19423 
19424   if (Subtarget.hasAVX512()) {
19425     if (!Subtarget.hasVLX()) {
19426       // Let generic type legalization widen this.
19427       if (!IsStrict)
19428         return SDValue();
19429       // Otherwise pad the integer input with 0s and widen the operation.
19430       N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19431                        DAG.getConstant(0, DL, MVT::v2i32));
19432       SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19433                                 {Op.getOperand(0), N0});
19434       SDValue Chain = Res.getValue(1);
19435       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
19436                         DAG.getIntPtrConstant(0, DL));
19437       return DAG.getMergeValues({Res, Chain}, DL);
19438     }
19439 
19440     // Legalize to v4i32 type.
19441     N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19442                      DAG.getUNDEF(MVT::v2i32));
19443     if (IsStrict)
19444       return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
19445                          {Op.getOperand(0), N0});
19446     return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
19447   }
19448 
19449   // Zero extend to 2i64, OR with the floating point representation of 2^52.
19450   // This gives us the floating point equivalent of 2^52 + the i32 integer
19451   // since double has 52-bits of mantissa. Then subtract 2^52 in floating
19452   // point leaving just our i32 integers in double format.
19453   SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19454   SDValue VBias = DAG.getConstantFP(
19455       llvm::bit_cast<double>(0x4330000000000000ULL), DL, MVT::v2f64);
19456   SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19457                            DAG.getBitcast(MVT::v2i64, VBias));
19458   Or = DAG.getBitcast(MVT::v2f64, Or);
19459 
19460   if (IsStrict)
19461     return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
19462                        {Op.getOperand(0), Or, VBias});
19463   return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
19464 }
19465 
19466 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
19467                                      const X86Subtarget &Subtarget) {
19468   SDLoc DL(Op);
19469   bool IsStrict = Op->isStrictFPOpcode();
19470   SDValue V = Op->getOperand(IsStrict ? 1 : 0);
19471   MVT VecIntVT = V.getSimpleValueType();
19472   assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
19473          "Unsupported custom type");
19474 
19475   if (Subtarget.hasAVX512()) {
19476     // With AVX512, but not VLX we need to widen to get a 512-bit result type.
19477     assert(!Subtarget.hasVLX() && "Unexpected features");
19478     MVT VT = Op->getSimpleValueType(0);
19479 
19480     // v8i32->v8f64 is legal with AVX512 so just return it.
19481     if (VT == MVT::v8f64)
19482       return Op;
19483 
19484     assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
19485            "Unexpected VT!");
19486     MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
19487     MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
19488     // Need to concat with zero vector for strict fp to avoid spurious
19489     // exceptions.
19490     SDValue Tmp =
19491         IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
19492     V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
19493                     DAG.getIntPtrConstant(0, DL));
19494     SDValue Res, Chain;
19495     if (IsStrict) {
19496       Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
19497                         {Op->getOperand(0), V});
19498       Chain = Res.getValue(1);
19499     } else {
19500       Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
19501     }
19502 
19503     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19504                       DAG.getIntPtrConstant(0, DL));
19505 
19506     if (IsStrict)
19507       return DAG.getMergeValues({Res, Chain}, DL);
19508     return Res;
19509   }
19510 
19511   if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 &&
19512       Op->getSimpleValueType(0) == MVT::v4f64) {
19513     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19514     Constant *Bias = ConstantFP::get(
19515         *DAG.getContext(),
19516         APFloat(APFloat::IEEEdouble(), APInt(64, 0x4330000000000000ULL)));
19517     auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19518     SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
19519     SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other);
19520     SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
19521     SDValue VBias = DAG.getMemIntrinsicNode(
19522         X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
19523         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8),
19524         MachineMemOperand::MOLoad);
19525 
19526     SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19527                              DAG.getBitcast(MVT::v4i64, VBias));
19528     Or = DAG.getBitcast(MVT::v4f64, Or);
19529 
19530     if (IsStrict)
19531       return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
19532                          {Op.getOperand(0), Or, VBias});
19533     return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
19534   }
19535 
19536   // The algorithm is the following:
19537   // #ifdef __SSE4_1__
19538   //     uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19539   //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19540   //                                 (uint4) 0x53000000, 0xaa);
19541   // #else
19542   //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19543   //     uint4 hi = (v >> 16) | (uint4) 0x53000000;
19544   // #endif
19545   //     float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19546   //     return (float4) lo + fhi;
19547 
19548   bool Is128 = VecIntVT == MVT::v4i32;
19549   MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
19550   // If we convert to something else than the supported type, e.g., to v4f64,
19551   // abort early.
19552   if (VecFloatVT != Op->getSimpleValueType(0))
19553     return SDValue();
19554 
19555   // In the #idef/#else code, we have in common:
19556   // - The vector of constants:
19557   // -- 0x4b000000
19558   // -- 0x53000000
19559   // - A shift:
19560   // -- v >> 16
19561 
19562   // Create the splat vector for 0x4b000000.
19563   SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19564   // Create the splat vector for 0x53000000.
19565   SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19566 
19567   // Create the right shift.
19568   SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19569   SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19570 
19571   SDValue Low, High;
19572   if (Subtarget.hasSSE41()) {
19573     MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
19574     //     uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19575     SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
19576     SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
19577     // Low will be bitcasted right away, so do not bother bitcasting back to its
19578     // original type.
19579     Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
19580                       VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19581     //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19582     //                                 (uint4) 0x53000000, 0xaa);
19583     SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
19584     SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
19585     // High will be bitcasted right away, so do not bother bitcasting back to
19586     // its original type.
19587     High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
19588                        VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19589   } else {
19590     SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19591     //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19592     SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19593     Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
19594 
19595     //     uint4 hi = (v >> 16) | (uint4) 0x53000000;
19596     High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
19597   }
19598 
19599   // Create the vector constant for (0x1.0p39f + 0x1.0p23f).
19600   SDValue VecCstFSub = DAG.getConstantFP(
19601       APFloat(APFloat::IEEEsingle(), APInt(32, 0x53000080)), DL, VecFloatVT);
19602 
19603   //     float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19604   // NOTE: By using fsub of a positive constant instead of fadd of a negative
19605   // constant, we avoid reassociation in MachineCombiner when unsafe-fp-math is
19606   // enabled. See PR24512.
19607   SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
19608   // TODO: Are there any fast-math-flags to propagate here?
19609   //     (float4) lo;
19610   SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
19611   //     return (float4) lo + fhi;
19612   if (IsStrict) {
19613     SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19614                                 {Op.getOperand(0), HighBitcast, VecCstFSub});
19615     return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
19616                        {FHigh.getValue(1), LowBitcast, FHigh});
19617   }
19618 
19619   SDValue FHigh =
19620       DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
19621   return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
19622 }
19623 
19624 static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
19625                                    const X86Subtarget &Subtarget) {
19626   unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19627   SDValue N0 = Op.getOperand(OpNo);
19628   MVT SrcVT = N0.getSimpleValueType();
19629   SDLoc dl(Op);
19630 
19631   switch (SrcVT.SimpleTy) {
19632   default:
19633     llvm_unreachable("Custom UINT_TO_FP is not supported!");
19634   case MVT::v2i32:
19635     return lowerUINT_TO_FP_v2i32(Op, DAG, Subtarget, dl);
19636   case MVT::v4i32:
19637   case MVT::v8i32:
19638     return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget);
19639   case MVT::v2i64:
19640   case MVT::v4i64:
19641     return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
19642   }
19643 }
19644 
19645 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
19646                                            SelectionDAG &DAG) const {
19647   bool IsStrict = Op->isStrictFPOpcode();
19648   unsigned OpNo = IsStrict ? 1 : 0;
19649   SDValue Src = Op.getOperand(OpNo);
19650   SDLoc dl(Op);
19651   auto PtrVT = getPointerTy(DAG.getDataLayout());
19652   MVT SrcVT = Src.getSimpleValueType();
19653   MVT DstVT = Op->getSimpleValueType(0);
19654   SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19655 
19656   // Bail out when we don't have native conversion instructions.
19657   if (DstVT == MVT::f128)
19658     return SDValue();
19659 
19660   if (isSoftF16(DstVT, Subtarget))
19661     return promoteXINT_TO_FP(Op, DAG);
19662   else if (isLegalConversion(SrcVT, false, Subtarget))
19663     return Op;
19664 
19665   if (DstVT.isVector())
19666     return lowerUINT_TO_FP_vec(Op, DAG, Subtarget);
19667 
19668   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19669     return LowerWin64_INT128_TO_FP(Op, DAG);
19670 
19671   if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19672     return Extract;
19673 
19674   if (Subtarget.hasAVX512() && isScalarFPTypeInSSEReg(DstVT) &&
19675       (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) {
19676     // Conversions from unsigned i32 to f32/f64 are legal,
19677     // using VCVTUSI2SS/SD.  Same for i64 in 64-bit mode.
19678     return Op;
19679   }
19680 
19681   // Promote i32 to i64 and use a signed conversion on 64-bit targets.
19682   if (SrcVT == MVT::i32 && Subtarget.is64Bit()) {
19683     Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);
19684     if (IsStrict)
19685       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},
19686                          {Chain, Src});
19687     return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
19688   }
19689 
19690   if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19691     return V;
19692   if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget))
19693     return V;
19694 
19695   // The transform for i64->f64 isn't correct for 0 when rounding to negative
19696   // infinity. It produces -0.0, so disable under strictfp.
19697   if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() &&
19698       !IsStrict)
19699     return LowerUINT_TO_FP_i64(Op, DAG, Subtarget);
19700   // The transform for i32->f64/f32 isn't correct for 0 when rounding to
19701   // negative infinity. So disable under strictfp. Using FILD instead.
19702   if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 &&
19703       !IsStrict)
19704     return LowerUINT_TO_FP_i32(Op, DAG, Subtarget);
19705   if (Subtarget.is64Bit() && SrcVT == MVT::i64 &&
19706       (DstVT == MVT::f32 || DstVT == MVT::f64))
19707     return SDValue();
19708 
19709   // Make a 64-bit buffer, and use it to build an FILD.
19710   SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
19711   int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
19712   Align SlotAlign(8);
19713   MachinePointerInfo MPI =
19714     MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19715   if (SrcVT == MVT::i32) {
19716     SDValue OffsetSlot =
19717         DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
19718     SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign);
19719     SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
19720                                   OffsetSlot, MPI.getWithOffset(4), SlotAlign);
19721     std::pair<SDValue, SDValue> Tmp =
19722         BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG);
19723     if (IsStrict)
19724       return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19725 
19726     return Tmp.first;
19727   }
19728 
19729   assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
19730   SDValue ValueToStore = Src;
19731   if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit()) {
19732     // Bitcasting to f64 here allows us to do a single 64-bit store from
19733     // an SSE register, avoiding the store forwarding penalty that would come
19734     // with two 32-bit stores.
19735     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19736   }
19737   SDValue Store =
19738       DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign);
19739   // For i64 source, we need to add the appropriate power of 2 if the input
19740   // was negative. We must be careful to do the computation in x87 extended
19741   // precision, not in SSE.
19742   SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19743   SDValue Ops[] = { Store, StackSlot };
19744   SDValue Fild =
19745       DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,
19746                               SlotAlign, MachineMemOperand::MOLoad);
19747   Chain = Fild.getValue(1);
19748 
19749 
19750   // Check whether the sign bit is set.
19751   SDValue SignSet = DAG.getSetCC(
19752       dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
19753       Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
19754 
19755   // Build a 64 bit pair (FF, 0) in the constant pool, with FF in the hi bits.
19756   APInt FF(64, 0x5F80000000000000ULL);
19757   SDValue FudgePtr = DAG.getConstantPool(
19758       ConstantInt::get(*DAG.getContext(), FF), PtrVT);
19759   Align CPAlignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlign();
19760 
19761   // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
19762   SDValue Zero = DAG.getIntPtrConstant(0, dl);
19763   SDValue Four = DAG.getIntPtrConstant(4, dl);
19764   SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
19765   FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
19766 
19767   // Load the value out, extending it from f32 to f80.
19768   SDValue Fudge = DAG.getExtLoad(
19769       ISD::EXTLOAD, dl, MVT::f80, Chain, FudgePtr,
19770       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
19771       CPAlignment);
19772   Chain = Fudge.getValue(1);
19773   // Extend everything to 80 bits to force it to be done on x87.
19774   // TODO: Are there any fast-math-flags to propagate here?
19775   if (IsStrict) {
19776     unsigned Opc = ISD::STRICT_FADD;
19777     // Windows needs the precision control changed to 80bits around this add.
19778     if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19779       Opc = X86ISD::STRICT_FP80_ADD;
19780 
19781     SDValue Add =
19782         DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge});
19783     // STRICT_FP_ROUND can't handle equal types.
19784     if (DstVT == MVT::f80)
19785       return Add;
19786     return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},
19787                        {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)});
19788   }
19789   unsigned Opc = ISD::FADD;
19790   // Windows needs the precision control changed to 80bits around this add.
19791   if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19792     Opc = X86ISD::FP80_ADD;
19793 
19794   SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge);
19795   return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
19796                      DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
19797 }
19798 
19799 // If the given FP_TO_SINT (IsSigned) or FP_TO_UINT (!IsSigned) operation
19800 // is legal, or has an fp128 or f16 source (which needs to be promoted to f32),
19801 // just return an SDValue().
19802 // Otherwise it is assumed to be a conversion from one of f32, f64 or f80
19803 // to i16, i32 or i64, and we lower it to a legal sequence and return the
19804 // result.
19805 SDValue
19806 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
19807                                    bool IsSigned, SDValue &Chain) const {
19808   bool IsStrict = Op->isStrictFPOpcode();
19809   SDLoc DL(Op);
19810 
19811   EVT DstTy = Op.getValueType();
19812   SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
19813   EVT TheVT = Value.getValueType();
19814   auto PtrVT = getPointerTy(DAG.getDataLayout());
19815 
19816   if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {
19817     // f16 must be promoted before using the lowering in this routine.
19818     // fp128 does not use this lowering.
19819     return SDValue();
19820   }
19821 
19822   // If using FIST to compute an unsigned i64, we'll need some fixup
19823   // to handle values above the maximum signed i64.  A FIST is always
19824   // used for the 32-bit subtarget, but also for f80 on a 64-bit target.
19825   bool UnsignedFixup = !IsSigned && DstTy == MVT::i64;
19826 
19827   // FIXME: This does not generate an invalid exception if the input does not
19828   // fit in i32. PR44019
19829   if (!IsSigned && DstTy != MVT::i64) {
19830     // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST.
19831     // The low 32 bits of the fist result will have the correct uint32 result.
19832     assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
19833     DstTy = MVT::i64;
19834   }
19835 
19836   assert(DstTy.getSimpleVT() <= MVT::i64 &&
19837          DstTy.getSimpleVT() >= MVT::i16 &&
19838          "Unknown FP_TO_INT to lower!");
19839 
19840   // We lower FP->int64 into FISTP64 followed by a load from a temporary
19841   // stack slot.
19842   MachineFunction &MF = DAG.getMachineFunction();
19843   unsigned MemSize = DstTy.getStoreSize();
19844   int SSFI =
19845       MF.getFrameInfo().CreateStackObject(MemSize, Align(MemSize), false);
19846   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19847 
19848   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19849 
19850   SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
19851 
19852   if (UnsignedFixup) {
19853     //
19854     // Conversion to unsigned i64 is implemented with a select,
19855     // depending on whether the source value fits in the range
19856     // of a signed i64.  Let Thresh be the FP equivalent of
19857     // 0x8000000000000000ULL.
19858     //
19859     //  Adjust = (Value >= Thresh) ? 0x80000000 : 0;
19860     //  FltOfs = (Value >= Thresh) ? 0x80000000 : 0;
19861     //  FistSrc = (Value - FltOfs);
19862     //  Fist-to-mem64 FistSrc
19863     //  Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent
19864     //  to XOR'ing the high 32 bits with Adjust.
19865     //
19866     // Being a power of 2, Thresh is exactly representable in all FP formats.
19867     // For X87 we'd like to use the smallest FP type for this constant, but
19868     // for DAG type consistency we have to match the FP operand type.
19869 
19870     APFloat Thresh(APFloat::IEEEsingle(), APInt(32, 0x5f000000));
19871     LLVM_ATTRIBUTE_UNUSED APFloat::opStatus Status = APFloat::opOK;
19872     bool LosesInfo = false;
19873     if (TheVT == MVT::f64)
19874       // The rounding mode is irrelevant as the conversion should be exact.
19875       Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
19876                               &LosesInfo);
19877     else if (TheVT == MVT::f80)
19878       Status = Thresh.convert(APFloat::x87DoubleExtended(),
19879                               APFloat::rmNearestTiesToEven, &LosesInfo);
19880 
19881     assert(Status == APFloat::opOK && !LosesInfo &&
19882            "FP conversion should have been exact");
19883 
19884     SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
19885 
19886     EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
19887                                    *DAG.getContext(), TheVT);
19888     SDValue Cmp;
19889     if (IsStrict) {
19890       Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
19891                          /*IsSignaling*/ true);
19892       Chain = Cmp.getValue(1);
19893     } else {
19894       Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE);
19895     }
19896 
19897     // Our preferred lowering of
19898     //
19899     // (Value >= Thresh) ? 0x8000000000000000ULL : 0
19900     //
19901     // is
19902     //
19903     // (Value >= Thresh) << 63
19904     //
19905     // but since we can get here after LegalOperations, DAGCombine might do the
19906     // wrong thing if we create a select. So, directly create the preferred
19907     // version.
19908     SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);
19909     SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);
19910     Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63);
19911 
19912     SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal,
19913                                    DAG.getConstantFP(0.0, DL, TheVT));
19914 
19915     if (IsStrict) {
19916       Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
19917                           { Chain, Value, FltOfs });
19918       Chain = Value.getValue(1);
19919     } else
19920       Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
19921   }
19922 
19923   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
19924 
19925   // FIXME This causes a redundant load/store if the SSE-class value is already
19926   // in memory, such as if it is on the callstack.
19927   if (isScalarFPTypeInSSEReg(TheVT)) {
19928     assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
19929     Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
19930     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19931     SDValue Ops[] = { Chain, StackSlot };
19932 
19933     unsigned FLDSize = TheVT.getStoreSize();
19934     assert(FLDSize <= MemSize && "Stack slot not big enough");
19935     MachineMemOperand *MMO = MF.getMachineMemOperand(
19936         MPI, MachineMemOperand::MOLoad, FLDSize, Align(FLDSize));
19937     Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
19938     Chain = Value.getValue(1);
19939   }
19940 
19941   // Build the FP_TO_INT*_IN_MEM
19942   MachineMemOperand *MMO = MF.getMachineMemOperand(
19943       MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
19944   SDValue Ops[] = { Chain, Value, StackSlot };
19945   SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
19946                                          DAG.getVTList(MVT::Other),
19947                                          Ops, DstTy, MMO);
19948 
19949   SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
19950   Chain = Res.getValue(1);
19951 
19952   // If we need an unsigned fixup, XOR the result with adjust.
19953   if (UnsignedFixup)
19954     Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
19955 
19956   return Res;
19957 }
19958 
19959 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
19960                               const X86Subtarget &Subtarget) {
19961   MVT VT = Op.getSimpleValueType();
19962   SDValue In = Op.getOperand(0);
19963   MVT InVT = In.getSimpleValueType();
19964   SDLoc dl(Op);
19965   unsigned Opc = Op.getOpcode();
19966 
19967   assert(VT.isVector() && InVT.isVector() && "Expected vector type");
19968   assert((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&
19969          "Unexpected extension opcode");
19970   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
19971          "Expected same number of elements");
19972   assert((VT.getVectorElementType() == MVT::i16 ||
19973           VT.getVectorElementType() == MVT::i32 ||
19974           VT.getVectorElementType() == MVT::i64) &&
19975          "Unexpected element type");
19976   assert((InVT.getVectorElementType() == MVT::i8 ||
19977           InVT.getVectorElementType() == MVT::i16 ||
19978           InVT.getVectorElementType() == MVT::i32) &&
19979          "Unexpected element type");
19980 
19981   unsigned ExtendInVecOpc = DAG.getOpcode_EXTEND_VECTOR_INREG(Opc);
19982 
19983   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
19984     assert(InVT == MVT::v32i8 && "Unexpected VT!");
19985     return splitVectorIntUnary(Op, DAG);
19986   }
19987 
19988   if (Subtarget.hasInt256())
19989     return Op;
19990 
19991   // Optimize vectors in AVX mode:
19992   //
19993   //   v8i16 -> v8i32
19994   //   Use vpmovzwd for 4 lower elements  v8i16 -> v4i32.
19995   //   Use vpunpckhwd for 4 upper elements  v8i16 -> v4i32.
19996   //   Concat upper and lower parts.
19997   //
19998   //   v4i32 -> v4i64
19999   //   Use vpmovzdq for 4 lower elements  v4i32 -> v2i64.
20000   //   Use vpunpckhdq for 4 upper elements  v4i32 -> v2i64.
20001   //   Concat upper and lower parts.
20002   //
20003   MVT HalfVT = VT.getHalfNumVectorElementsVT();
20004   SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20005 
20006   // Short-circuit if we can determine that each 128-bit half is the same value.
20007   // Otherwise, this is difficult to match and optimize.
20008   if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(In))
20009     if (hasIdenticalHalvesShuffleMask(Shuf->getMask()))
20010       return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo);
20011 
20012   SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20013   SDValue Undef = DAG.getUNDEF(InVT);
20014   bool NeedZero = Opc == ISD::ZERO_EXTEND;
20015   SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20016   OpHi = DAG.getBitcast(HalfVT, OpHi);
20017 
20018   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
20019 }
20020 
20021 // Helper to split and extend a v16i1 mask to v16i8 or v16i16.
20022 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
20023                                    const SDLoc &dl, SelectionDAG &DAG) {
20024   assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.");
20025   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20026                            DAG.getIntPtrConstant(0, dl));
20027   SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20028                            DAG.getIntPtrConstant(8, dl));
20029   Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);
20030   Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);
20031   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20032   return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20033 }
20034 
20035 static  SDValue LowerZERO_EXTEND_Mask(SDValue Op,
20036                                       const X86Subtarget &Subtarget,
20037                                       SelectionDAG &DAG) {
20038   MVT VT = Op->getSimpleValueType(0);
20039   SDValue In = Op->getOperand(0);
20040   MVT InVT = In.getSimpleValueType();
20041   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
20042   SDLoc DL(Op);
20043   unsigned NumElts = VT.getVectorNumElements();
20044 
20045   // For all vectors, but vXi8 we can just emit a sign_extend and a shift. This
20046   // avoids a constant pool load.
20047   if (VT.getVectorElementType() != MVT::i8) {
20048     SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20049     return DAG.getNode(ISD::SRL, DL, VT, Extend,
20050                        DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
20051   }
20052 
20053   // Extend VT if BWI is not supported.
20054   MVT ExtVT = VT;
20055   if (!Subtarget.hasBWI()) {
20056     // If v16i32 is to be avoided, we'll need to split and concatenate.
20057     if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
20058       return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
20059 
20060     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
20061   }
20062 
20063   // Widen to 512-bits if VLX is not supported.
20064   MVT WideVT = ExtVT;
20065   if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
20066     NumElts *= 512 / ExtVT.getSizeInBits();
20067     InVT = MVT::getVectorVT(MVT::i1, NumElts);
20068     In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT),
20069                      In, DAG.getIntPtrConstant(0, DL));
20070     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(),
20071                               NumElts);
20072   }
20073 
20074   SDValue One = DAG.getConstant(1, DL, WideVT);
20075   SDValue Zero = DAG.getConstant(0, DL, WideVT);
20076 
20077   SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20078 
20079   // Truncate if we had to extend above.
20080   if (VT != ExtVT) {
20081     WideVT = MVT::getVectorVT(MVT::i8, NumElts);
20082     SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
20083   }
20084 
20085   // Extract back to 128/256-bit if we widened.
20086   if (WideVT != VT)
20087     SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
20088                               DAG.getIntPtrConstant(0, DL));
20089 
20090   return SelectedVal;
20091 }
20092 
20093 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
20094                                 SelectionDAG &DAG) {
20095   SDValue In = Op.getOperand(0);
20096   MVT SVT = In.getSimpleValueType();
20097 
20098   if (SVT.getVectorElementType() == MVT::i1)
20099     return LowerZERO_EXTEND_Mask(Op, Subtarget, DAG);
20100 
20101   assert(Subtarget.hasAVX() && "Expected AVX support");
20102   return LowerAVXExtend(Op, DAG, Subtarget);
20103 }
20104 
20105 /// Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
20106 /// It makes use of the fact that vectors with enough leading sign/zero bits
20107 /// prevent the PACKSS/PACKUS from saturating the results.
20108 /// AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates
20109 /// within each 128-bit lane.
20110 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
20111                                       const SDLoc &DL, SelectionDAG &DAG,
20112                                       const X86Subtarget &Subtarget) {
20113   assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
20114          "Unexpected PACK opcode");
20115   assert(DstVT.isVector() && "VT not a vector?");
20116 
20117   // Requires SSE2 for PACKSS (SSE41 PACKUSDW is handled below).
20118   if (!Subtarget.hasSSE2())
20119     return SDValue();
20120 
20121   EVT SrcVT = In.getValueType();
20122 
20123   // No truncation required, we might get here due to recursive calls.
20124   if (SrcVT == DstVT)
20125     return In;
20126 
20127   unsigned NumElems = SrcVT.getVectorNumElements();
20128   if (NumElems < 2 || !isPowerOf2_32(NumElems) )
20129     return SDValue();
20130 
20131   unsigned DstSizeInBits = DstVT.getSizeInBits();
20132   unsigned SrcSizeInBits = SrcVT.getSizeInBits();
20133   assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
20134   assert(SrcSizeInBits > DstSizeInBits && "Illegal truncation");
20135 
20136   LLVMContext &Ctx = *DAG.getContext();
20137   EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
20138   EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
20139 
20140   // Pack to the largest type possible:
20141   // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
20142   EVT InVT = MVT::i16, OutVT = MVT::i8;
20143   if (SrcVT.getScalarSizeInBits() > 16 &&
20144       (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) {
20145     InVT = MVT::i32;
20146     OutVT = MVT::i16;
20147   }
20148 
20149   // Sub-128-bit truncation - widen to 128-bit src and pack in the lower half.
20150   // On pre-AVX512, pack the src in both halves to help value tracking.
20151   if (SrcSizeInBits <= 128) {
20152     InVT = EVT::getVectorVT(Ctx, InVT, 128 / InVT.getSizeInBits());
20153     OutVT = EVT::getVectorVT(Ctx, OutVT, 128 / OutVT.getSizeInBits());
20154     In = widenSubVector(In, false, Subtarget, DAG, DL, 128);
20155     SDValue LHS = DAG.getBitcast(InVT, In);
20156     SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS;
20157     SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS);
20158     Res = extractSubVector(Res, 0, DAG, DL, SrcSizeInBits / 2);
20159     Res = DAG.getBitcast(PackedVT, Res);
20160     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20161   }
20162 
20163   // Split lower/upper subvectors.
20164   SDValue Lo, Hi;
20165   std::tie(Lo, Hi) = splitVector(In, DAG, DL);
20166 
20167   // If Hi is undef, then don't bother packing it and widen the result instead.
20168   if (Hi.isUndef()) {
20169     EVT DstHalfVT = DstVT.getHalfNumVectorElementsVT(Ctx);
20170     if (SDValue Res =
20171             truncateVectorWithPACK(Opcode, DstHalfVT, Lo, DL, DAG, Subtarget))
20172       return widenSubVector(Res, false, Subtarget, DAG, DL, DstSizeInBits);
20173   }
20174 
20175   unsigned SubSizeInBits = SrcSizeInBits / 2;
20176   InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
20177   OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
20178 
20179   // 256bit -> 128bit truncate - PACK lower/upper 128-bit subvectors.
20180   if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
20181     Lo = DAG.getBitcast(InVT, Lo);
20182     Hi = DAG.getBitcast(InVT, Hi);
20183     SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20184     return DAG.getBitcast(DstVT, Res);
20185   }
20186 
20187   // AVX2: 512bit -> 256bit truncate - PACK lower/upper 256-bit subvectors.
20188   // AVX2: 512bit -> 128bit truncate - PACK(PACK, PACK).
20189   if (SrcVT.is512BitVector() && Subtarget.hasInt256()) {
20190     Lo = DAG.getBitcast(InVT, Lo);
20191     Hi = DAG.getBitcast(InVT, Hi);
20192     SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20193 
20194     // 256-bit PACK(ARG0, ARG1) leaves us with ((LO0,LO1),(HI0,HI1)),
20195     // so we need to shuffle to get ((LO0,HI0),(LO1,HI1)).
20196     // Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
20197     SmallVector<int, 64> Mask;
20198     int Scale = 64 / OutVT.getScalarSizeInBits();
20199     narrowShuffleMaskElts(Scale, { 0, 2, 1, 3 }, Mask);
20200     Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
20201 
20202     if (DstVT.is256BitVector())
20203       return DAG.getBitcast(DstVT, Res);
20204 
20205     // If 512bit -> 128bit truncate another stage.
20206     Res = DAG.getBitcast(PackedVT, Res);
20207     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20208   }
20209 
20210   // Recursively pack lower/upper subvectors, concat result and pack again.
20211   assert(SrcSizeInBits >= 256 && "Expected 256-bit vector or greater");
20212 
20213   if (PackedVT.is128BitVector()) {
20214     // Avoid CONCAT_VECTORS on sub-128bit nodes as these can fail after
20215     // type legalization.
20216     SDValue Res =
20217         truncateVectorWithPACK(Opcode, PackedVT, In, DL, DAG, Subtarget);
20218     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20219   }
20220 
20221   EVT HalfPackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
20222   Lo = truncateVectorWithPACK(Opcode, HalfPackedVT, Lo, DL, DAG, Subtarget);
20223   Hi = truncateVectorWithPACK(Opcode, HalfPackedVT, Hi, DL, DAG, Subtarget);
20224   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20225   return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20226 }
20227 
20228 /// Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
20229 /// e.g. trunc <8 x i32> X to <8 x i16> -->
20230 /// MaskX = X & 0xffff (clear high bits to prevent saturation)
20231 /// packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)
20232 static SDValue truncateVectorWithPACKUS(EVT DstVT, SDValue In, const SDLoc &DL,
20233                                         const X86Subtarget &Subtarget,
20234                                         SelectionDAG &DAG) {
20235   In = DAG.getZeroExtendInReg(In, DL, DstVT);
20236   return truncateVectorWithPACK(X86ISD::PACKUS, DstVT, In, DL, DAG, Subtarget);
20237 }
20238 
20239 /// Truncate using inreg sign extension and X86ISD::PACKSS.
20240 static SDValue truncateVectorWithPACKSS(EVT DstVT, SDValue In, const SDLoc &DL,
20241                                         const X86Subtarget &Subtarget,
20242                                         SelectionDAG &DAG) {
20243   EVT SrcVT = In.getValueType();
20244   In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SrcVT, In,
20245                    DAG.getValueType(DstVT));
20246   return truncateVectorWithPACK(X86ISD::PACKSS, DstVT, In, DL, DAG, Subtarget);
20247 }
20248 
20249 /// Helper to determine if \p In truncated to \p DstVT has the necessary
20250 /// signbits / leading zero bits to be truncated with PACKSS / PACKUS,
20251 /// possibly by converting a SRL node to SRA for sign extension.
20252 static SDValue matchTruncateWithPACK(unsigned &PackOpcode, EVT DstVT,
20253                                      SDValue In, const SDLoc &DL,
20254                                      SelectionDAG &DAG,
20255                                      const X86Subtarget &Subtarget) {
20256   // Requires SSE2.
20257   if (!Subtarget.hasSSE2())
20258     return SDValue();
20259 
20260   EVT SrcVT = In.getValueType();
20261   EVT DstSVT = DstVT.getVectorElementType();
20262   EVT SrcSVT = SrcVT.getVectorElementType();
20263 
20264   // Check we have a truncation suited for PACKSS/PACKUS.
20265   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20266         (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20267     return SDValue();
20268 
20269   assert(SrcSVT.getSizeInBits() > DstSVT.getSizeInBits() && "Bad truncation");
20270   unsigned NumStages = Log2_32(SrcSVT.getSizeInBits() / DstSVT.getSizeInBits());
20271 
20272   // Truncation from 128-bit to vXi32 can be better handled with PSHUFD.
20273   // Truncation to sub-64-bit vXi16 can be better handled with PSHUFD/PSHUFLW.
20274   // Truncation from v2i64 to v2i8 can be better handled with PSHUFB.
20275   if ((DstSVT == MVT::i32 && SrcVT.getSizeInBits() <= 128) ||
20276       (DstSVT == MVT::i16 && SrcVT.getSizeInBits() <= (64 * NumStages)) ||
20277       (DstVT == MVT::v2i8 && SrcVT == MVT::v2i64 && Subtarget.hasSSSE3()))
20278     return SDValue();
20279 
20280   // Prefer to lower v4i64 -> v4i32 as a shuffle unless we can cheaply
20281   // split this for packing.
20282   if (SrcVT == MVT::v4i64 && DstVT == MVT::v4i32 &&
20283       !isFreeToSplitVector(In.getNode(), DAG) &&
20284       (!Subtarget.hasAVX() || DAG.ComputeNumSignBits(In) != 64))
20285     return SDValue();
20286 
20287   // Don't truncate AVX512 targets as multiple PACK nodes stages.
20288   if (Subtarget.hasAVX512() && NumStages > 1)
20289     return SDValue();
20290 
20291   unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
20292   unsigned NumPackedSignBits = std::min<unsigned>(DstSVT.getSizeInBits(), 16);
20293   unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8;
20294 
20295   // Truncate with PACKUS if we are truncating a vector with leading zero
20296   // bits that extend all the way to the packed/truncated value.
20297   // e.g. Masks, zext_in_reg, etc.
20298   // Pre-SSE41 we can only use PACKUSWB.
20299   KnownBits Known = DAG.computeKnownBits(In);
20300   if ((NumSrcEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros()) {
20301     PackOpcode = X86ISD::PACKUS;
20302     return In;
20303   }
20304 
20305   // Truncate with PACKSS if we are truncating a vector with sign-bits
20306   // that extend all the way to the packed/truncated value.
20307   // e.g. Comparison result, sext_in_reg, etc.
20308   unsigned NumSignBits = DAG.ComputeNumSignBits(In);
20309 
20310   // Don't use PACKSS for vXi64 -> vXi32 truncations unless we're dealing with
20311   // a sign splat (or AVX512 VPSRAQ support). ComputeNumSignBits struggles to
20312   // see through BITCASTs later on and combines/simplifications can't then use
20313   // it.
20314   if (DstSVT == MVT::i32 && NumSignBits != SrcSVT.getSizeInBits() &&
20315       !Subtarget.hasAVX512())
20316     return SDValue();
20317 
20318   unsigned MinSignBits = NumSrcEltBits - NumPackedSignBits;
20319   if (MinSignBits < NumSignBits) {
20320     PackOpcode = X86ISD::PACKSS;
20321     return In;
20322   }
20323 
20324   // If we have a srl that only generates signbits that we will discard in
20325   // the truncation then we can use PACKSS by converting the srl to a sra.
20326   // SimplifyDemandedBits often relaxes sra to srl so we need to reverse it.
20327   if (In.getOpcode() == ISD::SRL && In->hasOneUse())
20328     if (const APInt *ShAmt = DAG.getValidShiftAmountConstant(
20329             In, APInt::getAllOnes(SrcVT.getVectorNumElements()))) {
20330       if (*ShAmt == MinSignBits) {
20331         PackOpcode = X86ISD::PACKSS;
20332         return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops());
20333       }
20334     }
20335 
20336   return SDValue();
20337 }
20338 
20339 /// This function lowers a vector truncation of 'extended sign-bits' or
20340 /// 'extended zero-bits' values.
20341 /// vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
20342 static SDValue LowerTruncateVecPackWithSignBits(MVT DstVT, SDValue In,
20343                                                 const SDLoc &DL,
20344                                                 const X86Subtarget &Subtarget,
20345                                                 SelectionDAG &DAG) {
20346   MVT SrcVT = In.getSimpleValueType();
20347   MVT DstSVT = DstVT.getVectorElementType();
20348   MVT SrcSVT = SrcVT.getVectorElementType();
20349   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20350         (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20351     return SDValue();
20352 
20353   // If the upper half of the source is undef, then attempt to split and
20354   // only truncate the lower half.
20355   if (DstVT.getSizeInBits() >= 128) {
20356     SmallVector<SDValue> LowerOps;
20357     if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20358       MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20359       if (SDValue Res = LowerTruncateVecPackWithSignBits(DstHalfVT, Lo, DL,
20360                                                          Subtarget, DAG))
20361         return widenSubVector(Res, false, Subtarget, DAG, DL,
20362                               DstVT.getSizeInBits());
20363     }
20364   }
20365 
20366   unsigned PackOpcode;
20367   if (SDValue Src =
20368           matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG, Subtarget))
20369     return truncateVectorWithPACK(PackOpcode, DstVT, Src, DL, DAG, Subtarget);
20370 
20371   return SDValue();
20372 }
20373 
20374 /// This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into
20375 /// X86ISD::PACKUS/X86ISD::PACKSS operations.
20376 static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL,
20377                                     const X86Subtarget &Subtarget,
20378                                     SelectionDAG &DAG) {
20379   MVT SrcVT = In.getSimpleValueType();
20380   MVT DstSVT = DstVT.getVectorElementType();
20381   MVT SrcSVT = SrcVT.getVectorElementType();
20382   unsigned NumElems = DstVT.getVectorNumElements();
20383   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20384         (DstSVT == MVT::i8 || DstSVT == MVT::i16) && isPowerOf2_32(NumElems) &&
20385         NumElems >= 8))
20386     return SDValue();
20387 
20388   // SSSE3's pshufb results in less instructions in the cases below.
20389   if (Subtarget.hasSSSE3() && NumElems == 8) {
20390     if (SrcSVT == MVT::i16)
20391       return SDValue();
20392     if (SrcSVT == MVT::i32 && (DstSVT == MVT::i8 || !Subtarget.hasSSE41()))
20393       return SDValue();
20394   }
20395 
20396   // If the upper half of the source is undef, then attempt to split and
20397   // only truncate the lower half.
20398   if (DstVT.getSizeInBits() >= 128) {
20399     SmallVector<SDValue> LowerOps;
20400     if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20401       MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20402       if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG))
20403         return widenSubVector(Res, false, Subtarget, DAG, DL,
20404                               DstVT.getSizeInBits());
20405     }
20406   }
20407 
20408   // SSE2 provides PACKUS for only 2 x v8i16 -> v16i8 and SSE4.1 provides PACKUS
20409   // for 2 x v4i32 -> v8i16. For SSSE3 and below, we need to use PACKSS to
20410   // truncate 2 x v4i32 to v8i16.
20411   if (Subtarget.hasSSE41() || DstSVT == MVT::i8)
20412     return truncateVectorWithPACKUS(DstVT, In, DL, Subtarget, DAG);
20413 
20414   if (SrcSVT == MVT::i16 || SrcSVT == MVT::i32)
20415     return truncateVectorWithPACKSS(DstVT, In, DL, Subtarget, DAG);
20416 
20417   // Special case vXi64 -> vXi16, shuffle to vXi32 and then use PACKSS.
20418   if (DstSVT == MVT::i16 && SrcSVT == MVT::i64) {
20419     MVT TruncVT = MVT::getVectorVT(MVT::i32, NumElems);
20420     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In);
20421     return truncateVectorWithPACKSS(DstVT, Trunc, DL, Subtarget, DAG);
20422   }
20423 
20424   return SDValue();
20425 }
20426 
20427 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
20428                                   const X86Subtarget &Subtarget) {
20429 
20430   SDLoc DL(Op);
20431   MVT VT = Op.getSimpleValueType();
20432   SDValue In = Op.getOperand(0);
20433   MVT InVT = In.getSimpleValueType();
20434 
20435   assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");
20436 
20437   // Shift LSB to MSB and use VPMOVB/W2M or TESTD/Q.
20438   unsigned ShiftInx = InVT.getScalarSizeInBits() - 1;
20439   if (InVT.getScalarSizeInBits() <= 16) {
20440     if (Subtarget.hasBWI()) {
20441       // legal, will go to VPMOVB2M, VPMOVW2M
20442       if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20443         // We need to shift to get the lsb into sign position.
20444         // Shift packed bytes not supported natively, bitcast to word
20445         MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
20446         In = DAG.getNode(ISD::SHL, DL, ExtVT,
20447                          DAG.getBitcast(ExtVT, In),
20448                          DAG.getConstant(ShiftInx, DL, ExtVT));
20449         In = DAG.getBitcast(InVT, In);
20450       }
20451       return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
20452                           In, ISD::SETGT);
20453     }
20454     // Use TESTD/Q, extended vector to packed dword/qword.
20455     assert((InVT.is256BitVector() || InVT.is128BitVector()) &&
20456            "Unexpected vector type.");
20457     unsigned NumElts = InVT.getVectorNumElements();
20458     assert((NumElts == 8 || NumElts == 16) && "Unexpected number of elements");
20459     // We need to change to a wider element type that we have support for.
20460     // For 8 element vectors this is easy, we either extend to v8i32 or v8i64.
20461     // For 16 element vectors we extend to v16i32 unless we are explicitly
20462     // trying to avoid 512-bit vectors. If we are avoiding 512-bit vectors
20463     // we need to split into two 8 element vectors which we can extend to v8i32,
20464     // truncate and concat the results. There's an additional complication if
20465     // the original type is v16i8. In that case we can't split the v16i8
20466     // directly, so we need to shuffle high elements to low and use
20467     // sign_extend_vector_inreg.
20468     if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) {
20469       SDValue Lo, Hi;
20470       if (InVT == MVT::v16i8) {
20471         Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);
20472         Hi = DAG.getVectorShuffle(
20473             InVT, DL, In, In,
20474             {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
20475         Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);
20476       } else {
20477         assert(InVT == MVT::v16i16 && "Unexpected VT!");
20478         Lo = extract128BitVector(In, 0, DAG, DL);
20479         Hi = extract128BitVector(In, 8, DAG, DL);
20480       }
20481       // We're split now, just emit two truncates and a concat. The two
20482       // truncates will trigger legalization to come back to this function.
20483       Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
20484       Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
20485       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20486     }
20487     // We either have 8 elements or we're allowed to use 512-bit vectors.
20488     // If we have VLX, we want to use the narrowest vector that can get the
20489     // job done so we use vXi32.
20490     MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
20491     MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
20492     In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
20493     InVT = ExtVT;
20494     ShiftInx = InVT.getScalarSizeInBits() - 1;
20495   }
20496 
20497   if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20498     // We need to shift to get the lsb into sign position.
20499     In = DAG.getNode(ISD::SHL, DL, InVT, In,
20500                      DAG.getConstant(ShiftInx, DL, InVT));
20501   }
20502   // If we have DQI, emit a pattern that will be iseled as vpmovq2m/vpmovd2m.
20503   if (Subtarget.hasDQI())
20504     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
20505   return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
20506 }
20507 
20508 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
20509   SDLoc DL(Op);
20510   MVT VT = Op.getSimpleValueType();
20511   SDValue In = Op.getOperand(0);
20512   MVT InVT = In.getSimpleValueType();
20513   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
20514          "Invalid TRUNCATE operation");
20515 
20516   // If we're called by the type legalizer, handle a few cases.
20517   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20518   if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(InVT)) {
20519     if ((InVT == MVT::v8i64 || InVT == MVT::v16i32 || InVT == MVT::v16i64) &&
20520         VT.is128BitVector() && Subtarget.hasAVX512()) {
20521       assert((InVT == MVT::v16i64 || Subtarget.hasVLX()) &&
20522              "Unexpected subtarget!");
20523       // The default behavior is to truncate one step, concatenate, and then
20524       // truncate the remainder. We'd rather produce two 64-bit results and
20525       // concatenate those.
20526       SDValue Lo, Hi;
20527       std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
20528 
20529       EVT LoVT, HiVT;
20530       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
20531 
20532       Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
20533       Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
20534       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20535     }
20536 
20537     // Pre-AVX512 (or prefer-256bit) see if we can make use of PACKSS/PACKUS.
20538     if (!Subtarget.hasAVX512() ||
20539         (InVT.is512BitVector() && VT.is256BitVector()))
20540       if (SDValue SignPack =
20541               LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20542         return SignPack;
20543 
20544     // Pre-AVX512 see if we can make use of PACKSS/PACKUS.
20545     if (!Subtarget.hasAVX512())
20546       return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG);
20547 
20548     // Otherwise let default legalization handle it.
20549     return SDValue();
20550   }
20551 
20552   if (VT.getVectorElementType() == MVT::i1)
20553     return LowerTruncateVecI1(Op, DAG, Subtarget);
20554 
20555   // Attempt to truncate with PACKUS/PACKSS even on AVX512 if we'd have to
20556   // concat from subvectors to use VPTRUNC etc.
20557   if (!Subtarget.hasAVX512() || isFreeToSplitVector(In.getNode(), DAG))
20558     if (SDValue SignPack =
20559             LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20560       return SignPack;
20561 
20562   // vpmovqb/w/d, vpmovdb/w, vpmovwb
20563   if (Subtarget.hasAVX512()) {
20564     if (InVT == MVT::v32i16 && !Subtarget.hasBWI()) {
20565       assert(VT == MVT::v32i8 && "Unexpected VT!");
20566       return splitVectorIntUnary(Op, DAG);
20567     }
20568 
20569     // word to byte only under BWI. Otherwise we have to promoted to v16i32
20570     // and then truncate that. But we should only do that if we haven't been
20571     // asked to avoid 512-bit vectors. The actual promotion to v16i32 will be
20572     // handled by isel patterns.
20573     if (InVT != MVT::v16i16 || Subtarget.hasBWI() ||
20574         Subtarget.canExtendTo512DQ())
20575       return Op;
20576   }
20577 
20578   // Handle truncation of V256 to V128 using shuffles.
20579   assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!");
20580 
20581   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
20582     // On AVX2, v4i64 -> v4i32 becomes VPERMD.
20583     if (Subtarget.hasInt256()) {
20584       static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
20585       In = DAG.getBitcast(MVT::v8i32, In);
20586       In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
20587       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
20588                          DAG.getIntPtrConstant(0, DL));
20589     }
20590 
20591     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20592                                DAG.getIntPtrConstant(0, DL));
20593     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20594                                DAG.getIntPtrConstant(2, DL));
20595     static const int ShufMask[] = {0, 2, 4, 6};
20596     return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo),
20597                                 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask);
20598   }
20599 
20600   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
20601     // On AVX2, v8i32 -> v8i16 becomes PSHUFB.
20602     if (Subtarget.hasInt256()) {
20603       // The PSHUFB mask:
20604       static const int ShufMask1[] = { 0,  1,  4,  5,  8,  9, 12, 13,
20605                                       -1, -1, -1, -1, -1, -1, -1, -1,
20606                                       16, 17, 20, 21, 24, 25, 28, 29,
20607                                       -1, -1, -1, -1, -1, -1, -1, -1 };
20608       In = DAG.getBitcast(MVT::v32i8, In);
20609       In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
20610       In = DAG.getBitcast(MVT::v4i64, In);
20611 
20612       static const int ShufMask2[] = {0, 2, -1, -1};
20613       In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
20614       In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20615                        DAG.getIntPtrConstant(0, DL));
20616       return DAG.getBitcast(MVT::v8i16, In);
20617     }
20618 
20619     return Subtarget.hasSSE41()
20620                ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG)
20621                : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG);
20622   }
20623 
20624   if (VT == MVT::v16i8 && InVT == MVT::v16i16)
20625     return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG);
20626 
20627   llvm_unreachable("All 256->128 cases should have been handled above!");
20628 }
20629 
20630 // We can leverage the specific way the "cvttps2dq/cvttpd2dq" instruction
20631 // behaves on out of range inputs to generate optimized conversions.
20632 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl,
20633                                     SelectionDAG &DAG,
20634                                     const X86Subtarget &Subtarget) {
20635   MVT SrcVT = Src.getSimpleValueType();
20636   unsigned DstBits = VT.getScalarSizeInBits();
20637   assert(DstBits == 32 && "expandFP_TO_UINT_SSE - only vXi32 supported");
20638 
20639   // Calculate the converted result for values in the range 0 to
20640   // 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
20641   SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src);
20642   SDValue Big =
20643       DAG.getNode(X86ISD::CVTTP2SI, dl, VT,
20644                   DAG.getNode(ISD::FSUB, dl, SrcVT, Src,
20645                               DAG.getConstantFP(2147483648.0f, dl, SrcVT)));
20646 
20647   // The "CVTTP2SI" instruction conveniently sets the sign bit if
20648   // and only if the value was out of range. So we can use that
20649   // as our indicator that we rather use "Big" instead of "Small".
20650   //
20651   // Use "Small" if "IsOverflown" has all bits cleared
20652   // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
20653 
20654   // AVX1 can't use the signsplat masking for 256-bit vectors - we have to
20655   // use the slightly slower blendv select instead.
20656   if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) {
20657     SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big);
20658     return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small);
20659   }
20660 
20661   SDValue IsOverflown =
20662       DAG.getNode(X86ISD::VSRAI, dl, VT, Small,
20663                   DAG.getTargetConstant(DstBits - 1, dl, MVT::i8));
20664   return DAG.getNode(ISD::OR, dl, VT, Small,
20665                      DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
20666 }
20667 
20668 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
20669   bool IsStrict = Op->isStrictFPOpcode();
20670   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
20671                   Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
20672   MVT VT = Op->getSimpleValueType(0);
20673   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
20674   SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue();
20675   MVT SrcVT = Src.getSimpleValueType();
20676   SDLoc dl(Op);
20677 
20678   SDValue Res;
20679   if (isSoftF16(SrcVT, Subtarget)) {
20680     MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
20681     if (IsStrict)
20682       return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
20683                          {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
20684                                              {NVT, MVT::Other}, {Chain, Src})});
20685     return DAG.getNode(Op.getOpcode(), dl, VT,
20686                        DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
20687   } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) {
20688     return Op;
20689   }
20690 
20691   if (VT.isVector()) {
20692     if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) {
20693       MVT ResVT = MVT::v4i32;
20694       MVT TruncVT = MVT::v4i1;
20695       unsigned Opc;
20696       if (IsStrict)
20697         Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
20698       else
20699         Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20700 
20701       if (!IsSigned && !Subtarget.hasVLX()) {
20702         assert(Subtarget.useAVX512Regs() && "Unexpected features!");
20703         // Widen to 512-bits.
20704         ResVT = MVT::v8i32;
20705         TruncVT = MVT::v8i1;
20706         Opc = Op.getOpcode();
20707         // Need to concat with zero vector for strict fp to avoid spurious
20708         // exceptions.
20709         // TODO: Should we just do this for non-strict as well?
20710         SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
20711                                : DAG.getUNDEF(MVT::v8f64);
20712         Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,
20713                           DAG.getIntPtrConstant(0, dl));
20714       }
20715       if (IsStrict) {
20716         Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src});
20717         Chain = Res.getValue(1);
20718       } else {
20719         Res = DAG.getNode(Opc, dl, ResVT, Src);
20720       }
20721 
20722       Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res);
20723       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,
20724                         DAG.getIntPtrConstant(0, dl));
20725       if (IsStrict)
20726         return DAG.getMergeValues({Res, Chain}, dl);
20727       return Res;
20728     }
20729 
20730     if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) {
20731       if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16)
20732         return Op;
20733 
20734       MVT ResVT = VT;
20735       MVT EleVT = VT.getVectorElementType();
20736       if (EleVT != MVT::i64)
20737         ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
20738 
20739       if (SrcVT != MVT::v8f16) {
20740         SDValue Tmp =
20741             IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
20742         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
20743         Ops[0] = Src;
20744         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
20745       }
20746 
20747       if (IsStrict) {
20748         Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI
20749                                    : X86ISD::STRICT_CVTTP2UI,
20750                           dl, {ResVT, MVT::Other}, {Chain, Src});
20751         Chain = Res.getValue(1);
20752       } else {
20753         Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl,
20754                           ResVT, Src);
20755       }
20756 
20757       // TODO: Need to add exception check code for strict FP.
20758       if (EleVT.getSizeInBits() < 16) {
20759         ResVT = MVT::getVectorVT(EleVT, 8);
20760         Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res);
20761       }
20762 
20763       if (ResVT != VT)
20764         Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20765                           DAG.getIntPtrConstant(0, dl));
20766 
20767       if (IsStrict)
20768         return DAG.getMergeValues({Res, Chain}, dl);
20769       return Res;
20770     }
20771 
20772     // v8f32/v16f32/v8f64->v8i16/v16i16 need to widen first.
20773     if (VT.getVectorElementType() == MVT::i16) {
20774       assert((SrcVT.getVectorElementType() == MVT::f32 ||
20775               SrcVT.getVectorElementType() == MVT::f64) &&
20776              "Expected f32/f64 vector!");
20777       MVT NVT = VT.changeVectorElementType(MVT::i32);
20778       if (IsStrict) {
20779         Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT
20780                                    : ISD::STRICT_FP_TO_UINT,
20781                           dl, {NVT, MVT::Other}, {Chain, Src});
20782         Chain = Res.getValue(1);
20783       } else {
20784         Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
20785                           NVT, Src);
20786       }
20787 
20788       // TODO: Need to add exception check code for strict FP.
20789       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20790 
20791       if (IsStrict)
20792         return DAG.getMergeValues({Res, Chain}, dl);
20793       return Res;
20794     }
20795 
20796     // v8f64->v8i32 is legal, but we need v8i32 to be custom for v8f32.
20797     if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) {
20798       assert(!IsSigned && "Expected unsigned conversion!");
20799       assert(Subtarget.useAVX512Regs() && "Requires avx512f");
20800       return Op;
20801     }
20802 
20803     // Widen vXi32 fp_to_uint with avx512f to 512-bit source.
20804     if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&
20805         (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32) &&
20806         Subtarget.useAVX512Regs()) {
20807       assert(!IsSigned && "Expected unsigned conversion!");
20808       assert(!Subtarget.hasVLX() && "Unexpected features!");
20809       MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
20810       MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
20811       // Need to concat with zero vector for strict fp to avoid spurious
20812       // exceptions.
20813       // TODO: Should we just do this for non-strict as well?
20814       SDValue Tmp =
20815           IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20816       Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20817                         DAG.getIntPtrConstant(0, dl));
20818 
20819       if (IsStrict) {
20820         Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
20821                           {Chain, Src});
20822         Chain = Res.getValue(1);
20823       } else {
20824         Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
20825       }
20826 
20827       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20828                         DAG.getIntPtrConstant(0, dl));
20829 
20830       if (IsStrict)
20831         return DAG.getMergeValues({Res, Chain}, dl);
20832       return Res;
20833     }
20834 
20835     // Widen vXi64 fp_to_uint/fp_to_sint with avx512dq to 512-bit source.
20836     if ((VT == MVT::v2i64 || VT == MVT::v4i64) &&
20837         (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32) &&
20838         Subtarget.useAVX512Regs() && Subtarget.hasDQI()) {
20839       assert(!Subtarget.hasVLX() && "Unexpected features!");
20840       MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
20841       // Need to concat with zero vector for strict fp to avoid spurious
20842       // exceptions.
20843       // TODO: Should we just do this for non-strict as well?
20844       SDValue Tmp =
20845           IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20846       Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20847                         DAG.getIntPtrConstant(0, dl));
20848 
20849       if (IsStrict) {
20850         Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
20851                           {Chain, Src});
20852         Chain = Res.getValue(1);
20853       } else {
20854         Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);
20855       }
20856 
20857       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20858                         DAG.getIntPtrConstant(0, dl));
20859 
20860       if (IsStrict)
20861         return DAG.getMergeValues({Res, Chain}, dl);
20862       return Res;
20863     }
20864 
20865     if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) {
20866       if (!Subtarget.hasVLX()) {
20867         // Non-strict nodes without VLX can we widened to v4f32->v4i64 by type
20868         // legalizer and then widened again by vector op legalization.
20869         if (!IsStrict)
20870           return SDValue();
20871 
20872         SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
20873         SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
20874                                   {Src, Zero, Zero, Zero});
20875         Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
20876                           {Chain, Tmp});
20877         SDValue Chain = Tmp.getValue(1);
20878         Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp,
20879                           DAG.getIntPtrConstant(0, dl));
20880         return DAG.getMergeValues({Tmp, Chain}, dl);
20881       }
20882 
20883       assert(Subtarget.hasDQI() && Subtarget.hasVLX() && "Requires AVX512DQVL");
20884       SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
20885                                 DAG.getUNDEF(MVT::v2f32));
20886       if (IsStrict) {
20887         unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
20888                                 : X86ISD::STRICT_CVTTP2UI;
20889         return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
20890       }
20891       unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20892       return DAG.getNode(Opc, dl, VT, Tmp);
20893     }
20894 
20895     // Generate optimized instructions for pre AVX512 unsigned conversions from
20896     // vXf32 to vXi32.
20897     if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) ||
20898         (VT == MVT::v4i32 && SrcVT == MVT::v4f64) ||
20899         (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) {
20900       assert(!IsSigned && "Expected unsigned conversion!");
20901       return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget);
20902     }
20903 
20904     return SDValue();
20905   }
20906 
20907   assert(!VT.isVector());
20908 
20909   bool UseSSEReg = isScalarFPTypeInSSEReg(SrcVT);
20910 
20911   if (!IsSigned && UseSSEReg) {
20912     // Conversions from f32/f64 with AVX512 should be legal.
20913     if (Subtarget.hasAVX512())
20914       return Op;
20915 
20916     // We can leverage the specific way the "cvttss2si/cvttsd2si" instruction
20917     // behaves on out of range inputs to generate optimized conversions.
20918     if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) ||
20919                       (VT == MVT::i64 && Subtarget.is64Bit()))) {
20920       unsigned DstBits = VT.getScalarSizeInBits();
20921       APInt UIntLimit = APInt::getSignMask(DstBits);
20922       SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT,
20923                                         DAG.getConstant(UIntLimit, dl, VT));
20924       MVT SrcVecVT = MVT::getVectorVT(SrcVT, 128 / SrcVT.getScalarSizeInBits());
20925 
20926       // Calculate the converted result for values in the range:
20927       // (i32) 0 to 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
20928       // (i64) 0 to 2^63-1 ("Small") and from 2^63 to 2^64-1 ("Big").
20929       SDValue Small =
20930           DAG.getNode(X86ISD::CVTTS2SI, dl, VT,
20931                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src));
20932       SDValue Big = DAG.getNode(
20933           X86ISD::CVTTS2SI, dl, VT,
20934           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT,
20935                       DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset)));
20936 
20937       // The "CVTTS2SI" instruction conveniently sets the sign bit if
20938       // and only if the value was out of range. So we can use that
20939       // as our indicator that we rather use "Big" instead of "Small".
20940       //
20941       // Use "Small" if "IsOverflown" has all bits cleared
20942       // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
20943       SDValue IsOverflown = DAG.getNode(
20944           ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));
20945       return DAG.getNode(ISD::OR, dl, VT, Small,
20946                          DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
20947     }
20948 
20949     // Use default expansion for i64.
20950     if (VT == MVT::i64)
20951       return SDValue();
20952 
20953     assert(VT == MVT::i32 && "Unexpected VT!");
20954 
20955     // Promote i32 to i64 and use a signed operation on 64-bit targets.
20956     // FIXME: This does not generate an invalid exception if the input does not
20957     // fit in i32. PR44019
20958     if (Subtarget.is64Bit()) {
20959       if (IsStrict) {
20960         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},
20961                           {Chain, Src});
20962         Chain = Res.getValue(1);
20963       } else
20964         Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
20965 
20966       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20967       if (IsStrict)
20968         return DAG.getMergeValues({Res, Chain}, dl);
20969       return Res;
20970     }
20971 
20972     // Use default expansion for SSE1/2 targets without SSE3. With SSE3 we can
20973     // use fisttp which will be handled later.
20974     if (!Subtarget.hasSSE3())
20975       return SDValue();
20976   }
20977 
20978   // Promote i16 to i32 if we can use a SSE operation or the type is f128.
20979   // FIXME: This does not generate an invalid exception if the input does not
20980   // fit in i16. PR44019
20981   if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) {
20982     assert(IsSigned && "Expected i16 FP_TO_UINT to have been promoted!");
20983     if (IsStrict) {
20984       Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other},
20985                         {Chain, Src});
20986       Chain = Res.getValue(1);
20987     } else
20988       Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
20989 
20990     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20991     if (IsStrict)
20992       return DAG.getMergeValues({Res, Chain}, dl);
20993     return Res;
20994   }
20995 
20996   // If this is a FP_TO_SINT using SSEReg we're done.
20997   if (UseSSEReg && IsSigned)
20998     return Op;
20999 
21000   // fp128 needs to use a libcall.
21001   if (SrcVT == MVT::f128) {
21002     RTLIB::Libcall LC;
21003     if (IsSigned)
21004       LC = RTLIB::getFPTOSINT(SrcVT, VT);
21005     else
21006       LC = RTLIB::getFPTOUINT(SrcVT, VT);
21007 
21008     MakeLibCallOptions CallOptions;
21009     std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions,
21010                                                   SDLoc(Op), Chain);
21011 
21012     if (IsStrict)
21013       return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
21014 
21015     return Tmp.first;
21016   }
21017 
21018   // Fall back to X87.
21019   if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
21020     if (IsStrict)
21021       return DAG.getMergeValues({V, Chain}, dl);
21022     return V;
21023   }
21024 
21025   llvm_unreachable("Expected FP_TO_INTHelper to handle all remaining cases.");
21026 }
21027 
21028 SDValue X86TargetLowering::LowerLRINT_LLRINT(SDValue Op,
21029                                              SelectionDAG &DAG) const {
21030   SDValue Src = Op.getOperand(0);
21031   MVT SrcVT = Src.getSimpleValueType();
21032 
21033   if (SrcVT == MVT::f16)
21034     return SDValue();
21035 
21036   // If the source is in an SSE register, the node is Legal.
21037   if (isScalarFPTypeInSSEReg(SrcVT))
21038     return Op;
21039 
21040   return LRINT_LLRINTHelper(Op.getNode(), DAG);
21041 }
21042 
21043 SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
21044                                               SelectionDAG &DAG) const {
21045   EVT DstVT = N->getValueType(0);
21046   SDValue Src = N->getOperand(0);
21047   EVT SrcVT = Src.getValueType();
21048 
21049   if (SrcVT != MVT::f32 && SrcVT != MVT::f64 && SrcVT != MVT::f80) {
21050     // f16 must be promoted before using the lowering in this routine.
21051     // fp128 does not use this lowering.
21052     return SDValue();
21053   }
21054 
21055   SDLoc DL(N);
21056   SDValue Chain = DAG.getEntryNode();
21057 
21058   bool UseSSE = isScalarFPTypeInSSEReg(SrcVT);
21059 
21060   // If we're converting from SSE, the stack slot needs to hold both types.
21061   // Otherwise it only needs to hold the DstVT.
21062   EVT OtherVT = UseSSE ? SrcVT : DstVT;
21063   SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
21064   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
21065   MachinePointerInfo MPI =
21066       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
21067 
21068   if (UseSSE) {
21069     assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");
21070     Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
21071     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
21072     SDValue Ops[] = { Chain, StackPtr };
21073 
21074     Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
21075                                   /*Align*/ std::nullopt,
21076                                   MachineMemOperand::MOLoad);
21077     Chain = Src.getValue(1);
21078   }
21079 
21080   SDValue StoreOps[] = { Chain, Src, StackPtr };
21081   Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
21082                                   StoreOps, DstVT, MPI, /*Align*/ std::nullopt,
21083                                   MachineMemOperand::MOStore);
21084 
21085   return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
21086 }
21087 
21088 SDValue
21089 X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
21090   // This is based on the TargetLowering::expandFP_TO_INT_SAT implementation,
21091   // but making use of X86 specifics to produce better instruction sequences.
21092   SDNode *Node = Op.getNode();
21093   bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
21094   unsigned FpToIntOpcode = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
21095   SDLoc dl(SDValue(Node, 0));
21096   SDValue Src = Node->getOperand(0);
21097 
21098   // There are three types involved here: SrcVT is the source floating point
21099   // type, DstVT is the type of the result, and TmpVT is the result of the
21100   // intermediate FP_TO_*INT operation we'll use (which may be a promotion of
21101   // DstVT).
21102   EVT SrcVT = Src.getValueType();
21103   EVT DstVT = Node->getValueType(0);
21104   EVT TmpVT = DstVT;
21105 
21106   // This code is only for floats and doubles. Fall back to generic code for
21107   // anything else.
21108   if (!isScalarFPTypeInSSEReg(SrcVT) || isSoftF16(SrcVT, Subtarget))
21109     return SDValue();
21110 
21111   EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
21112   unsigned SatWidth = SatVT.getScalarSizeInBits();
21113   unsigned DstWidth = DstVT.getScalarSizeInBits();
21114   unsigned TmpWidth = TmpVT.getScalarSizeInBits();
21115   assert(SatWidth <= DstWidth && SatWidth <= TmpWidth &&
21116          "Expected saturation width smaller than result width");
21117 
21118   // Promote result of FP_TO_*INT to at least 32 bits.
21119   if (TmpWidth < 32) {
21120     TmpVT = MVT::i32;
21121     TmpWidth = 32;
21122   }
21123 
21124   // Promote conversions to unsigned 32-bit to 64-bit, because it will allow
21125   // us to use a native signed conversion instead.
21126   if (SatWidth == 32 && !IsSigned && Subtarget.is64Bit()) {
21127     TmpVT = MVT::i64;
21128     TmpWidth = 64;
21129   }
21130 
21131   // If the saturation width is smaller than the size of the temporary result,
21132   // we can always use signed conversion, which is native.
21133   if (SatWidth < TmpWidth)
21134     FpToIntOpcode = ISD::FP_TO_SINT;
21135 
21136   // Determine minimum and maximum integer values and their corresponding
21137   // floating-point values.
21138   APInt MinInt, MaxInt;
21139   if (IsSigned) {
21140     MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
21141     MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
21142   } else {
21143     MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
21144     MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
21145   }
21146 
21147   APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
21148   APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
21149 
21150   APFloat::opStatus MinStatus = MinFloat.convertFromAPInt(
21151     MinInt, IsSigned, APFloat::rmTowardZero);
21152   APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt(
21153     MaxInt, IsSigned, APFloat::rmTowardZero);
21154   bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact)
21155                           && !(MaxStatus & APFloat::opStatus::opInexact);
21156 
21157   SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
21158   SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
21159 
21160   // If the integer bounds are exactly representable as floats, emit a
21161   // min+max+fptoi sequence. Otherwise use comparisons and selects.
21162   if (AreExactFloatBounds) {
21163     if (DstVT != TmpVT) {
21164       // Clamp by MinFloat from below. If Src is NaN, propagate NaN.
21165       SDValue MinClamped = DAG.getNode(
21166         X86ISD::FMAX, dl, SrcVT, MinFloatNode, Src);
21167       // Clamp by MaxFloat from above. If Src is NaN, propagate NaN.
21168       SDValue BothClamped = DAG.getNode(
21169         X86ISD::FMIN, dl, SrcVT, MaxFloatNode, MinClamped);
21170       // Convert clamped value to integer.
21171       SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped);
21172 
21173       // NaN will become INDVAL, with the top bit set and the rest zero.
21174       // Truncation will discard the top bit, resulting in zero.
21175       return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21176     }
21177 
21178     // Clamp by MinFloat from below. If Src is NaN, the result is MinFloat.
21179     SDValue MinClamped = DAG.getNode(
21180       X86ISD::FMAX, dl, SrcVT, Src, MinFloatNode);
21181     // Clamp by MaxFloat from above. NaN cannot occur.
21182     SDValue BothClamped = DAG.getNode(
21183       X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
21184     // Convert clamped value to integer.
21185     SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
21186 
21187     if (!IsSigned) {
21188       // In the unsigned case we're done, because we mapped NaN to MinFloat,
21189       // which is zero.
21190       return FpToInt;
21191     }
21192 
21193     // Otherwise, select zero if Src is NaN.
21194     SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21195     return DAG.getSelectCC(
21196       dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
21197   }
21198 
21199   SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
21200   SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
21201 
21202   // Result of direct conversion, which may be selected away.
21203   SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src);
21204 
21205   if (DstVT != TmpVT) {
21206     // NaN will become INDVAL, with the top bit set and the rest zero.
21207     // Truncation will discard the top bit, resulting in zero.
21208     FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21209   }
21210 
21211   SDValue Select = FpToInt;
21212   // For signed conversions where we saturate to the same size as the
21213   // result type of the fptoi instructions, INDVAL coincides with integer
21214   // minimum, so we don't need to explicitly check it.
21215   if (!IsSigned || SatWidth != TmpVT.getScalarSizeInBits()) {
21216     // If Src ULT MinFloat, select MinInt. In particular, this also selects
21217     // MinInt if Src is NaN.
21218     Select = DAG.getSelectCC(
21219       dl, Src, MinFloatNode, MinIntNode, Select, ISD::CondCode::SETULT);
21220   }
21221 
21222   // If Src OGT MaxFloat, select MaxInt.
21223   Select = DAG.getSelectCC(
21224     dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT);
21225 
21226   // In the unsigned case we are done, because we mapped NaN to MinInt, which
21227   // is already zero. The promoted case was already handled above.
21228   if (!IsSigned || DstVT != TmpVT) {
21229     return Select;
21230   }
21231 
21232   // Otherwise, select 0 if Src is NaN.
21233   SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21234   return DAG.getSelectCC(
21235     dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
21236 }
21237 
21238 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21239   bool IsStrict = Op->isStrictFPOpcode();
21240 
21241   SDLoc DL(Op);
21242   MVT VT = Op.getSimpleValueType();
21243   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21244   SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21245   MVT SVT = In.getSimpleValueType();
21246 
21247   // Let f16->f80 get lowered to a libcall, except for darwin, where we should
21248   // lower it to an fp_extend via f32 (as only f16<>f32 libcalls are available)
21249   if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80 &&
21250                           !Subtarget.getTargetTriple().isOSDarwin()))
21251     return SDValue();
21252 
21253   if ((SVT == MVT::v8f16 && Subtarget.hasF16C()) ||
21254       (SVT == MVT::v16f16 && Subtarget.useAVX512Regs()))
21255     return Op;
21256 
21257   if (SVT == MVT::f16) {
21258     if (Subtarget.hasFP16())
21259       return Op;
21260 
21261     if (VT != MVT::f32) {
21262       if (IsStrict)
21263         return DAG.getNode(
21264             ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other},
21265             {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL,
21266                                 {MVT::f32, MVT::Other}, {Chain, In})});
21267 
21268       return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21269                          DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In));
21270     }
21271 
21272     if (!Subtarget.hasF16C()) {
21273       if (!Subtarget.getTargetTriple().isOSDarwin())
21274         return SDValue();
21275 
21276       assert(VT == MVT::f32 && SVT == MVT::f16 && "unexpected extend libcall");
21277 
21278       // Need a libcall, but ABI for f16 is soft-float on MacOS.
21279       TargetLowering::CallLoweringInfo CLI(DAG);
21280       Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21281 
21282       In = DAG.getBitcast(MVT::i16, In);
21283       TargetLowering::ArgListTy Args;
21284       TargetLowering::ArgListEntry Entry;
21285       Entry.Node = In;
21286       Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext());
21287       Entry.IsSExt = false;
21288       Entry.IsZExt = true;
21289       Args.push_back(Entry);
21290 
21291       SDValue Callee = DAG.getExternalSymbol(
21292           getLibcallName(RTLIB::FPEXT_F16_F32),
21293           getPointerTy(DAG.getDataLayout()));
21294       CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21295           CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee,
21296           std::move(Args));
21297 
21298       SDValue Res;
21299       std::tie(Res,Chain) = LowerCallTo(CLI);
21300       if (IsStrict)
21301         Res = DAG.getMergeValues({Res, Chain}, DL);
21302 
21303       return Res;
21304     }
21305 
21306     In = DAG.getBitcast(MVT::i16, In);
21307     In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,
21308                      getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In,
21309                      DAG.getIntPtrConstant(0, DL));
21310     SDValue Res;
21311     if (IsStrict) {
21312       Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other},
21313                         {Chain, In});
21314       Chain = Res.getValue(1);
21315     } else {
21316       Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,
21317                         DAG.getTargetConstant(4, DL, MVT::i32));
21318     }
21319     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res,
21320                       DAG.getIntPtrConstant(0, DL));
21321     if (IsStrict)
21322       return DAG.getMergeValues({Res, Chain}, DL);
21323     return Res;
21324   }
21325 
21326   if (!SVT.isVector())
21327     return Op;
21328 
21329   if (SVT.getVectorElementType() == MVT::bf16) {
21330     // FIXME: Do we need to support strict FP?
21331     assert(!IsStrict && "Strict FP doesn't support BF16");
21332     if (VT.getVectorElementType() == MVT::f64) {
21333       MVT TmpVT = VT.changeVectorElementType(MVT::f32);
21334       return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21335                          DAG.getNode(ISD::FP_EXTEND, DL, TmpVT, In));
21336     }
21337     assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
21338     MVT NVT = SVT.changeVectorElementType(MVT::i32);
21339     In = DAG.getBitcast(SVT.changeTypeToInteger(), In);
21340     In = DAG.getNode(ISD::ZERO_EXTEND, DL, NVT, In);
21341     In = DAG.getNode(ISD::SHL, DL, NVT, In, DAG.getConstant(16, DL, NVT));
21342     return DAG.getBitcast(VT, In);
21343   }
21344 
21345   if (SVT.getVectorElementType() == MVT::f16) {
21346     if (Subtarget.hasFP16() && isTypeLegal(SVT))
21347       return Op;
21348     assert(Subtarget.hasF16C() && "Unexpected features!");
21349     if (SVT == MVT::v2f16)
21350       In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In,
21351                        DAG.getUNDEF(MVT::v2f16));
21352     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,
21353                               DAG.getUNDEF(MVT::v4f16));
21354     if (IsStrict)
21355       return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21356                          {Op->getOperand(0), Res});
21357     return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21358   } else if (VT == MVT::v4f64 || VT == MVT::v8f64) {
21359     return Op;
21360   }
21361 
21362   assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
21363 
21364   SDValue Res =
21365       DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
21366   if (IsStrict)
21367     return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21368                        {Op->getOperand(0), Res});
21369   return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21370 }
21371 
21372 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21373   bool IsStrict = Op->isStrictFPOpcode();
21374 
21375   SDLoc DL(Op);
21376   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21377   SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21378   MVT VT = Op.getSimpleValueType();
21379   MVT SVT = In.getSimpleValueType();
21380 
21381   if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80))
21382     return SDValue();
21383 
21384   if (VT == MVT::f16 && (SVT == MVT::f64 || SVT == MVT::f32) &&
21385       !Subtarget.hasFP16() && (SVT == MVT::f64 || !Subtarget.hasF16C())) {
21386     if (!Subtarget.getTargetTriple().isOSDarwin())
21387       return SDValue();
21388 
21389     // We need a libcall but the ABI for f16 libcalls on MacOS is soft.
21390     TargetLowering::CallLoweringInfo CLI(DAG);
21391     Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21392 
21393     TargetLowering::ArgListTy Args;
21394     TargetLowering::ArgListEntry Entry;
21395     Entry.Node = In;
21396     Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext());
21397     Entry.IsSExt = false;
21398     Entry.IsZExt = true;
21399     Args.push_back(Entry);
21400 
21401     SDValue Callee = DAG.getExternalSymbol(
21402         getLibcallName(SVT == MVT::f64 ? RTLIB::FPROUND_F64_F16
21403                                        : RTLIB::FPROUND_F32_F16),
21404         getPointerTy(DAG.getDataLayout()));
21405     CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21406         CallingConv::C, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()), Callee,
21407         std::move(Args));
21408 
21409     SDValue Res;
21410     std::tie(Res, Chain) = LowerCallTo(CLI);
21411 
21412     Res = DAG.getBitcast(MVT::f16, Res);
21413 
21414     if (IsStrict)
21415       Res = DAG.getMergeValues({Res, Chain}, DL);
21416 
21417     return Res;
21418   }
21419 
21420   if (VT.getScalarType() == MVT::bf16) {
21421     if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT))
21422       return Op;
21423     return SDValue();
21424   }
21425 
21426   if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) {
21427     if (!Subtarget.hasF16C() || SVT.getScalarType() != MVT::f32)
21428       return SDValue();
21429 
21430     if (VT.isVector())
21431       return Op;
21432 
21433     SDValue Res;
21434     SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
21435                                         MVT::i32);
21436     if (IsStrict) {
21437       Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32,
21438                         DAG.getConstantFP(0, DL, MVT::v4f32), In,
21439                         DAG.getIntPtrConstant(0, DL));
21440       Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other},
21441                         {Chain, Res, Rnd});
21442       Chain = Res.getValue(1);
21443     } else {
21444       // FIXME: Should we use zeros for upper elements for non-strict?
21445       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In);
21446       Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd);
21447     }
21448 
21449     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
21450                       DAG.getIntPtrConstant(0, DL));
21451     Res = DAG.getBitcast(MVT::f16, Res);
21452 
21453     if (IsStrict)
21454       return DAG.getMergeValues({Res, Chain}, DL);
21455 
21456     return Res;
21457   }
21458 
21459   return Op;
21460 }
21461 
21462 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
21463   bool IsStrict = Op->isStrictFPOpcode();
21464   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21465   assert(Src.getValueType() == MVT::i16 && Op.getValueType() == MVT::f32 &&
21466          "Unexpected VT!");
21467 
21468   SDLoc dl(Op);
21469   SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
21470                             DAG.getConstant(0, dl, MVT::v8i16), Src,
21471                             DAG.getIntPtrConstant(0, dl));
21472 
21473   SDValue Chain;
21474   if (IsStrict) {
21475     Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},
21476                       {Op.getOperand(0), Res});
21477     Chain = Res.getValue(1);
21478   } else {
21479     Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
21480   }
21481 
21482   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
21483                     DAG.getIntPtrConstant(0, dl));
21484 
21485   if (IsStrict)
21486     return DAG.getMergeValues({Res, Chain}, dl);
21487 
21488   return Res;
21489 }
21490 
21491 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
21492   bool IsStrict = Op->isStrictFPOpcode();
21493   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21494   assert(Src.getValueType() == MVT::f32 && Op.getValueType() == MVT::i16 &&
21495          "Unexpected VT!");
21496 
21497   SDLoc dl(Op);
21498   SDValue Res, Chain;
21499   if (IsStrict) {
21500     Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,
21501                       DAG.getConstantFP(0, dl, MVT::v4f32), Src,
21502                       DAG.getIntPtrConstant(0, dl));
21503     Res = DAG.getNode(
21504         X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
21505         {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});
21506     Chain = Res.getValue(1);
21507   } else {
21508     // FIXME: Should we use zeros for upper elements for non-strict?
21509     Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);
21510     Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
21511                       DAG.getTargetConstant(4, dl, MVT::i32));
21512   }
21513 
21514   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
21515                     DAG.getIntPtrConstant(0, dl));
21516 
21517   if (IsStrict)
21518     return DAG.getMergeValues({Res, Chain}, dl);
21519 
21520   return Res;
21521 }
21522 
21523 SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
21524                                            SelectionDAG &DAG) const {
21525   SDLoc DL(Op);
21526   MakeLibCallOptions CallOptions;
21527   RTLIB::Libcall LC =
21528       RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);
21529   SDValue Res =
21530       makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first;
21531   return DAG.getBitcast(MVT::i16, Res);
21532 }
21533 
21534 /// Depending on uarch and/or optimizing for size, we might prefer to use a
21535 /// vector operation in place of the typical scalar operation.
21536 static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
21537                                          const X86Subtarget &Subtarget) {
21538   // If both operands have other uses, this is probably not profitable.
21539   SDValue LHS = Op.getOperand(0);
21540   SDValue RHS = Op.getOperand(1);
21541   if (!LHS.hasOneUse() && !RHS.hasOneUse())
21542     return Op;
21543 
21544   // FP horizontal add/sub were added with SSE3. Integer with SSSE3.
21545   bool IsFP = Op.getSimpleValueType().isFloatingPoint();
21546   if (IsFP && !Subtarget.hasSSE3())
21547     return Op;
21548   if (!IsFP && !Subtarget.hasSSSE3())
21549     return Op;
21550 
21551   // Extract from a common vector.
21552   if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21553       RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21554       LHS.getOperand(0) != RHS.getOperand(0) ||
21555       !isa<ConstantSDNode>(LHS.getOperand(1)) ||
21556       !isa<ConstantSDNode>(RHS.getOperand(1)) ||
21557       !shouldUseHorizontalOp(true, DAG, Subtarget))
21558     return Op;
21559 
21560   // Allow commuted 'hadd' ops.
21561   // TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
21562   unsigned HOpcode;
21563   switch (Op.getOpcode()) {
21564     case ISD::ADD: HOpcode = X86ISD::HADD; break;
21565     case ISD::SUB: HOpcode = X86ISD::HSUB; break;
21566     case ISD::FADD: HOpcode = X86ISD::FHADD; break;
21567     case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
21568     default:
21569       llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
21570   }
21571   unsigned LExtIndex = LHS.getConstantOperandVal(1);
21572   unsigned RExtIndex = RHS.getConstantOperandVal(1);
21573   if ((LExtIndex & 1) == 1 && (RExtIndex & 1) == 0 &&
21574       (HOpcode == X86ISD::HADD || HOpcode == X86ISD::FHADD))
21575     std::swap(LExtIndex, RExtIndex);
21576 
21577   if ((LExtIndex & 1) != 0 || RExtIndex != (LExtIndex + 1))
21578     return Op;
21579 
21580   SDValue X = LHS.getOperand(0);
21581   EVT VecVT = X.getValueType();
21582   unsigned BitWidth = VecVT.getSizeInBits();
21583   unsigned NumLanes = BitWidth / 128;
21584   unsigned NumEltsPerLane = VecVT.getVectorNumElements() / NumLanes;
21585   assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
21586          "Not expecting illegal vector widths here");
21587 
21588   // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
21589   // equivalent, so extract the 256/512-bit source op to 128-bit if we can.
21590   SDLoc DL(Op);
21591   if (BitWidth == 256 || BitWidth == 512) {
21592     unsigned LaneIdx = LExtIndex / NumEltsPerLane;
21593     X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
21594     LExtIndex %= NumEltsPerLane;
21595   }
21596 
21597   // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
21598   // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
21599   // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1
21600   // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
21601   SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
21602   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
21603                      DAG.getIntPtrConstant(LExtIndex / 2, DL));
21604 }
21605 
21606 /// Depending on uarch and/or optimizing for size, we might prefer to use a
21607 /// vector operation in place of the typical scalar operation.
21608 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
21609   assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&
21610          "Only expecting float/double");
21611   return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
21612 }
21613 
21614 /// ISD::FROUND is defined to round to nearest with ties rounding away from 0.
21615 /// This mode isn't supported in hardware on X86. But as long as we aren't
21616 /// compiling with trapping math, we can emulate this with
21617 /// trunc(X + copysign(nextafter(0.5, 0.0), X)).
21618 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
21619   SDValue N0 = Op.getOperand(0);
21620   SDLoc dl(Op);
21621   MVT VT = Op.getSimpleValueType();
21622 
21623   // N0 += copysign(nextafter(0.5, 0.0), N0)
21624   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21625   bool Ignored;
21626   APFloat Point5Pred = APFloat(0.5f);
21627   Point5Pred.convert(Sem, APFloat::rmNearestTiesToEven, &Ignored);
21628   Point5Pred.next(/*nextDown*/true);
21629 
21630   SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
21631                               DAG.getConstantFP(Point5Pred, dl, VT), N0);
21632   N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder);
21633 
21634   // Truncate the result to remove fraction.
21635   return DAG.getNode(ISD::FTRUNC, dl, VT, N0);
21636 }
21637 
21638 /// The only differences between FABS and FNEG are the mask and the logic op.
21639 /// FNEG also has a folding opportunity for FNEG(FABS(x)).
21640 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
21641   assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
21642          "Wrong opcode for lowering FABS or FNEG.");
21643 
21644   bool IsFABS = (Op.getOpcode() == ISD::FABS);
21645 
21646   // If this is a FABS and it has an FNEG user, bail out to fold the combination
21647   // into an FNABS. We'll lower the FABS after that if it is still in use.
21648   if (IsFABS)
21649     for (SDNode *User : Op->uses())
21650       if (User->getOpcode() == ISD::FNEG)
21651         return Op;
21652 
21653   SDLoc dl(Op);
21654   MVT VT = Op.getSimpleValueType();
21655 
21656   bool IsF128 = (VT == MVT::f128);
21657   assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21658          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21659          "Unexpected type in LowerFABSorFNEG");
21660 
21661   // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOptLevel to
21662   // decide if we should generate a 16-byte constant mask when we only need 4 or
21663   // 8 bytes for the scalar case.
21664 
21665   // There are no scalar bitwise logical SSE/AVX instructions, so we
21666   // generate a 16-byte vector constant and logic op even for the scalar case.
21667   // Using a 16-byte mask allows folding the load of the mask with
21668   // the logic op, so it can save (~4 bytes) on code size.
21669   bool IsFakeVector = !VT.isVector() && !IsF128;
21670   MVT LogicVT = VT;
21671   if (IsFakeVector)
21672     LogicVT = (VT == MVT::f64)   ? MVT::v2f64
21673               : (VT == MVT::f32) ? MVT::v4f32
21674                                  : MVT::v8f16;
21675 
21676   unsigned EltBits = VT.getScalarSizeInBits();
21677   // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
21678   APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
21679                            APInt::getSignMask(EltBits);
21680   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21681   SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
21682 
21683   SDValue Op0 = Op.getOperand(0);
21684   bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
21685   unsigned LogicOp = IsFABS  ? X86ISD::FAND :
21686                      IsFNABS ? X86ISD::FOR  :
21687                                X86ISD::FXOR;
21688   SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
21689 
21690   if (VT.isVector() || IsF128)
21691     return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21692 
21693   // For the scalar case extend to a 128-bit vector, perform the logic op,
21694   // and extract the scalar result back out.
21695   Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
21696   SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21697   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
21698                      DAG.getIntPtrConstant(0, dl));
21699 }
21700 
21701 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
21702   SDValue Mag = Op.getOperand(0);
21703   SDValue Sign = Op.getOperand(1);
21704   SDLoc dl(Op);
21705 
21706   // If the sign operand is smaller, extend it first.
21707   MVT VT = Op.getSimpleValueType();
21708   if (Sign.getSimpleValueType().bitsLT(VT))
21709     Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);
21710 
21711   // And if it is bigger, shrink it first.
21712   if (Sign.getSimpleValueType().bitsGT(VT))
21713     Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign,
21714                        DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
21715 
21716   // At this point the operands and the result should have the same
21717   // type, and that won't be f80 since that is not custom lowered.
21718   bool IsF128 = (VT == MVT::f128);
21719   assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21720          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21721          "Unexpected type in LowerFCOPYSIGN");
21722 
21723   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21724 
21725   // Perform all scalar logic operations as 16-byte vectors because there are no
21726   // scalar FP logic instructions in SSE.
21727   // TODO: This isn't necessary. If we used scalar types, we might avoid some
21728   // unnecessary splats, but we might miss load folding opportunities. Should
21729   // this decision be based on OptimizeForSize?
21730   bool IsFakeVector = !VT.isVector() && !IsF128;
21731   MVT LogicVT = VT;
21732   if (IsFakeVector)
21733     LogicVT = (VT == MVT::f64)   ? MVT::v2f64
21734               : (VT == MVT::f32) ? MVT::v4f32
21735                                  : MVT::v8f16;
21736 
21737   // The mask constants are automatically splatted for vector types.
21738   unsigned EltSizeInBits = VT.getScalarSizeInBits();
21739   SDValue SignMask = DAG.getConstantFP(
21740       APFloat(Sem, APInt::getSignMask(EltSizeInBits)), dl, LogicVT);
21741   SDValue MagMask = DAG.getConstantFP(
21742       APFloat(Sem, APInt::getSignedMaxValue(EltSizeInBits)), dl, LogicVT);
21743 
21744   // First, clear all bits but the sign bit from the second operand (sign).
21745   if (IsFakeVector)
21746     Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign);
21747   SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
21748 
21749   // Next, clear the sign bit from the first operand (magnitude).
21750   // TODO: If we had general constant folding for FP logic ops, this check
21751   // wouldn't be necessary.
21752   SDValue MagBits;
21753   if (ConstantFPSDNode *Op0CN = isConstOrConstSplatFP(Mag)) {
21754     APFloat APF = Op0CN->getValueAPF();
21755     APF.clearSign();
21756     MagBits = DAG.getConstantFP(APF, dl, LogicVT);
21757   } else {
21758     // If the magnitude operand wasn't a constant, we need to AND out the sign.
21759     if (IsFakeVector)
21760       Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag);
21761     MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
21762   }
21763 
21764   // OR the magnitude value with the sign bit.
21765   SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
21766   return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,
21767                                           DAG.getIntPtrConstant(0, dl));
21768 }
21769 
21770 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
21771   SDValue N0 = Op.getOperand(0);
21772   SDLoc dl(Op);
21773   MVT VT = Op.getSimpleValueType();
21774 
21775   MVT OpVT = N0.getSimpleValueType();
21776   assert((OpVT == MVT::f32 || OpVT == MVT::f64) &&
21777          "Unexpected type for FGETSIGN");
21778 
21779   // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
21780   MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64);
21781   SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
21782   Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
21783   Res = DAG.getZExtOrTrunc(Res, dl, VT);
21784   Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));
21785   return Res;
21786 }
21787 
21788 /// Helper for attempting to create a X86ISD::BT node.
21789 static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
21790   // If Src is i8, promote it to i32 with any_extend.  There is no i8 BT
21791   // instruction.  Since the shift amount is in-range-or-undefined, we know
21792   // that doing a bittest on the i32 value is ok.  We extend to i32 because
21793   // the encoding for the i16 version is larger than the i32 version.
21794   // Also promote i16 to i32 for performance / code size reason.
21795   if (Src.getValueType().getScalarSizeInBits() < 32)
21796     Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
21797 
21798   // No legal type found, give up.
21799   if (!DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
21800     return SDValue();
21801 
21802   // See if we can use the 32-bit instruction instead of the 64-bit one for a
21803   // shorter encoding. Since the former takes the modulo 32 of BitNo and the
21804   // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
21805   // known to be zero.
21806   if (Src.getValueType() == MVT::i64 &&
21807       DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
21808     Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src);
21809 
21810   // If the operand types disagree, extend the shift amount to match.  Since
21811   // BT ignores high bits (like shifts) we can use anyextend.
21812   if (Src.getValueType() != BitNo.getValueType()) {
21813     // Peek through a mask/modulo operation.
21814     // TODO: DAGCombine fails to do this as it just checks isTruncateFree, but
21815     // we probably need a better IsDesirableToPromoteOp to handle this as well.
21816     if (BitNo.getOpcode() == ISD::AND && BitNo->hasOneUse())
21817       BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(),
21818                           DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
21819                                       BitNo.getOperand(0)),
21820                           DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
21821                                       BitNo.getOperand(1)));
21822     else
21823       BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo);
21824   }
21825 
21826   return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo);
21827 }
21828 
21829 /// Helper for creating a X86ISD::SETCC node.
21830 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
21831                         SelectionDAG &DAG) {
21832   return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
21833                      DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
21834 }
21835 
21836 /// Recursive helper for combineVectorSizedSetCCEquality() to see if we have a
21837 /// recognizable memcmp expansion.
21838 static bool isOrXorXorTree(SDValue X, bool Root = true) {
21839   if (X.getOpcode() == ISD::OR)
21840     return isOrXorXorTree(X.getOperand(0), false) &&
21841            isOrXorXorTree(X.getOperand(1), false);
21842   if (Root)
21843     return false;
21844   return X.getOpcode() == ISD::XOR;
21845 }
21846 
21847 /// Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp
21848 /// expansion.
21849 template <typename F>
21850 static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG,
21851                                 EVT VecVT, EVT CmpVT, bool HasPT, F SToV) {
21852   SDValue Op0 = X.getOperand(0);
21853   SDValue Op1 = X.getOperand(1);
21854   if (X.getOpcode() == ISD::OR) {
21855     SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
21856     SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
21857     if (VecVT != CmpVT)
21858       return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
21859     if (HasPT)
21860       return DAG.getNode(ISD::OR, DL, VecVT, A, B);
21861     return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
21862   }
21863   if (X.getOpcode() == ISD::XOR) {
21864     SDValue A = SToV(Op0);
21865     SDValue B = SToV(Op1);
21866     if (VecVT != CmpVT)
21867       return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
21868     if (HasPT)
21869       return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
21870     return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
21871   }
21872   llvm_unreachable("Impossible");
21873 }
21874 
21875 /// Try to map a 128-bit or larger integer comparison to vector instructions
21876 /// before type legalization splits it up into chunks.
21877 static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y,
21878                                                ISD::CondCode CC,
21879                                                const SDLoc &DL,
21880                                                SelectionDAG &DAG,
21881                                                const X86Subtarget &Subtarget) {
21882   assert((CC == ISD::SETNE || CC == ISD::SETEQ) && "Bad comparison predicate");
21883 
21884   // We're looking for an oversized integer equality comparison.
21885   EVT OpVT = X.getValueType();
21886   unsigned OpSize = OpVT.getSizeInBits();
21887   if (!OpVT.isScalarInteger() || OpSize < 128)
21888     return SDValue();
21889 
21890   // Ignore a comparison with zero because that gets special treatment in
21891   // EmitTest(). But make an exception for the special case of a pair of
21892   // logically-combined vector-sized operands compared to zero. This pattern may
21893   // be generated by the memcmp expansion pass with oversized integer compares
21894   // (see PR33325).
21895   bool IsOrXorXorTreeCCZero = isNullConstant(Y) && isOrXorXorTree(X);
21896   if (isNullConstant(Y) && !IsOrXorXorTreeCCZero)
21897     return SDValue();
21898 
21899   // Don't perform this combine if constructing the vector will be expensive.
21900   auto IsVectorBitCastCheap = [](SDValue X) {
21901     X = peekThroughBitcasts(X);
21902     return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
21903            X.getOpcode() == ISD::LOAD;
21904   };
21905   if ((!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y)) &&
21906       !IsOrXorXorTreeCCZero)
21907     return SDValue();
21908 
21909   // Use XOR (plus OR) and PTEST after SSE4.1 for 128/256-bit operands.
21910   // Use PCMPNEQ (plus OR) and KORTEST for 512-bit operands.
21911   // Otherwise use PCMPEQ (plus AND) and mask testing.
21912   bool NoImplicitFloatOps =
21913       DAG.getMachineFunction().getFunction().hasFnAttribute(
21914           Attribute::NoImplicitFloat);
21915   if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
21916       ((OpSize == 128 && Subtarget.hasSSE2()) ||
21917        (OpSize == 256 && Subtarget.hasAVX()) ||
21918        (OpSize == 512 && Subtarget.useAVX512Regs()))) {
21919     bool HasPT = Subtarget.hasSSE41();
21920 
21921     // PTEST and MOVMSK are slow on Knights Landing and Knights Mill and widened
21922     // vector registers are essentially free. (Technically, widening registers
21923     // prevents load folding, but the tradeoff is worth it.)
21924     bool PreferKOT = Subtarget.preferMaskRegisters();
21925     bool NeedZExt = PreferKOT && !Subtarget.hasVLX() && OpSize != 512;
21926 
21927     EVT VecVT = MVT::v16i8;
21928     EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;
21929     if (OpSize == 256) {
21930       VecVT = MVT::v32i8;
21931       CmpVT = PreferKOT ? MVT::v32i1 : VecVT;
21932     }
21933     EVT CastVT = VecVT;
21934     bool NeedsAVX512FCast = false;
21935     if (OpSize == 512 || NeedZExt) {
21936       if (Subtarget.hasBWI()) {
21937         VecVT = MVT::v64i8;
21938         CmpVT = MVT::v64i1;
21939         if (OpSize == 512)
21940           CastVT = VecVT;
21941       } else {
21942         VecVT = MVT::v16i32;
21943         CmpVT = MVT::v16i1;
21944         CastVT = OpSize == 512   ? VecVT
21945                  : OpSize == 256 ? MVT::v8i32
21946                                  : MVT::v4i32;
21947         NeedsAVX512FCast = true;
21948       }
21949     }
21950 
21951     auto ScalarToVector = [&](SDValue X) -> SDValue {
21952       bool TmpZext = false;
21953       EVT TmpCastVT = CastVT;
21954       if (X.getOpcode() == ISD::ZERO_EXTEND) {
21955         SDValue OrigX = X.getOperand(0);
21956         unsigned OrigSize = OrigX.getScalarValueSizeInBits();
21957         if (OrigSize < OpSize) {
21958           if (OrigSize == 128) {
21959             TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;
21960             X = OrigX;
21961             TmpZext = true;
21962           } else if (OrigSize == 256) {
21963             TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;
21964             X = OrigX;
21965             TmpZext = true;
21966           }
21967         }
21968       }
21969       X = DAG.getBitcast(TmpCastVT, X);
21970       if (!NeedZExt && !TmpZext)
21971         return X;
21972       return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
21973                          DAG.getConstant(0, DL, VecVT), X,
21974                          DAG.getVectorIdxConstant(0, DL));
21975     };
21976 
21977     SDValue Cmp;
21978     if (IsOrXorXorTreeCCZero) {
21979       // This is a bitwise-combined equality comparison of 2 pairs of vectors:
21980       // setcc i128 (or (xor A, B), (xor C, D)), 0, eq|ne
21981       // Use 2 vector equality compares and 'and' the results before doing a
21982       // MOVMSK.
21983       Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
21984     } else {
21985       SDValue VecX = ScalarToVector(X);
21986       SDValue VecY = ScalarToVector(Y);
21987       if (VecVT != CmpVT) {
21988         Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
21989       } else if (HasPT) {
21990         Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
21991       } else {
21992         Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
21993       }
21994     }
21995     // AVX512 should emit a setcc that will lower to kortest.
21996     if (VecVT != CmpVT) {
21997       EVT KRegVT = CmpVT == MVT::v64i1   ? MVT::i64
21998                    : CmpVT == MVT::v32i1 ? MVT::i32
21999                                          : MVT::i16;
22000       return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
22001                           DAG.getConstant(0, DL, KRegVT), CC);
22002     }
22003     if (HasPT) {
22004       SDValue BCCmp =
22005           DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64, Cmp);
22006       SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
22007       X86::CondCode X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
22008       SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
22009       return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0));
22010     }
22011     // If all bytes match (bitmask is 0x(FFFF)FFFF), that's equality.
22012     // setcc i128 X, Y, eq --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, eq
22013     // setcc i128 X, Y, ne --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, ne
22014     assert(Cmp.getValueType() == MVT::v16i8 &&
22015            "Non 128-bit vector on pre-SSE41 target");
22016     SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
22017     SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
22018     return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
22019   }
22020 
22021   return SDValue();
22022 }
22023 
22024 /// Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...))
22025 /// style scalarized (associative) reduction patterns. Partial reductions
22026 /// are supported when the pointer SrcMask is non-null.
22027 /// TODO - move this to SelectionDAG?
22028 static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
22029                                  SmallVectorImpl<SDValue> &SrcOps,
22030                                  SmallVectorImpl<APInt> *SrcMask = nullptr) {
22031   SmallVector<SDValue, 8> Opnds;
22032   DenseMap<SDValue, APInt> SrcOpMap;
22033   EVT VT = MVT::Other;
22034 
22035   // Recognize a special case where a vector is casted into wide integer to
22036   // test all 0s.
22037   assert(Op.getOpcode() == unsigned(BinOp) &&
22038          "Unexpected bit reduction opcode");
22039   Opnds.push_back(Op.getOperand(0));
22040   Opnds.push_back(Op.getOperand(1));
22041 
22042   for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
22043     SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
22044     // BFS traverse all BinOp operands.
22045     if (I->getOpcode() == unsigned(BinOp)) {
22046       Opnds.push_back(I->getOperand(0));
22047       Opnds.push_back(I->getOperand(1));
22048       // Re-evaluate the number of nodes to be traversed.
22049       e += 2; // 2 more nodes (LHS and RHS) are pushed.
22050       continue;
22051     }
22052 
22053     // Quit if a non-EXTRACT_VECTOR_ELT
22054     if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
22055       return false;
22056 
22057     // Quit if without a constant index.
22058     auto *Idx = dyn_cast<ConstantSDNode>(I->getOperand(1));
22059     if (!Idx)
22060       return false;
22061 
22062     SDValue Src = I->getOperand(0);
22063     DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
22064     if (M == SrcOpMap.end()) {
22065       VT = Src.getValueType();
22066       // Quit if not the same type.
22067       if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType())
22068         return false;
22069       unsigned NumElts = VT.getVectorNumElements();
22070       APInt EltCount = APInt::getZero(NumElts);
22071       M = SrcOpMap.insert(std::make_pair(Src, EltCount)).first;
22072       SrcOps.push_back(Src);
22073     }
22074 
22075     // Quit if element already used.
22076     unsigned CIdx = Idx->getZExtValue();
22077     if (M->second[CIdx])
22078       return false;
22079     M->second.setBit(CIdx);
22080   }
22081 
22082   if (SrcMask) {
22083     // Collect the source partial masks.
22084     for (SDValue &SrcOp : SrcOps)
22085       SrcMask->push_back(SrcOpMap[SrcOp]);
22086   } else {
22087     // Quit if not all elements are used.
22088     for (const auto &I : SrcOpMap)
22089       if (!I.second.isAllOnes())
22090         return false;
22091   }
22092 
22093   return true;
22094 }
22095 
22096 // Helper function for comparing all bits of two vectors.
22097 static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS,
22098                                    ISD::CondCode CC, const APInt &OriginalMask,
22099                                    const X86Subtarget &Subtarget,
22100                                    SelectionDAG &DAG, X86::CondCode &X86CC) {
22101   EVT VT = LHS.getValueType();
22102   unsigned ScalarSize = VT.getScalarSizeInBits();
22103   if (OriginalMask.getBitWidth() != ScalarSize) {
22104     assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch");
22105     return SDValue();
22106   }
22107 
22108   // Quit if not convertable to legal scalar or 128/256-bit vector.
22109   if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22110     return SDValue();
22111 
22112   // FCMP may use ISD::SETNE when nnan - early out if we manage to get here.
22113   if (VT.isFloatingPoint())
22114     return SDValue();
22115 
22116   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22117   X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
22118 
22119   APInt Mask = OriginalMask;
22120 
22121   auto MaskBits = [&](SDValue Src) {
22122     if (Mask.isAllOnes())
22123       return Src;
22124     EVT SrcVT = Src.getValueType();
22125     SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
22126     return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
22127   };
22128 
22129   // For sub-128-bit vector, cast to (legal) integer and compare with zero.
22130   if (VT.getSizeInBits() < 128) {
22131     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
22132     if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) {
22133       if (IntVT != MVT::i64)
22134         return SDValue();
22135       auto SplitLHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(LHS)), DL,
22136                                       MVT::i32, MVT::i32);
22137       auto SplitRHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(RHS)), DL,
22138                                       MVT::i32, MVT::i32);
22139       SDValue Lo =
22140           DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.first, SplitRHS.first);
22141       SDValue Hi =
22142           DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.second, SplitRHS.second);
22143       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22144                          DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi),
22145                          DAG.getConstant(0, DL, MVT::i32));
22146     }
22147     return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22148                        DAG.getBitcast(IntVT, MaskBits(LHS)),
22149                        DAG.getBitcast(IntVT, MaskBits(RHS)));
22150   }
22151 
22152   // Without PTEST, a masked v2i64 or-reduction is not faster than
22153   // scalarization.
22154   bool UseKORTEST = Subtarget.useAVX512Regs();
22155   bool UsePTEST = Subtarget.hasSSE41();
22156   if (!UsePTEST && !Mask.isAllOnes() && ScalarSize > 32)
22157     return SDValue();
22158 
22159   // Split down to 128/256/512-bit vector.
22160   unsigned TestSize = UseKORTEST ? 512 : (Subtarget.hasAVX() ? 256 : 128);
22161 
22162   // If the input vector has vector elements wider than the target test size,
22163   // then cast to <X x i64> so it will safely split.
22164   if (ScalarSize > TestSize) {
22165     if (!Mask.isAllOnes())
22166       return SDValue();
22167     VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64);
22168     LHS = DAG.getBitcast(VT, LHS);
22169     RHS = DAG.getBitcast(VT, RHS);
22170     Mask = APInt::getAllOnes(64);
22171   }
22172 
22173   if (VT.getSizeInBits() > TestSize) {
22174     KnownBits KnownRHS = DAG.computeKnownBits(RHS);
22175     if (KnownRHS.isConstant() && KnownRHS.getConstant() == Mask) {
22176       // If ICMP(AND(LHS,MASK),MASK) - reduce using AND splits.
22177       while (VT.getSizeInBits() > TestSize) {
22178         auto Split = DAG.SplitVector(LHS, DL);
22179         VT = Split.first.getValueType();
22180         LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22181       }
22182       RHS = DAG.getAllOnesConstant(DL, VT);
22183     } else if (!UsePTEST && !KnownRHS.isZero()) {
22184       // MOVMSK Special Case:
22185       // ALLOF(CMPEQ(X,Y)) -> AND(CMPEQ(X[0],Y[0]),CMPEQ(X[1],Y[1]),....)
22186       MVT SVT = ScalarSize >= 32 ? MVT::i32 : MVT::i8;
22187       VT = MVT::getVectorVT(SVT, VT.getSizeInBits() / SVT.getSizeInBits());
22188       LHS = DAG.getBitcast(VT, MaskBits(LHS));
22189       RHS = DAG.getBitcast(VT, MaskBits(RHS));
22190       EVT BoolVT = VT.changeVectorElementType(MVT::i1);
22191       SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ);
22192       V = DAG.getSExtOrTrunc(V, DL, VT);
22193       while (VT.getSizeInBits() > TestSize) {
22194         auto Split = DAG.SplitVector(V, DL);
22195         VT = Split.first.getValueType();
22196         V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22197       }
22198       V = DAG.getNOT(DL, V, VT);
22199       V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22200       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22201                          DAG.getConstant(0, DL, MVT::i32));
22202     } else {
22203       // Convert to a ICMP_EQ(XOR(LHS,RHS),0) pattern.
22204       SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
22205       while (VT.getSizeInBits() > TestSize) {
22206         auto Split = DAG.SplitVector(V, DL);
22207         VT = Split.first.getValueType();
22208         V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second);
22209       }
22210       LHS = V;
22211       RHS = DAG.getConstant(0, DL, VT);
22212     }
22213   }
22214 
22215   if (UseKORTEST && VT.is512BitVector()) {
22216     MVT TestVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
22217     MVT BoolVT = TestVT.changeVectorElementType(MVT::i1);
22218     LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22219     RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22220     SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE);
22221     return DAG.getNode(X86ISD::KORTEST, DL, MVT::i32, V, V);
22222   }
22223 
22224   if (UsePTEST) {
22225     MVT TestVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
22226     LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22227     RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22228     SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS);
22229     return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
22230   }
22231 
22232   assert(VT.getSizeInBits() == 128 && "Failure to split to 128-bits");
22233   MVT MaskVT = ScalarSize >= 32 ? MVT::v4i32 : MVT::v16i8;
22234   LHS = DAG.getBitcast(MaskVT, MaskBits(LHS));
22235   RHS = DAG.getBitcast(MaskVT, MaskBits(RHS));
22236   SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS);
22237   V = DAG.getNOT(DL, V, MaskVT);
22238   V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22239   return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22240                      DAG.getConstant(0, DL, MVT::i32));
22241 }
22242 
22243 // Check whether an AND/OR'd reduction tree is PTEST-able, or if we can fallback
22244 // to CMP(MOVMSK(PCMPEQB(X,Y))).
22245 static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS,
22246                                        ISD::CondCode CC, const SDLoc &DL,
22247                                        const X86Subtarget &Subtarget,
22248                                        SelectionDAG &DAG,
22249                                        X86::CondCode &X86CC) {
22250   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22251 
22252   bool CmpNull = isNullConstant(RHS);
22253   bool CmpAllOnes = isAllOnesConstant(RHS);
22254   if (!CmpNull && !CmpAllOnes)
22255     return SDValue();
22256 
22257   SDValue Op = LHS;
22258   if (!Subtarget.hasSSE2() || !Op->hasOneUse())
22259     return SDValue();
22260 
22261   // Check whether we're masking/truncating an OR-reduction result, in which
22262   // case track the masked bits.
22263   // TODO: Add CmpAllOnes support.
22264   APInt Mask = APInt::getAllOnes(Op.getScalarValueSizeInBits());
22265   if (CmpNull) {
22266     switch (Op.getOpcode()) {
22267     case ISD::TRUNCATE: {
22268       SDValue Src = Op.getOperand(0);
22269       Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
22270                                   Op.getScalarValueSizeInBits());
22271       Op = Src;
22272       break;
22273     }
22274     case ISD::AND: {
22275       if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
22276         Mask = Cst->getAPIntValue();
22277         Op = Op.getOperand(0);
22278       }
22279       break;
22280     }
22281     }
22282   }
22283 
22284   ISD::NodeType LogicOp = CmpNull ? ISD::OR : ISD::AND;
22285 
22286   // Match icmp(or(extract(X,0),extract(X,1)),0) anyof reduction patterns.
22287   // Match icmp(and(extract(X,0),extract(X,1)),-1) allof reduction patterns.
22288   SmallVector<SDValue, 8> VecIns;
22289   if (Op.getOpcode() == LogicOp && matchScalarReduction(Op, LogicOp, VecIns)) {
22290     EVT VT = VecIns[0].getValueType();
22291     assert(llvm::all_of(VecIns,
22292                         [VT](SDValue V) { return VT == V.getValueType(); }) &&
22293            "Reduction source vector mismatch");
22294 
22295     // Quit if not splittable to scalar/128/256/512-bit vector.
22296     if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22297       return SDValue();
22298 
22299     // If more than one full vector is evaluated, AND/OR them first before
22300     // PTEST.
22301     for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1;
22302          Slot += 2, e += 1) {
22303       // Each iteration will AND/OR 2 nodes and append the result until there is
22304       // only 1 node left, i.e. the final value of all vectors.
22305       SDValue LHS = VecIns[Slot];
22306       SDValue RHS = VecIns[Slot + 1];
22307       VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS));
22308     }
22309 
22310     return LowerVectorAllEqual(DL, VecIns.back(),
22311                                CmpNull ? DAG.getConstant(0, DL, VT)
22312                                        : DAG.getAllOnesConstant(DL, VT),
22313                                CC, Mask, Subtarget, DAG, X86CC);
22314   }
22315 
22316   // Match icmp(reduce_or(X),0) anyof reduction patterns.
22317   // Match icmp(reduce_and(X),-1) allof reduction patterns.
22318   if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22319     ISD::NodeType BinOp;
22320     if (SDValue Match =
22321             DAG.matchBinOpReduction(Op.getNode(), BinOp, {LogicOp})) {
22322       EVT MatchVT = Match.getValueType();
22323       return LowerVectorAllEqual(DL, Match,
22324                                  CmpNull ? DAG.getConstant(0, DL, MatchVT)
22325                                          : DAG.getAllOnesConstant(DL, MatchVT),
22326                                  CC, Mask, Subtarget, DAG, X86CC);
22327     }
22328   }
22329 
22330   if (Mask.isAllOnes()) {
22331     assert(!Op.getValueType().isVector() &&
22332            "Illegal vector type for reduction pattern");
22333     SDValue Src = peekThroughBitcasts(Op);
22334     if (Src.getValueType().isFixedLengthVector() &&
22335         Src.getValueType().getScalarType() == MVT::i1) {
22336       // Match icmp(bitcast(icmp_ne(X,Y)),0) reduction patterns.
22337       // Match icmp(bitcast(icmp_eq(X,Y)),-1) reduction patterns.
22338       if (Src.getOpcode() == ISD::SETCC) {
22339         SDValue LHS = Src.getOperand(0);
22340         SDValue RHS = Src.getOperand(1);
22341         EVT LHSVT = LHS.getValueType();
22342         ISD::CondCode SrcCC = cast<CondCodeSDNode>(Src.getOperand(2))->get();
22343         if (SrcCC == (CmpNull ? ISD::SETNE : ISD::SETEQ) &&
22344             llvm::has_single_bit<uint32_t>(LHSVT.getSizeInBits())) {
22345           APInt SrcMask = APInt::getAllOnes(LHSVT.getScalarSizeInBits());
22346           return LowerVectorAllEqual(DL, LHS, RHS, CC, SrcMask, Subtarget, DAG,
22347                                      X86CC);
22348         }
22349       }
22350       // Match icmp(bitcast(vXi1 trunc(Y)),0) reduction patterns.
22351       // Match icmp(bitcast(vXi1 trunc(Y)),-1) reduction patterns.
22352       // Peek through truncation, mask the LSB and compare against zero/LSB.
22353       if (Src.getOpcode() == ISD::TRUNCATE) {
22354         SDValue Inner = Src.getOperand(0);
22355         EVT InnerVT = Inner.getValueType();
22356         if (llvm::has_single_bit<uint32_t>(InnerVT.getSizeInBits())) {
22357           unsigned BW = InnerVT.getScalarSizeInBits();
22358           APInt SrcMask = APInt(BW, 1);
22359           APInt Cmp = CmpNull ? APInt::getZero(BW) : SrcMask;
22360           return LowerVectorAllEqual(DL, Inner,
22361                                      DAG.getConstant(Cmp, DL, InnerVT), CC,
22362                                      SrcMask, Subtarget, DAG, X86CC);
22363         }
22364       }
22365     }
22366   }
22367 
22368   return SDValue();
22369 }
22370 
22371 /// return true if \c Op has a use that doesn't just read flags.
22372 static bool hasNonFlagsUse(SDValue Op) {
22373   for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
22374        ++UI) {
22375     SDNode *User = *UI;
22376     unsigned UOpNo = UI.getOperandNo();
22377     if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
22378       // Look pass truncate.
22379       UOpNo = User->use_begin().getOperandNo();
22380       User = *User->use_begin();
22381     }
22382 
22383     if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
22384         !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
22385       return true;
22386   }
22387   return false;
22388 }
22389 
22390 // Transform to an x86-specific ALU node with flags if there is a chance of
22391 // using an RMW op or only the flags are used. Otherwise, leave
22392 // the node alone and emit a 'cmp' or 'test' instruction.
22393 static bool isProfitableToUseFlagOp(SDValue Op) {
22394   for (SDNode *U : Op->uses())
22395     if (U->getOpcode() != ISD::CopyToReg &&
22396         U->getOpcode() != ISD::SETCC &&
22397         U->getOpcode() != ISD::STORE)
22398       return false;
22399 
22400   return true;
22401 }
22402 
22403 /// Emit nodes that will be selected as "test Op0,Op0", or something
22404 /// equivalent.
22405 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
22406                         SelectionDAG &DAG, const X86Subtarget &Subtarget) {
22407   // CF and OF aren't always set the way we want. Determine which
22408   // of these we need.
22409   bool NeedCF = false;
22410   bool NeedOF = false;
22411   switch (X86CC) {
22412   default: break;
22413   case X86::COND_A: case X86::COND_AE:
22414   case X86::COND_B: case X86::COND_BE:
22415     NeedCF = true;
22416     break;
22417   case X86::COND_G: case X86::COND_GE:
22418   case X86::COND_L: case X86::COND_LE:
22419   case X86::COND_O: case X86::COND_NO: {
22420     // Check if we really need to set the
22421     // Overflow flag. If NoSignedWrap is present
22422     // that is not actually needed.
22423     switch (Op->getOpcode()) {
22424     case ISD::ADD:
22425     case ISD::SUB:
22426     case ISD::MUL:
22427     case ISD::SHL:
22428       if (Op.getNode()->getFlags().hasNoSignedWrap())
22429         break;
22430       [[fallthrough]];
22431     default:
22432       NeedOF = true;
22433       break;
22434     }
22435     break;
22436   }
22437   }
22438   // See if we can use the EFLAGS value from the operand instead of
22439   // doing a separate TEST. TEST always sets OF and CF to 0, so unless
22440   // we prove that the arithmetic won't overflow, we can't use OF or CF.
22441   if (Op.getResNo() != 0 || NeedOF || NeedCF) {
22442     // Emit a CMP with 0, which is the TEST pattern.
22443     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22444                        DAG.getConstant(0, dl, Op.getValueType()));
22445   }
22446   unsigned Opcode = 0;
22447   unsigned NumOperands = 0;
22448 
22449   SDValue ArithOp = Op;
22450 
22451   // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
22452   // which may be the result of a CAST.  We use the variable 'Op', which is the
22453   // non-casted variable when we check for possible users.
22454   switch (ArithOp.getOpcode()) {
22455   case ISD::AND:
22456     // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
22457     // because a TEST instruction will be better.
22458     if (!hasNonFlagsUse(Op))
22459       break;
22460 
22461     [[fallthrough]];
22462   case ISD::ADD:
22463   case ISD::SUB:
22464   case ISD::OR:
22465   case ISD::XOR:
22466     if (!isProfitableToUseFlagOp(Op))
22467       break;
22468 
22469     // Otherwise use a regular EFLAGS-setting instruction.
22470     switch (ArithOp.getOpcode()) {
22471     default: llvm_unreachable("unexpected operator!");
22472     case ISD::ADD: Opcode = X86ISD::ADD; break;
22473     case ISD::SUB: Opcode = X86ISD::SUB; break;
22474     case ISD::XOR: Opcode = X86ISD::XOR; break;
22475     case ISD::AND: Opcode = X86ISD::AND; break;
22476     case ISD::OR:  Opcode = X86ISD::OR;  break;
22477     }
22478 
22479     NumOperands = 2;
22480     break;
22481   case X86ISD::ADD:
22482   case X86ISD::SUB:
22483   case X86ISD::OR:
22484   case X86ISD::XOR:
22485   case X86ISD::AND:
22486     return SDValue(Op.getNode(), 1);
22487   case ISD::SSUBO:
22488   case ISD::USUBO: {
22489     // /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
22490     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22491     return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
22492                        Op->getOperand(1)).getValue(1);
22493   }
22494   default:
22495     break;
22496   }
22497 
22498   if (Opcode == 0) {
22499     // Emit a CMP with 0, which is the TEST pattern.
22500     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22501                        DAG.getConstant(0, dl, Op.getValueType()));
22502   }
22503   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22504   SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
22505 
22506   SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
22507   DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
22508   return SDValue(New.getNode(), 1);
22509 }
22510 
22511 /// Emit nodes that will be selected as "cmp Op0,Op1", or something
22512 /// equivalent.
22513 static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
22514                        const SDLoc &dl, SelectionDAG &DAG,
22515                        const X86Subtarget &Subtarget) {
22516   if (isNullConstant(Op1))
22517     return EmitTest(Op0, X86CC, dl, DAG, Subtarget);
22518 
22519   EVT CmpVT = Op0.getValueType();
22520 
22521   assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
22522           CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
22523 
22524   // Only promote the compare up to I32 if it is a 16 bit operation
22525   // with an immediate.  16 bit immediates are to be avoided.
22526   if (CmpVT == MVT::i16 && !Subtarget.isAtom() &&
22527       !DAG.getMachineFunction().getFunction().hasMinSize()) {
22528     ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
22529     ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
22530     // Don't do this if the immediate can fit in 8-bits.
22531     if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) ||
22532         (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) {
22533       unsigned ExtendOp =
22534           isX86CCSigned(X86CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
22535       if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
22536         // For equality comparisons try to use SIGN_EXTEND if the input was
22537         // truncate from something with enough sign bits.
22538         if (Op0.getOpcode() == ISD::TRUNCATE) {
22539           if (DAG.ComputeMaxSignificantBits(Op0.getOperand(0)) <= 16)
22540             ExtendOp = ISD::SIGN_EXTEND;
22541         } else if (Op1.getOpcode() == ISD::TRUNCATE) {
22542           if (DAG.ComputeMaxSignificantBits(Op1.getOperand(0)) <= 16)
22543             ExtendOp = ISD::SIGN_EXTEND;
22544         }
22545       }
22546 
22547       CmpVT = MVT::i32;
22548       Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0);
22549       Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1);
22550     }
22551   }
22552 
22553   // Try to shrink i64 compares if the input has enough zero bits.
22554   // FIXME: Do this for non-constant compares for constant on LHS?
22555   if (CmpVT == MVT::i64 && isa<ConstantSDNode>(Op1) && !isX86CCSigned(X86CC) &&
22556       Op0.hasOneUse() && // Hacky way to not break CSE opportunities with sub.
22557       Op1->getAsAPIntVal().getActiveBits() <= 32 &&
22558       DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) {
22559     CmpVT = MVT::i32;
22560     Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0);
22561     Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1);
22562   }
22563 
22564   // 0-x == y --> x+y == 0
22565   // 0-x != y --> x+y != 0
22566   if (Op0.getOpcode() == ISD::SUB && isNullConstant(Op0.getOperand(0)) &&
22567       Op0.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22568     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22569     SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
22570     return Add.getValue(1);
22571   }
22572 
22573   // x == 0-y --> x+y == 0
22574   // x != 0-y --> x+y != 0
22575   if (Op1.getOpcode() == ISD::SUB && isNullConstant(Op1.getOperand(0)) &&
22576       Op1.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22577     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22578     SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
22579     return Add.getValue(1);
22580   }
22581 
22582   // Use SUB instead of CMP to enable CSE between SUB and CMP.
22583   SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22584   SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
22585   return Sub.getValue(1);
22586 }
22587 
22588 bool X86TargetLowering::isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
22589                                                           EVT VT) const {
22590   return !VT.isVector() || Cond != ISD::CondCode::SETEQ;
22591 }
22592 
22593 bool X86TargetLowering::optimizeFMulOrFDivAsShiftAddBitcast(
22594     SDNode *N, SDValue, SDValue IntPow2) const {
22595   if (N->getOpcode() == ISD::FDIV)
22596     return true;
22597 
22598   EVT FPVT = N->getValueType(0);
22599   EVT IntVT = IntPow2.getValueType();
22600 
22601   // This indicates a non-free bitcast.
22602   // TODO: This is probably overly conservative as we will need to scale the
22603   // integer vector anyways for the int->fp cast.
22604   if (FPVT.isVector() &&
22605       FPVT.getScalarSizeInBits() != IntVT.getScalarSizeInBits())
22606     return false;
22607 
22608   return true;
22609 }
22610 
22611 /// Check if replacement of SQRT with RSQRT should be disabled.
22612 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
22613   EVT VT = Op.getValueType();
22614 
22615   // We don't need to replace SQRT with RSQRT for half type.
22616   if (VT.getScalarType() == MVT::f16)
22617     return true;
22618 
22619   // We never want to use both SQRT and RSQRT instructions for the same input.
22620   if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
22621     return false;
22622 
22623   if (VT.isVector())
22624     return Subtarget.hasFastVectorFSQRT();
22625   return Subtarget.hasFastScalarFSQRT();
22626 }
22627 
22628 /// The minimum architected relative accuracy is 2^-12. We need one
22629 /// Newton-Raphson step to have a good float result (24 bits of precision).
22630 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
22631                                            SelectionDAG &DAG, int Enabled,
22632                                            int &RefinementSteps,
22633                                            bool &UseOneConstNR,
22634                                            bool Reciprocal) const {
22635   SDLoc DL(Op);
22636   EVT VT = Op.getValueType();
22637 
22638   // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
22639   // It is likely not profitable to do this for f64 because a double-precision
22640   // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
22641   // instructions: convert to single, rsqrtss, convert back to double, refine
22642   // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
22643   // along with FMA, this could be a throughput win.
22644   // TODO: SQRT requires SSE2 to prevent the introduction of an illegal v4i32
22645   // after legalize types.
22646   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22647       (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||
22648       (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||
22649       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22650       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22651     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22652       RefinementSteps = 1;
22653 
22654     UseOneConstNR = false;
22655     // There is no FSQRT for 512-bits, but there is RSQRT14.
22656     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
22657     SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
22658     if (RefinementSteps == 0 && !Reciprocal)
22659       Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate);
22660     return Estimate;
22661   }
22662 
22663   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22664       Subtarget.hasFP16()) {
22665     assert(Reciprocal && "Don't replace SQRT with RSQRT for half type");
22666     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22667       RefinementSteps = 0;
22668 
22669     if (VT == MVT::f16) {
22670       SDValue Zero = DAG.getIntPtrConstant(0, DL);
22671       SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22672       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22673       Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);
22674       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22675     }
22676 
22677     return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op);
22678   }
22679   return SDValue();
22680 }
22681 
22682 /// The minimum architected relative accuracy is 2^-12. We need one
22683 /// Newton-Raphson step to have a good float result (24 bits of precision).
22684 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
22685                                             int Enabled,
22686                                             int &RefinementSteps) const {
22687   SDLoc DL(Op);
22688   EVT VT = Op.getValueType();
22689 
22690   // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
22691   // It is likely not profitable to do this for f64 because a double-precision
22692   // reciprocal estimate with refinement on x86 prior to FMA requires
22693   // 15 instructions: convert to single, rcpss, convert back to double, refine
22694   // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
22695   // along with FMA, this could be a throughput win.
22696 
22697   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22698       (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
22699       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22700       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22701     // Enable estimate codegen with 1 refinement step for vector division.
22702     // Scalar division estimates are disabled because they break too much
22703     // real-world code. These defaults are intended to match GCC behavior.
22704     if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
22705       return SDValue();
22706 
22707     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22708       RefinementSteps = 1;
22709 
22710     // There is no FSQRT for 512-bits, but there is RCP14.
22711     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
22712     return DAG.getNode(Opcode, DL, VT, Op);
22713   }
22714 
22715   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22716       Subtarget.hasFP16()) {
22717     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22718       RefinementSteps = 0;
22719 
22720     if (VT == MVT::f16) {
22721       SDValue Zero = DAG.getIntPtrConstant(0, DL);
22722       SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22723       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22724       Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);
22725       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22726     }
22727 
22728     return DAG.getNode(X86ISD::RCP14, DL, VT, Op);
22729   }
22730   return SDValue();
22731 }
22732 
22733 /// If we have at least two divisions that use the same divisor, convert to
22734 /// multiplication by a reciprocal. This may need to be adjusted for a given
22735 /// CPU if a division's cost is not at least twice the cost of a multiplication.
22736 /// This is because we still need one division to calculate the reciprocal and
22737 /// then we need two multiplies by that reciprocal as replacements for the
22738 /// original divisions.
22739 unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
22740   return 2;
22741 }
22742 
22743 SDValue
22744 X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
22745                                  SelectionDAG &DAG,
22746                                  SmallVectorImpl<SDNode *> &Created) const {
22747   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
22748   if (isIntDivCheap(N->getValueType(0), Attr))
22749     return SDValue(N,0); // Lower SDIV as SDIV
22750 
22751   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
22752          "Unexpected divisor!");
22753 
22754   // Only perform this transform if CMOV is supported otherwise the select
22755   // below will become a branch.
22756   if (!Subtarget.canUseCMOV())
22757     return SDValue();
22758 
22759   // fold (sdiv X, pow2)
22760   EVT VT = N->getValueType(0);
22761   // FIXME: Support i8.
22762   if (VT != MVT::i16 && VT != MVT::i32 &&
22763       !(Subtarget.is64Bit() && VT == MVT::i64))
22764     return SDValue();
22765 
22766   // If the divisor is 2 or -2, the default expansion is better.
22767   if (Divisor == 2 ||
22768       Divisor == APInt(Divisor.getBitWidth(), -2, /*isSigned*/ true))
22769     return SDValue();
22770 
22771   return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
22772 }
22773 
22774 /// Result of 'and' is compared against zero. Change to a BT node if possible.
22775 /// Returns the BT node and the condition code needed to use it.
22776 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl,
22777                             SelectionDAG &DAG, X86::CondCode &X86CC) {
22778   assert(And.getOpcode() == ISD::AND && "Expected AND node!");
22779   SDValue Op0 = And.getOperand(0);
22780   SDValue Op1 = And.getOperand(1);
22781   if (Op0.getOpcode() == ISD::TRUNCATE)
22782     Op0 = Op0.getOperand(0);
22783   if (Op1.getOpcode() == ISD::TRUNCATE)
22784     Op1 = Op1.getOperand(0);
22785 
22786   SDValue Src, BitNo;
22787   if (Op1.getOpcode() == ISD::SHL)
22788     std::swap(Op0, Op1);
22789   if (Op0.getOpcode() == ISD::SHL) {
22790     if (isOneConstant(Op0.getOperand(0))) {
22791       // If we looked past a truncate, check that it's only truncating away
22792       // known zeros.
22793       unsigned BitWidth = Op0.getValueSizeInBits();
22794       unsigned AndBitWidth = And.getValueSizeInBits();
22795       if (BitWidth > AndBitWidth) {
22796         KnownBits Known = DAG.computeKnownBits(Op0);
22797         if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth)
22798           return SDValue();
22799       }
22800       Src = Op1;
22801       BitNo = Op0.getOperand(1);
22802     }
22803   } else if (Op1.getOpcode() == ISD::Constant) {
22804     ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
22805     uint64_t AndRHSVal = AndRHS->getZExtValue();
22806     SDValue AndLHS = Op0;
22807 
22808     if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
22809       Src = AndLHS.getOperand(0);
22810       BitNo = AndLHS.getOperand(1);
22811     } else {
22812       // Use BT if the immediate can't be encoded in a TEST instruction or we
22813       // are optimizing for size and the immedaite won't fit in a byte.
22814       bool OptForSize = DAG.shouldOptForSize();
22815       if ((!isUInt<32>(AndRHSVal) || (OptForSize && !isUInt<8>(AndRHSVal))) &&
22816           isPowerOf2_64(AndRHSVal)) {
22817         Src = AndLHS;
22818         BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
22819                                 Src.getValueType());
22820       }
22821     }
22822   }
22823 
22824   // No patterns found, give up.
22825   if (!Src.getNode())
22826     return SDValue();
22827 
22828   // Remove any bit flip.
22829   if (isBitwiseNot(Src)) {
22830     Src = Src.getOperand(0);
22831     CC = CC == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
22832   }
22833 
22834   // Attempt to create the X86ISD::BT node.
22835   if (SDValue BT = getBT(Src, BitNo, dl, DAG)) {
22836     X86CC = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
22837     return BT;
22838   }
22839 
22840   return SDValue();
22841 }
22842 
22843 // Check if pre-AVX condcode can be performed by a single FCMP op.
22844 static bool cheapX86FSETCC_SSE(ISD::CondCode SetCCOpcode) {
22845   return (SetCCOpcode != ISD::SETONE) && (SetCCOpcode != ISD::SETUEQ);
22846 }
22847 
22848 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
22849 /// CMPs.
22850 static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
22851                                    SDValue &Op1, bool &IsAlwaysSignaling) {
22852   unsigned SSECC;
22853   bool Swap = false;
22854 
22855   // SSE Condition code mapping:
22856   //  0 - EQ
22857   //  1 - LT
22858   //  2 - LE
22859   //  3 - UNORD
22860   //  4 - NEQ
22861   //  5 - NLT
22862   //  6 - NLE
22863   //  7 - ORD
22864   switch (SetCCOpcode) {
22865   default: llvm_unreachable("Unexpected SETCC condition");
22866   case ISD::SETOEQ:
22867   case ISD::SETEQ:  SSECC = 0; break;
22868   case ISD::SETOGT:
22869   case ISD::SETGT:  Swap = true; [[fallthrough]];
22870   case ISD::SETLT:
22871   case ISD::SETOLT: SSECC = 1; break;
22872   case ISD::SETOGE:
22873   case ISD::SETGE:  Swap = true; [[fallthrough]];
22874   case ISD::SETLE:
22875   case ISD::SETOLE: SSECC = 2; break;
22876   case ISD::SETUO:  SSECC = 3; break;
22877   case ISD::SETUNE:
22878   case ISD::SETNE:  SSECC = 4; break;
22879   case ISD::SETULE: Swap = true; [[fallthrough]];
22880   case ISD::SETUGE: SSECC = 5; break;
22881   case ISD::SETULT: Swap = true; [[fallthrough]];
22882   case ISD::SETUGT: SSECC = 6; break;
22883   case ISD::SETO:   SSECC = 7; break;
22884   case ISD::SETUEQ: SSECC = 8; break;
22885   case ISD::SETONE: SSECC = 12; break;
22886   }
22887   if (Swap)
22888     std::swap(Op0, Op1);
22889 
22890   switch (SetCCOpcode) {
22891   default:
22892     IsAlwaysSignaling = true;
22893     break;
22894   case ISD::SETEQ:
22895   case ISD::SETOEQ:
22896   case ISD::SETUEQ:
22897   case ISD::SETNE:
22898   case ISD::SETONE:
22899   case ISD::SETUNE:
22900   case ISD::SETO:
22901   case ISD::SETUO:
22902     IsAlwaysSignaling = false;
22903     break;
22904   }
22905 
22906   return SSECC;
22907 }
22908 
22909 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
22910 /// concatenate the result back.
22911 static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS,
22912                               ISD::CondCode Cond, SelectionDAG &DAG,
22913                               const SDLoc &dl) {
22914   assert(VT.isInteger() && VT == LHS.getValueType() &&
22915          VT == RHS.getValueType() && "Unsupported VTs!");
22916 
22917   SDValue CC = DAG.getCondCode(Cond);
22918 
22919   // Extract the LHS Lo/Hi vectors
22920   SDValue LHS1, LHS2;
22921   std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl);
22922 
22923   // Extract the RHS Lo/Hi vectors
22924   SDValue RHS1, RHS2;
22925   std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl);
22926 
22927   // Issue the operation on the smaller types and concatenate the result back
22928   EVT LoVT, HiVT;
22929   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
22930   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
22931                      DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC),
22932                      DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC));
22933 }
22934 
22935 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
22936 
22937   SDValue Op0 = Op.getOperand(0);
22938   SDValue Op1 = Op.getOperand(1);
22939   SDValue CC = Op.getOperand(2);
22940   MVT VT = Op.getSimpleValueType();
22941   SDLoc dl(Op);
22942 
22943   assert(VT.getVectorElementType() == MVT::i1 &&
22944          "Cannot set masked compare for this operation");
22945 
22946   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
22947 
22948   // Prefer SETGT over SETLT.
22949   if (SetCCOpcode == ISD::SETLT) {
22950     SetCCOpcode = ISD::getSetCCSwappedOperands(SetCCOpcode);
22951     std::swap(Op0, Op1);
22952   }
22953 
22954   return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);
22955 }
22956 
22957 /// Given a buildvector constant, return a new vector constant with each element
22958 /// incremented or decremented. If incrementing or decrementing would result in
22959 /// unsigned overflow or underflow or this is not a simple vector constant,
22960 /// return an empty value.
22961 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc,
22962                                     bool NSW) {
22963   auto *BV = dyn_cast<BuildVectorSDNode>(V.getNode());
22964   if (!BV || !V.getValueType().isSimple())
22965     return SDValue();
22966 
22967   MVT VT = V.getSimpleValueType();
22968   MVT EltVT = VT.getVectorElementType();
22969   unsigned NumElts = VT.getVectorNumElements();
22970   SmallVector<SDValue, 8> NewVecC;
22971   SDLoc DL(V);
22972   for (unsigned i = 0; i < NumElts; ++i) {
22973     auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
22974     if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT)
22975       return SDValue();
22976 
22977     // Avoid overflow/underflow.
22978     const APInt &EltC = Elt->getAPIntValue();
22979     if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isZero()))
22980       return SDValue();
22981     if (NSW && ((IsInc && EltC.isMaxSignedValue()) ||
22982                 (!IsInc && EltC.isMinSignedValue())))
22983       return SDValue();
22984 
22985     NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
22986   }
22987 
22988   return DAG.getBuildVector(VT, DL, NewVecC);
22989 }
22990 
22991 /// As another special case, use PSUBUS[BW] when it's profitable. E.g. for
22992 /// Op0 u<= Op1:
22993 ///   t = psubus Op0, Op1
22994 ///   pcmpeq t, <0..0>
22995 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
22996                                     ISD::CondCode Cond, const SDLoc &dl,
22997                                     const X86Subtarget &Subtarget,
22998                                     SelectionDAG &DAG) {
22999   if (!Subtarget.hasSSE2())
23000     return SDValue();
23001 
23002   MVT VET = VT.getVectorElementType();
23003   if (VET != MVT::i8 && VET != MVT::i16)
23004     return SDValue();
23005 
23006   switch (Cond) {
23007   default:
23008     return SDValue();
23009   case ISD::SETULT: {
23010     // If the comparison is against a constant we can turn this into a
23011     // setule.  With psubus, setule does not require a swap.  This is
23012     // beneficial because the constant in the register is no longer
23013     // destructed as the destination so it can be hoisted out of a loop.
23014     // Only do this pre-AVX since vpcmp* is no longer destructive.
23015     if (Subtarget.hasAVX())
23016       return SDValue();
23017     SDValue ULEOp1 =
23018         incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false);
23019     if (!ULEOp1)
23020       return SDValue();
23021     Op1 = ULEOp1;
23022     break;
23023   }
23024   case ISD::SETUGT: {
23025     // If the comparison is against a constant, we can turn this into a setuge.
23026     // This is beneficial because materializing a constant 0 for the PCMPEQ is
23027     // probably cheaper than XOR+PCMPGT using 2 different vector constants:
23028     // cmpgt (xor X, SignMaskC) CmpC --> cmpeq (usubsat (CmpC+1), X), 0
23029     SDValue UGEOp1 =
23030         incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false);
23031     if (!UGEOp1)
23032       return SDValue();
23033     Op1 = Op0;
23034     Op0 = UGEOp1;
23035     break;
23036   }
23037   // Psubus is better than flip-sign because it requires no inversion.
23038   case ISD::SETUGE:
23039     std::swap(Op0, Op1);
23040     break;
23041   case ISD::SETULE:
23042     break;
23043   }
23044 
23045   SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
23046   return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
23047                      DAG.getConstant(0, dl, VT));
23048 }
23049 
23050 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
23051                            SelectionDAG &DAG) {
23052   bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23053                   Op.getOpcode() == ISD::STRICT_FSETCCS;
23054   SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23055   SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23056   SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
23057   MVT VT = Op->getSimpleValueType(0);
23058   ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get();
23059   bool isFP = Op1.getSimpleValueType().isFloatingPoint();
23060   SDLoc dl(Op);
23061 
23062   if (isFP) {
23063     MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
23064     assert(EltVT == MVT::f16 || EltVT == MVT::f32 || EltVT == MVT::f64);
23065     if (isSoftF16(EltVT, Subtarget))
23066       return SDValue();
23067 
23068     bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23069     SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23070 
23071     // If we have a strict compare with a vXi1 result and the input is 128/256
23072     // bits we can't use a masked compare unless we have VLX. If we use a wider
23073     // compare like we do for non-strict, we might trigger spurious exceptions
23074     // from the upper elements. Instead emit a AVX compare and convert to mask.
23075     unsigned Opc;
23076     if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 &&
23077         (!IsStrict || Subtarget.hasVLX() ||
23078          Op0.getSimpleValueType().is512BitVector())) {
23079 #ifndef NDEBUG
23080       unsigned Num = VT.getVectorNumElements();
23081       assert(Num <= 16 || (Num == 32 && EltVT == MVT::f16));
23082 #endif
23083       Opc = IsStrict ? X86ISD::STRICT_CMPM : X86ISD::CMPM;
23084     } else {
23085       Opc = IsStrict ? X86ISD::STRICT_CMPP : X86ISD::CMPP;
23086       // The SSE/AVX packed FP comparison nodes are defined with a
23087       // floating-point vector result that matches the operand type. This allows
23088       // them to work with an SSE1 target (integer vector types are not legal).
23089       VT = Op0.getSimpleValueType();
23090     }
23091 
23092     SDValue Cmp;
23093     bool IsAlwaysSignaling;
23094     unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1, IsAlwaysSignaling);
23095     if (!Subtarget.hasAVX()) {
23096       // TODO: We could use following steps to handle a quiet compare with
23097       // signaling encodings.
23098       // 1. Get ordered masks from a quiet ISD::SETO
23099       // 2. Use the masks to mask potential unordered elements in operand A, B
23100       // 3. Get the compare results of masked A, B
23101       // 4. Calculating final result using the mask and result from 3
23102       // But currently, we just fall back to scalar operations.
23103       if (IsStrict && IsAlwaysSignaling && !IsSignaling)
23104         return SDValue();
23105 
23106       // Insert an extra signaling instruction to raise exception.
23107       if (IsStrict && !IsAlwaysSignaling && IsSignaling) {
23108         SDValue SignalCmp = DAG.getNode(
23109             Opc, dl, {VT, MVT::Other},
23110             {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS
23111         // FIXME: It seems we need to update the flags of all new strict nodes.
23112         // Otherwise, mayRaiseFPException in MI will return false due to
23113         // NoFPExcept = false by default. However, I didn't find it in other
23114         // patches.
23115         SignalCmp->setFlags(Op->getFlags());
23116         Chain = SignalCmp.getValue(1);
23117       }
23118 
23119       // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
23120       // emit two comparisons and a logic op to tie them together.
23121       if (!cheapX86FSETCC_SSE(Cond)) {
23122         // LLVM predicate is SETUEQ or SETONE.
23123         unsigned CC0, CC1;
23124         unsigned CombineOpc;
23125         if (Cond == ISD::SETUEQ) {
23126           CC0 = 3; // UNORD
23127           CC1 = 0; // EQ
23128           CombineOpc = X86ISD::FOR;
23129         } else {
23130           assert(Cond == ISD::SETONE);
23131           CC0 = 7; // ORD
23132           CC1 = 4; // NEQ
23133           CombineOpc = X86ISD::FAND;
23134         }
23135 
23136         SDValue Cmp0, Cmp1;
23137         if (IsStrict) {
23138           Cmp0 = DAG.getNode(
23139               Opc, dl, {VT, MVT::Other},
23140               {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});
23141           Cmp1 = DAG.getNode(
23142               Opc, dl, {VT, MVT::Other},
23143               {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
23144           Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
23145                               Cmp1.getValue(1));
23146         } else {
23147           Cmp0 = DAG.getNode(
23148               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
23149           Cmp1 = DAG.getNode(
23150               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
23151         }
23152         Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
23153       } else {
23154         if (IsStrict) {
23155           Cmp = DAG.getNode(
23156               Opc, dl, {VT, MVT::Other},
23157               {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23158           Chain = Cmp.getValue(1);
23159         } else
23160           Cmp = DAG.getNode(
23161               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23162       }
23163     } else {
23164       // Handle all other FP comparisons here.
23165       if (IsStrict) {
23166         // Make a flip on already signaling CCs before setting bit 4 of AVX CC.
23167         SSECC |= (IsAlwaysSignaling ^ IsSignaling) << 4;
23168         Cmp = DAG.getNode(
23169             Opc, dl, {VT, MVT::Other},
23170             {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23171         Chain = Cmp.getValue(1);
23172       } else
23173         Cmp = DAG.getNode(
23174             Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23175     }
23176 
23177     if (VT.getFixedSizeInBits() >
23178         Op.getSimpleValueType().getFixedSizeInBits()) {
23179       // We emitted a compare with an XMM/YMM result. Finish converting to a
23180       // mask register using a vptestm.
23181       EVT CastVT = EVT(VT).changeVectorElementTypeToInteger();
23182       Cmp = DAG.getBitcast(CastVT, Cmp);
23183       Cmp = DAG.getSetCC(dl, Op.getSimpleValueType(), Cmp,
23184                          DAG.getConstant(0, dl, CastVT), ISD::SETNE);
23185     } else {
23186       // If this is SSE/AVX CMPP, bitcast the result back to integer to match
23187       // the result type of SETCC. The bitcast is expected to be optimized
23188       // away during combining/isel.
23189       Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
23190     }
23191 
23192     if (IsStrict)
23193       return DAG.getMergeValues({Cmp, Chain}, dl);
23194 
23195     return Cmp;
23196   }
23197 
23198   assert(!IsStrict && "Strict SETCC only handles FP operands.");
23199 
23200   MVT VTOp0 = Op0.getSimpleValueType();
23201   (void)VTOp0;
23202   assert(VTOp0 == Op1.getSimpleValueType() &&
23203          "Expected operands with same type!");
23204   assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
23205          "Invalid number of packed elements for source and destination!");
23206 
23207   // The non-AVX512 code below works under the assumption that source and
23208   // destination types are the same.
23209   assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&
23210          "Value types for source and destination must be the same!");
23211 
23212   // The result is boolean, but operands are int/float
23213   if (VT.getVectorElementType() == MVT::i1) {
23214     // In AVX-512 architecture setcc returns mask with i1 elements,
23215     // But there is no compare instruction for i8 and i16 elements in KNL.
23216     assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) &&
23217            "Unexpected operand type");
23218     return LowerIntVSETCC_AVX512(Op, DAG);
23219   }
23220 
23221   // Lower using XOP integer comparisons.
23222   if (VT.is128BitVector() && Subtarget.hasXOP()) {
23223     // Translate compare code to XOP PCOM compare mode.
23224     unsigned CmpMode = 0;
23225     switch (Cond) {
23226     default: llvm_unreachable("Unexpected SETCC condition");
23227     case ISD::SETULT:
23228     case ISD::SETLT: CmpMode = 0x00; break;
23229     case ISD::SETULE:
23230     case ISD::SETLE: CmpMode = 0x01; break;
23231     case ISD::SETUGT:
23232     case ISD::SETGT: CmpMode = 0x02; break;
23233     case ISD::SETUGE:
23234     case ISD::SETGE: CmpMode = 0x03; break;
23235     case ISD::SETEQ: CmpMode = 0x04; break;
23236     case ISD::SETNE: CmpMode = 0x05; break;
23237     }
23238 
23239     // Are we comparing unsigned or signed integers?
23240     unsigned Opc =
23241         ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
23242 
23243     return DAG.getNode(Opc, dl, VT, Op0, Op1,
23244                        DAG.getTargetConstant(CmpMode, dl, MVT::i8));
23245   }
23246 
23247   // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2.
23248   // Revert part of the simplifySetCCWithAnd combine, to avoid an invert.
23249   if (Cond == ISD::SETNE && ISD::isBuildVectorAllZeros(Op1.getNode())) {
23250     SDValue BC0 = peekThroughBitcasts(Op0);
23251     if (BC0.getOpcode() == ISD::AND) {
23252       APInt UndefElts;
23253       SmallVector<APInt, 64> EltBits;
23254       if (getTargetConstantBitsFromNode(BC0.getOperand(1),
23255                                         VT.getScalarSizeInBits(), UndefElts,
23256                                         EltBits, false, false)) {
23257         if (llvm::all_of(EltBits, [](APInt &V) { return V.isPowerOf2(); })) {
23258           Cond = ISD::SETEQ;
23259           Op1 = DAG.getBitcast(VT, BC0.getOperand(1));
23260         }
23261       }
23262     }
23263   }
23264 
23265   // ICMP_EQ(AND(X,C),C) -> SRA(SHL(X,LOG2(C)),BW-1) iff C is power-of-2.
23266   if (Cond == ISD::SETEQ && Op0.getOpcode() == ISD::AND &&
23267       Op0.getOperand(1) == Op1 && Op0.hasOneUse()) {
23268     ConstantSDNode *C1 = isConstOrConstSplat(Op1);
23269     if (C1 && C1->getAPIntValue().isPowerOf2()) {
23270       unsigned BitWidth = VT.getScalarSizeInBits();
23271       unsigned ShiftAmt = BitWidth - C1->getAPIntValue().logBase2() - 1;
23272 
23273       SDValue Result = Op0.getOperand(0);
23274       Result = DAG.getNode(ISD::SHL, dl, VT, Result,
23275                            DAG.getConstant(ShiftAmt, dl, VT));
23276       Result = DAG.getNode(ISD::SRA, dl, VT, Result,
23277                            DAG.getConstant(BitWidth - 1, dl, VT));
23278       return Result;
23279     }
23280   }
23281 
23282   // Break 256-bit integer vector compare into smaller ones.
23283   if (VT.is256BitVector() && !Subtarget.hasInt256())
23284     return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23285 
23286   // Break 512-bit integer vector compare into smaller ones.
23287   // TODO: Try harder to use VPCMPx + VPMOV2x?
23288   if (VT.is512BitVector())
23289     return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23290 
23291   // If we have a limit constant, try to form PCMPGT (signed cmp) to avoid
23292   // not-of-PCMPEQ:
23293   // X != INT_MIN --> X >s INT_MIN
23294   // X != INT_MAX --> X <s INT_MAX --> INT_MAX >s X
23295   // +X != 0 --> +X >s 0
23296   APInt ConstValue;
23297   if (Cond == ISD::SETNE &&
23298       ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) {
23299     if (ConstValue.isMinSignedValue())
23300       Cond = ISD::SETGT;
23301     else if (ConstValue.isMaxSignedValue())
23302       Cond = ISD::SETLT;
23303     else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0))
23304       Cond = ISD::SETGT;
23305   }
23306 
23307   // If both operands are known non-negative, then an unsigned compare is the
23308   // same as a signed compare and there's no need to flip signbits.
23309   // TODO: We could check for more general simplifications here since we're
23310   // computing known bits.
23311   bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) &&
23312                    !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1));
23313 
23314   // Special case: Use min/max operations for unsigned compares.
23315   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23316   if (ISD::isUnsignedIntSetCC(Cond) &&
23317       (FlipSigns || ISD::isTrueWhenEqual(Cond)) &&
23318       TLI.isOperationLegal(ISD::UMIN, VT)) {
23319     // If we have a constant operand, increment/decrement it and change the
23320     // condition to avoid an invert.
23321     if (Cond == ISD::SETUGT) {
23322       // X > C --> X >= (C+1) --> X == umax(X, C+1)
23323       if (SDValue UGTOp1 =
23324               incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false)) {
23325         Op1 = UGTOp1;
23326         Cond = ISD::SETUGE;
23327       }
23328     }
23329     if (Cond == ISD::SETULT) {
23330       // X < C --> X <= (C-1) --> X == umin(X, C-1)
23331       if (SDValue ULTOp1 =
23332               incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false)) {
23333         Op1 = ULTOp1;
23334         Cond = ISD::SETULE;
23335       }
23336     }
23337     bool Invert = false;
23338     unsigned Opc;
23339     switch (Cond) {
23340     default: llvm_unreachable("Unexpected condition code");
23341     case ISD::SETUGT: Invert = true; [[fallthrough]];
23342     case ISD::SETULE: Opc = ISD::UMIN; break;
23343     case ISD::SETULT: Invert = true; [[fallthrough]];
23344     case ISD::SETUGE: Opc = ISD::UMAX; break;
23345     }
23346 
23347     SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23348     Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
23349 
23350     // If the logical-not of the result is required, perform that now.
23351     if (Invert)
23352       Result = DAG.getNOT(dl, Result, VT);
23353 
23354     return Result;
23355   }
23356 
23357   // Try to use SUBUS and PCMPEQ.
23358   if (FlipSigns)
23359     if (SDValue V =
23360             LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
23361       return V;
23362 
23363   // We are handling one of the integer comparisons here. Since SSE only has
23364   // GT and EQ comparisons for integer, swapping operands and multiple
23365   // operations may be required for some comparisons.
23366   unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
23367                                                             : X86ISD::PCMPGT;
23368   bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
23369               Cond == ISD::SETGE || Cond == ISD::SETUGE;
23370   bool Invert = Cond == ISD::SETNE ||
23371                 (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
23372 
23373   if (Swap)
23374     std::swap(Op0, Op1);
23375 
23376   // Check that the operation in question is available (most are plain SSE2,
23377   // but PCMPGTQ and PCMPEQQ have different requirements).
23378   if (VT == MVT::v2i64) {
23379     if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
23380       assert(Subtarget.hasSSE2() && "Don't know how to lower!");
23381 
23382       // Special case for sign bit test. We can use a v4i32 PCMPGT and shuffle
23383       // the odd elements over the even elements.
23384       if (!FlipSigns && !Invert && ISD::isBuildVectorAllZeros(Op0.getNode())) {
23385         Op0 = DAG.getConstant(0, dl, MVT::v4i32);
23386         Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23387 
23388         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23389         static const int MaskHi[] = { 1, 1, 3, 3 };
23390         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23391 
23392         return DAG.getBitcast(VT, Result);
23393       }
23394 
23395       if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
23396         Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23397         Op1 = DAG.getConstant(-1, dl, MVT::v4i32);
23398 
23399         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23400         static const int MaskHi[] = { 1, 1, 3, 3 };
23401         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23402 
23403         return DAG.getBitcast(VT, Result);
23404       }
23405 
23406       // Since SSE has no unsigned integer comparisons, we need to flip the sign
23407       // bits of the inputs before performing those operations. The lower
23408       // compare is always unsigned.
23409       SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL
23410                                              : 0x0000000080000000ULL,
23411                                    dl, MVT::v2i64);
23412 
23413       Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);
23414       Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);
23415 
23416       // Cast everything to the right type.
23417       Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23418       Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23419 
23420       // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
23421       SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23422       SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
23423 
23424       // Create masks for only the low parts/high parts of the 64 bit integers.
23425       static const int MaskHi[] = { 1, 1, 3, 3 };
23426       static const int MaskLo[] = { 0, 0, 2, 2 };
23427       SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
23428       SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
23429       SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23430 
23431       SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
23432       Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
23433 
23434       if (Invert)
23435         Result = DAG.getNOT(dl, Result, MVT::v4i32);
23436 
23437       return DAG.getBitcast(VT, Result);
23438     }
23439 
23440     if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
23441       // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
23442       // pcmpeqd + pshufd + pand.
23443       assert(Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!");
23444 
23445       // First cast everything to the right type.
23446       Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23447       Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23448 
23449       // Do the compare.
23450       SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
23451 
23452       // Make sure the lower and upper halves are both all-ones.
23453       static const int Mask[] = { 1, 0, 3, 2 };
23454       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
23455       Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
23456 
23457       if (Invert)
23458         Result = DAG.getNOT(dl, Result, MVT::v4i32);
23459 
23460       return DAG.getBitcast(VT, Result);
23461     }
23462   }
23463 
23464   // Since SSE has no unsigned integer comparisons, we need to flip the sign
23465   // bits of the inputs before performing those operations.
23466   if (FlipSigns) {
23467     MVT EltVT = VT.getVectorElementType();
23468     SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
23469                                  VT);
23470     Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
23471     Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
23472   }
23473 
23474   SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23475 
23476   // If the logical-not of the result is required, perform that now.
23477   if (Invert)
23478     Result = DAG.getNOT(dl, Result, VT);
23479 
23480   return Result;
23481 }
23482 
23483 // Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
23484 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
23485                               const SDLoc &dl, SelectionDAG &DAG,
23486                               const X86Subtarget &Subtarget,
23487                               SDValue &X86CC) {
23488   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
23489 
23490   // Must be a bitcast from vXi1.
23491   if (Op0.getOpcode() != ISD::BITCAST)
23492     return SDValue();
23493 
23494   Op0 = Op0.getOperand(0);
23495   MVT VT = Op0.getSimpleValueType();
23496   if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&
23497       !(Subtarget.hasDQI() && VT == MVT::v8i1) &&
23498       !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))
23499     return SDValue();
23500 
23501   X86::CondCode X86Cond;
23502   if (isNullConstant(Op1)) {
23503     X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
23504   } else if (isAllOnesConstant(Op1)) {
23505     // C flag is set for all ones.
23506     X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
23507   } else
23508     return SDValue();
23509 
23510   // If the input is an AND, we can combine it's operands into the KTEST.
23511   bool KTestable = false;
23512   if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1))
23513     KTestable = true;
23514   if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))
23515     KTestable = true;
23516   if (!isNullConstant(Op1))
23517     KTestable = false;
23518   if (KTestable && Op0.getOpcode() == ISD::AND && Op0.hasOneUse()) {
23519     SDValue LHS = Op0.getOperand(0);
23520     SDValue RHS = Op0.getOperand(1);
23521     X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23522     return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);
23523   }
23524 
23525   // If the input is an OR, we can combine it's operands into the KORTEST.
23526   SDValue LHS = Op0;
23527   SDValue RHS = Op0;
23528   if (Op0.getOpcode() == ISD::OR && Op0.hasOneUse()) {
23529     LHS = Op0.getOperand(0);
23530     RHS = Op0.getOperand(1);
23531   }
23532 
23533   X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23534   return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
23535 }
23536 
23537 /// Emit flags for the given setcc condition and operands. Also returns the
23538 /// corresponding X86 condition code constant in X86CC.
23539 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
23540                                              ISD::CondCode CC, const SDLoc &dl,
23541                                              SelectionDAG &DAG,
23542                                              SDValue &X86CC) const {
23543   // Equality Combines.
23544   if (CC == ISD::SETEQ || CC == ISD::SETNE) {
23545     X86::CondCode X86CondCode;
23546 
23547     // Optimize to BT if possible.
23548     // Lower (X & (1 << N)) == 0 to BT(X, N).
23549     // Lower ((X >>u N) & 1) != 0 to BT(X, N).
23550     // Lower ((X >>s N) & 1) != 0 to BT(X, N).
23551     if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1)) {
23552       if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) {
23553         X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23554         return BT;
23555       }
23556     }
23557 
23558     // Try to use PTEST/PMOVMSKB for a tree AND/ORs equality compared with -1/0.
23559     if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG,
23560                                                X86CondCode)) {
23561       X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23562       return CmpZ;
23563     }
23564 
23565     // Try to lower using KORTEST or KTEST.
23566     if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
23567       return Test;
23568 
23569     // Look for X == 0, X == 1, X != 0, or X != 1.  We can simplify some forms
23570     // of these.
23571     if (isOneConstant(Op1) || isNullConstant(Op1)) {
23572       // If the input is a setcc, then reuse the input setcc or use a new one
23573       // with the inverted condition.
23574       if (Op0.getOpcode() == X86ISD::SETCC) {
23575         bool Invert = (CC == ISD::SETNE) ^ isNullConstant(Op1);
23576 
23577         X86CC = Op0.getOperand(0);
23578         if (Invert) {
23579           X86CondCode = (X86::CondCode)Op0.getConstantOperandVal(0);
23580           X86CondCode = X86::GetOppositeBranchCondition(X86CondCode);
23581           X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23582         }
23583 
23584         return Op0.getOperand(1);
23585       }
23586     }
23587 
23588     // Try to use the carry flag from the add in place of an separate CMP for:
23589     // (seteq (add X, -1), -1). Similar for setne.
23590     if (isAllOnesConstant(Op1) && Op0.getOpcode() == ISD::ADD &&
23591         Op0.getOperand(1) == Op1) {
23592       if (isProfitableToUseFlagOp(Op0)) {
23593         SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
23594 
23595         SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
23596                                   Op0.getOperand(1));
23597         DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
23598         X86CondCode = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
23599         X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23600         return SDValue(New.getNode(), 1);
23601       }
23602     }
23603   }
23604 
23605   X86::CondCode CondCode =
23606       TranslateX86CC(CC, dl, /*IsFP*/ false, Op0, Op1, DAG);
23607   assert(CondCode != X86::COND_INVALID && "Unexpected condition code!");
23608 
23609   SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
23610   X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23611   return EFLAGS;
23612 }
23613 
23614 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
23615 
23616   bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23617                   Op.getOpcode() == ISD::STRICT_FSETCCS;
23618   MVT VT = Op->getSimpleValueType(0);
23619 
23620   if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
23621 
23622   assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
23623   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23624   SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23625   SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23626   SDLoc dl(Op);
23627   ISD::CondCode CC =
23628       cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
23629 
23630   if (isSoftF16(Op0.getValueType(), Subtarget))
23631     return SDValue();
23632 
23633   // Handle f128 first, since one possible outcome is a normal integer
23634   // comparison which gets handled by emitFlagsForSetcc.
23635   if (Op0.getValueType() == MVT::f128) {
23636     softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,
23637                         Op.getOpcode() == ISD::STRICT_FSETCCS);
23638 
23639     // If softenSetCCOperands returned a scalar, use it.
23640     if (!Op1.getNode()) {
23641       assert(Op0.getValueType() == Op.getValueType() &&
23642              "Unexpected setcc expansion!");
23643       if (IsStrict)
23644         return DAG.getMergeValues({Op0, Chain}, dl);
23645       return Op0;
23646     }
23647   }
23648 
23649   if (Op0.getSimpleValueType().isInteger()) {
23650     // Attempt to canonicalize SGT/UGT -> SGE/UGE compares with constant which
23651     // reduces the number of EFLAGs bit reads (the GE conditions don't read ZF),
23652     // this may translate to less uops depending on uarch implementation. The
23653     // equivalent for SLE/ULE -> SLT/ULT isn't likely to happen as we already
23654     // canonicalize to that CondCode.
23655     // NOTE: Only do this if incrementing the constant doesn't increase the bit
23656     // encoding size - so it must either already be a i8 or i32 immediate, or it
23657     // shrinks down to that. We don't do this for any i64's to avoid additional
23658     // constant materializations.
23659     // TODO: Can we move this to TranslateX86CC to handle jumps/branches too?
23660     if (auto *Op1C = dyn_cast<ConstantSDNode>(Op1)) {
23661       const APInt &Op1Val = Op1C->getAPIntValue();
23662       if (!Op1Val.isZero()) {
23663         // Ensure the constant+1 doesn't overflow.
23664         if ((CC == ISD::CondCode::SETGT && !Op1Val.isMaxSignedValue()) ||
23665             (CC == ISD::CondCode::SETUGT && !Op1Val.isMaxValue())) {
23666           APInt Op1ValPlusOne = Op1Val + 1;
23667           if (Op1ValPlusOne.isSignedIntN(32) &&
23668               (!Op1Val.isSignedIntN(8) || Op1ValPlusOne.isSignedIntN(8))) {
23669             Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType());
23670             CC = CC == ISD::CondCode::SETGT ? ISD::CondCode::SETGE
23671                                             : ISD::CondCode::SETUGE;
23672           }
23673         }
23674       }
23675     }
23676 
23677     SDValue X86CC;
23678     SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
23679     SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23680     return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23681   }
23682 
23683   // Handle floating point.
23684   X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
23685   if (CondCode == X86::COND_INVALID)
23686     return SDValue();
23687 
23688   SDValue EFLAGS;
23689   if (IsStrict) {
23690     bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23691     EFLAGS =
23692         DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP,
23693                     dl, {MVT::i32, MVT::Other}, {Chain, Op0, Op1});
23694     Chain = EFLAGS.getValue(1);
23695   } else {
23696     EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1);
23697   }
23698 
23699   SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23700   SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23701   return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23702 }
23703 
23704 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
23705   SDValue LHS = Op.getOperand(0);
23706   SDValue RHS = Op.getOperand(1);
23707   SDValue Carry = Op.getOperand(2);
23708   SDValue Cond = Op.getOperand(3);
23709   SDLoc DL(Op);
23710 
23711   assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
23712   X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
23713 
23714   // Recreate the carry if needed.
23715   EVT CarryVT = Carry.getValueType();
23716   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
23717                       Carry, DAG.getAllOnesConstant(DL, CarryVT));
23718 
23719   SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
23720   SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
23721   return getSETCC(CC, Cmp.getValue(1), DL, DAG);
23722 }
23723 
23724 // This function returns three things: the arithmetic computation itself
23725 // (Value), an EFLAGS result (Overflow), and a condition code (Cond).  The
23726 // flag and the condition code define the case in which the arithmetic
23727 // computation overflows.
23728 static std::pair<SDValue, SDValue>
23729 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
23730   assert(Op.getResNo() == 0 && "Unexpected result number!");
23731   SDValue Value, Overflow;
23732   SDValue LHS = Op.getOperand(0);
23733   SDValue RHS = Op.getOperand(1);
23734   unsigned BaseOp = 0;
23735   SDLoc DL(Op);
23736   switch (Op.getOpcode()) {
23737   default: llvm_unreachable("Unknown ovf instruction!");
23738   case ISD::SADDO:
23739     BaseOp = X86ISD::ADD;
23740     Cond = X86::COND_O;
23741     break;
23742   case ISD::UADDO:
23743     BaseOp = X86ISD::ADD;
23744     Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
23745     break;
23746   case ISD::SSUBO:
23747     BaseOp = X86ISD::SUB;
23748     Cond = X86::COND_O;
23749     break;
23750   case ISD::USUBO:
23751     BaseOp = X86ISD::SUB;
23752     Cond = X86::COND_B;
23753     break;
23754   case ISD::SMULO:
23755     BaseOp = X86ISD::SMUL;
23756     Cond = X86::COND_O;
23757     break;
23758   case ISD::UMULO:
23759     BaseOp = X86ISD::UMUL;
23760     Cond = X86::COND_O;
23761     break;
23762   }
23763 
23764   if (BaseOp) {
23765     // Also sets EFLAGS.
23766     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
23767     Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
23768     Overflow = Value.getValue(1);
23769   }
23770 
23771   return std::make_pair(Value, Overflow);
23772 }
23773 
23774 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
23775   // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
23776   // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
23777   // looks for this combo and may remove the "setcc" instruction if the "setcc"
23778   // has only one use.
23779   SDLoc DL(Op);
23780   X86::CondCode Cond;
23781   SDValue Value, Overflow;
23782   std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
23783 
23784   SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
23785   assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!");
23786   return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
23787 }
23788 
23789 /// Return true if opcode is a X86 logical comparison.
23790 static bool isX86LogicalCmp(SDValue Op) {
23791   unsigned Opc = Op.getOpcode();
23792   if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
23793       Opc == X86ISD::FCMP)
23794     return true;
23795   if (Op.getResNo() == 1 &&
23796       (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
23797        Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
23798        Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
23799     return true;
23800 
23801   return false;
23802 }
23803 
23804 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
23805   if (V.getOpcode() != ISD::TRUNCATE)
23806     return false;
23807 
23808   SDValue VOp0 = V.getOperand(0);
23809   unsigned InBits = VOp0.getValueSizeInBits();
23810   unsigned Bits = V.getValueSizeInBits();
23811   return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
23812 }
23813 
23814 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
23815   bool AddTest = true;
23816   SDValue Cond  = Op.getOperand(0);
23817   SDValue Op1 = Op.getOperand(1);
23818   SDValue Op2 = Op.getOperand(2);
23819   SDLoc DL(Op);
23820   MVT VT = Op1.getSimpleValueType();
23821   SDValue CC;
23822 
23823   if (isSoftF16(VT, Subtarget)) {
23824     MVT NVT = VT.changeTypeToInteger();
23825     return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond,
23826                                           DAG.getBitcast(NVT, Op1),
23827                                           DAG.getBitcast(NVT, Op2)));
23828   }
23829 
23830   // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
23831   // are available or VBLENDV if AVX is available.
23832   // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
23833   if (Cond.getOpcode() == ISD::SETCC && isScalarFPTypeInSSEReg(VT) &&
23834       VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
23835     SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
23836     bool IsAlwaysSignaling;
23837     unsigned SSECC =
23838         translateX86FSETCC(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
23839                            CondOp0, CondOp1, IsAlwaysSignaling);
23840 
23841     if (Subtarget.hasAVX512()) {
23842       SDValue Cmp =
23843           DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
23844                       DAG.getTargetConstant(SSECC, DL, MVT::i8));
23845       assert(!VT.isVector() && "Not a scalar type?");
23846       return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
23847     }
23848 
23849     if (SSECC < 8 || Subtarget.hasAVX()) {
23850       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
23851                                 DAG.getTargetConstant(SSECC, DL, MVT::i8));
23852 
23853       // If we have AVX, we can use a variable vector select (VBLENDV) instead
23854       // of 3 logic instructions for size savings and potentially speed.
23855       // Unfortunately, there is no scalar form of VBLENDV.
23856 
23857       // If either operand is a +0.0 constant, don't try this. We can expect to
23858       // optimize away at least one of the logic instructions later in that
23859       // case, so that sequence would be faster than a variable blend.
23860 
23861       // BLENDV was introduced with SSE 4.1, but the 2 register form implicitly
23862       // uses XMM0 as the selection register. That may need just as many
23863       // instructions as the AND/ANDN/OR sequence due to register moves, so
23864       // don't bother.
23865       if (Subtarget.hasAVX() && !isNullFPConstant(Op1) &&
23866           !isNullFPConstant(Op2)) {
23867         // Convert to vectors, do a VSELECT, and convert back to scalar.
23868         // All of the conversions should be optimized away.
23869         MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;
23870         SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
23871         SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
23872         SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
23873 
23874         MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;
23875         VCmp = DAG.getBitcast(VCmpVT, VCmp);
23876 
23877         SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
23878 
23879         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
23880                            VSel, DAG.getIntPtrConstant(0, DL));
23881       }
23882       SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
23883       SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
23884       return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
23885     }
23886   }
23887 
23888   // AVX512 fallback is to lower selects of scalar floats to masked moves.
23889   if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) {
23890     SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
23891     return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
23892   }
23893 
23894   if (Cond.getOpcode() == ISD::SETCC &&
23895       !isSoftF16(Cond.getOperand(0).getSimpleValueType(), Subtarget)) {
23896     if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
23897       Cond = NewCond;
23898       // If the condition was updated, it's possible that the operands of the
23899       // select were also updated (for example, EmitTest has a RAUW). Refresh
23900       // the local references to the select operands in case they got stale.
23901       Op1 = Op.getOperand(1);
23902       Op2 = Op.getOperand(2);
23903     }
23904   }
23905 
23906   // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
23907   // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
23908   // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
23909   // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
23910   // (select (and (x , 0x1) == 0), y, (z ^ y) ) -> (-(and (x , 0x1)) & z ) ^ y
23911   // (select (and (x , 0x1) == 0), y, (z | y) ) -> (-(and (x , 0x1)) & z ) | y
23912   // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
23913   // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
23914   if (Cond.getOpcode() == X86ISD::SETCC &&
23915       Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
23916       isNullConstant(Cond.getOperand(1).getOperand(1))) {
23917     SDValue Cmp = Cond.getOperand(1);
23918     SDValue CmpOp0 = Cmp.getOperand(0);
23919     unsigned CondCode = Cond.getConstantOperandVal(0);
23920 
23921     // Special handling for __builtin_ffs(X) - 1 pattern which looks like
23922     // (select (seteq X, 0), -1, (cttz_zero_undef X)). Disable the special
23923     // handle to keep the CMP with 0. This should be removed by
23924     // optimizeCompareInst by using the flags from the BSR/TZCNT used for the
23925     // cttz_zero_undef.
23926     auto MatchFFSMinus1 = [&](SDValue Op1, SDValue Op2) {
23927       return (Op1.getOpcode() == ISD::CTTZ_ZERO_UNDEF && Op1.hasOneUse() &&
23928               Op1.getOperand(0) == CmpOp0 && isAllOnesConstant(Op2));
23929     };
23930     if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) &&
23931         ((CondCode == X86::COND_NE && MatchFFSMinus1(Op1, Op2)) ||
23932          (CondCode == X86::COND_E && MatchFFSMinus1(Op2, Op1)))) {
23933       // Keep Cmp.
23934     } else if ((isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
23935         (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
23936       SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
23937       SDVTList CmpVTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
23938 
23939       // 'X - 1' sets the carry flag if X == 0.
23940       // '0 - X' sets the carry flag if X != 0.
23941       // Convert the carry flag to a -1/0 mask with sbb:
23942       // select (X != 0), -1, Y --> 0 - X; or (sbb), Y
23943       // select (X == 0), Y, -1 --> 0 - X; or (sbb), Y
23944       // select (X != 0), Y, -1 --> X - 1; or (sbb), Y
23945       // select (X == 0), -1, Y --> X - 1; or (sbb), Y
23946       SDValue Sub;
23947       if (isAllOnesConstant(Op1) == (CondCode == X86::COND_NE)) {
23948         SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
23949         Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0);
23950       } else {
23951         SDValue One = DAG.getConstant(1, DL, CmpOp0.getValueType());
23952         Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpOp0, One);
23953       }
23954       SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
23955                                 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
23956                                 Sub.getValue(1));
23957       return DAG.getNode(ISD::OR, DL, VT, SBB, Y);
23958     } else if (!Subtarget.canUseCMOV() && CondCode == X86::COND_E &&
23959                CmpOp0.getOpcode() == ISD::AND &&
23960                isOneConstant(CmpOp0.getOperand(1))) {
23961       SDValue Src1, Src2;
23962       // true if Op2 is XOR or OR operator and one of its operands
23963       // is equal to Op1
23964       // ( a , a op b) || ( b , a op b)
23965       auto isOrXorPattern = [&]() {
23966         if ((Op2.getOpcode() == ISD::XOR || Op2.getOpcode() == ISD::OR) &&
23967             (Op2.getOperand(0) == Op1 || Op2.getOperand(1) == Op1)) {
23968           Src1 =
23969               Op2.getOperand(0) == Op1 ? Op2.getOperand(1) : Op2.getOperand(0);
23970           Src2 = Op1;
23971           return true;
23972         }
23973         return false;
23974       };
23975 
23976       if (isOrXorPattern()) {
23977         SDValue Neg;
23978         unsigned int CmpSz = CmpOp0.getSimpleValueType().getSizeInBits();
23979         // we need mask of all zeros or ones with same size of the other
23980         // operands.
23981         if (CmpSz > VT.getSizeInBits())
23982           Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);
23983         else if (CmpSz < VT.getSizeInBits())
23984           Neg = DAG.getNode(ISD::AND, DL, VT,
23985               DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),
23986               DAG.getConstant(1, DL, VT));
23987         else
23988           Neg = CmpOp0;
23989         SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
23990                                    Neg); // -(and (x, 0x1))
23991         SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
23992         return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2);  // And Op y
23993       }
23994     } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) &&
23995                Cmp.getNode()->hasOneUse() && (CmpOp0 == Op1) &&
23996                ((CondCode == X86::COND_S) ||                    // smin(x, 0)
23997                 (CondCode == X86::COND_G && hasAndNot(Op1)))) { // smax(x, 0)
23998       // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
23999       //
24000       // If the comparison is testing for a positive value, we have to invert
24001       // the sign bit mask, so only do that transform if the target has a
24002       // bitwise 'and not' instruction (the invert is free).
24003       // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
24004       unsigned ShCt = VT.getSizeInBits() - 1;
24005       SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT);
24006       SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt);
24007       if (CondCode == X86::COND_G)
24008         Shift = DAG.getNOT(DL, Shift, VT);
24009       return DAG.getNode(ISD::AND, DL, VT, Shift, Op1);
24010     }
24011   }
24012 
24013   // Look past (and (setcc_carry (cmp ...)), 1).
24014   if (Cond.getOpcode() == ISD::AND &&
24015       Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
24016       isOneConstant(Cond.getOperand(1)))
24017     Cond = Cond.getOperand(0);
24018 
24019   // If condition flag is set by a X86ISD::CMP, then use it as the condition
24020   // setting operand in place of the X86ISD::SETCC.
24021   unsigned CondOpcode = Cond.getOpcode();
24022   if (CondOpcode == X86ISD::SETCC ||
24023       CondOpcode == X86ISD::SETCC_CARRY) {
24024     CC = Cond.getOperand(0);
24025 
24026     SDValue Cmp = Cond.getOperand(1);
24027     bool IllegalFPCMov = false;
24028     if (VT.isFloatingPoint() && !VT.isVector() &&
24029         !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV())  // FPStack?
24030       IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
24031 
24032     if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
24033         Cmp.getOpcode() == X86ISD::BT) { // FIXME
24034       Cond = Cmp;
24035       AddTest = false;
24036     }
24037   } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
24038              CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
24039              CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
24040     SDValue Value;
24041     X86::CondCode X86Cond;
24042     std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24043 
24044     CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
24045     AddTest = false;
24046   }
24047 
24048   if (AddTest) {
24049     // Look past the truncate if the high bits are known zero.
24050     if (isTruncWithZeroHighBitsInput(Cond, DAG))
24051       Cond = Cond.getOperand(0);
24052 
24053     // We know the result of AND is compared against zero. Try to match
24054     // it to BT.
24055     if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
24056       X86::CondCode X86CondCode;
24057       if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
24058         CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
24059         Cond = BT;
24060         AddTest = false;
24061       }
24062     }
24063   }
24064 
24065   if (AddTest) {
24066     CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
24067     Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
24068   }
24069 
24070   // a <  b ? -1 :  0 -> RES = ~setcc_carry
24071   // a <  b ?  0 : -1 -> RES = setcc_carry
24072   // a >= b ? -1 :  0 -> RES = setcc_carry
24073   // a >= b ?  0 : -1 -> RES = ~setcc_carry
24074   if (Cond.getOpcode() == X86ISD::SUB) {
24075     unsigned CondCode = CC->getAsZExtVal();
24076 
24077     if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
24078         (isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
24079         (isNullConstant(Op1) || isNullConstant(Op2))) {
24080       SDValue Res =
24081           DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
24082                       DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
24083       if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
24084         return DAG.getNOT(DL, Res, Res.getValueType());
24085       return Res;
24086     }
24087   }
24088 
24089   // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
24090   // widen the cmov and push the truncate through. This avoids introducing a new
24091   // branch during isel and doesn't add any extensions.
24092   if (Op.getValueType() == MVT::i8 &&
24093       Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
24094     SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
24095     if (T1.getValueType() == T2.getValueType() &&
24096         // Exclude CopyFromReg to avoid partial register stalls.
24097         T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
24098       SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
24099                                  CC, Cond);
24100       return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24101     }
24102   }
24103 
24104   // Or finally, promote i8 cmovs if we have CMOV,
24105   //                 or i16 cmovs if it won't prevent folding a load.
24106   // FIXME: we should not limit promotion of i8 case to only when the CMOV is
24107   //        legal, but EmitLoweredSelect() can not deal with these extensions
24108   //        being inserted between two CMOV's. (in i16 case too TBN)
24109   //        https://bugs.llvm.org/show_bug.cgi?id=40974
24110   if ((Op.getValueType() == MVT::i8 && Subtarget.canUseCMOV()) ||
24111       (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) &&
24112        !X86::mayFoldLoad(Op2, Subtarget))) {
24113     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
24114     Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
24115     SDValue Ops[] = { Op2, Op1, CC, Cond };
24116     SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
24117     return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24118   }
24119 
24120   // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
24121   // condition is true.
24122   SDValue Ops[] = { Op2, Op1, CC, Cond };
24123   return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
24124 }
24125 
24126 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
24127                                      const X86Subtarget &Subtarget,
24128                                      SelectionDAG &DAG) {
24129   MVT VT = Op->getSimpleValueType(0);
24130   SDValue In = Op->getOperand(0);
24131   MVT InVT = In.getSimpleValueType();
24132   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
24133   MVT VTElt = VT.getVectorElementType();
24134   SDLoc dl(Op);
24135 
24136   unsigned NumElts = VT.getVectorNumElements();
24137 
24138   // Extend VT if the scalar type is i8/i16 and BWI is not supported.
24139   MVT ExtVT = VT;
24140   if (!Subtarget.hasBWI() && VTElt.getSizeInBits() <= 16) {
24141     // If v16i32 is to be avoided, we'll need to split and concatenate.
24142     if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
24143       return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);
24144 
24145     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
24146   }
24147 
24148   // Widen to 512-bits if VLX is not supported.
24149   MVT WideVT = ExtVT;
24150   if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
24151     NumElts *= 512 / ExtVT.getSizeInBits();
24152     InVT = MVT::getVectorVT(MVT::i1, NumElts);
24153     In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT),
24154                      In, DAG.getIntPtrConstant(0, dl));
24155     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts);
24156   }
24157 
24158   SDValue V;
24159   MVT WideEltVT = WideVT.getVectorElementType();
24160   if ((Subtarget.hasDQI() && WideEltVT.getSizeInBits() >= 32) ||
24161       (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
24162     V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
24163   } else {
24164     SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
24165     SDValue Zero = DAG.getConstant(0, dl, WideVT);
24166     V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
24167   }
24168 
24169   // Truncate if we had to extend i16/i8 above.
24170   if (VT != ExtVT) {
24171     WideVT = MVT::getVectorVT(VTElt, NumElts);
24172     V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V);
24173   }
24174 
24175   // Extract back to 128/256-bit if we widened.
24176   if (WideVT != VT)
24177     V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,
24178                     DAG.getIntPtrConstant(0, dl));
24179 
24180   return V;
24181 }
24182 
24183 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24184                                SelectionDAG &DAG) {
24185   SDValue In = Op->getOperand(0);
24186   MVT InVT = In.getSimpleValueType();
24187 
24188   if (InVT.getVectorElementType() == MVT::i1)
24189     return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
24190 
24191   assert(Subtarget.hasAVX() && "Expected AVX support");
24192   return LowerAVXExtend(Op, DAG, Subtarget);
24193 }
24194 
24195 // Lowering for SIGN_EXTEND_VECTOR_INREG and ZERO_EXTEND_VECTOR_INREG.
24196 // For sign extend this needs to handle all vector sizes and SSE4.1 and
24197 // non-SSE4.1 targets. For zero extend this should only handle inputs of
24198 // MVT::v64i8 when BWI is not supported, but AVX512 is.
24199 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
24200                                         const X86Subtarget &Subtarget,
24201                                         SelectionDAG &DAG) {
24202   SDValue In = Op->getOperand(0);
24203   MVT VT = Op->getSimpleValueType(0);
24204   MVT InVT = In.getSimpleValueType();
24205 
24206   MVT SVT = VT.getVectorElementType();
24207   MVT InSVT = InVT.getVectorElementType();
24208   assert(SVT.getFixedSizeInBits() > InSVT.getFixedSizeInBits());
24209 
24210   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)
24211     return SDValue();
24212   if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
24213     return SDValue();
24214   if (!(VT.is128BitVector() && Subtarget.hasSSE2()) &&
24215       !(VT.is256BitVector() && Subtarget.hasAVX()) &&
24216       !(VT.is512BitVector() && Subtarget.hasAVX512()))
24217     return SDValue();
24218 
24219   SDLoc dl(Op);
24220   unsigned Opc = Op.getOpcode();
24221   unsigned NumElts = VT.getVectorNumElements();
24222 
24223   // For 256-bit vectors, we only need the lower (128-bit) half of the input.
24224   // For 512-bit vectors, we need 128-bits or 256-bits.
24225   if (InVT.getSizeInBits() > 128) {
24226     // Input needs to be at least the same number of elements as output, and
24227     // at least 128-bits.
24228     int InSize = InSVT.getSizeInBits() * NumElts;
24229     In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128));
24230     InVT = In.getSimpleValueType();
24231   }
24232 
24233   // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
24234   // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
24235   // need to be handled here for 256/512-bit results.
24236   if (Subtarget.hasInt256()) {
24237     assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");
24238 
24239     if (InVT.getVectorNumElements() != NumElts)
24240       return DAG.getNode(Op.getOpcode(), dl, VT, In);
24241 
24242     // FIXME: Apparently we create inreg operations that could be regular
24243     // extends.
24244     unsigned ExtOpc =
24245         Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
24246                                              : ISD::ZERO_EXTEND;
24247     return DAG.getNode(ExtOpc, dl, VT, In);
24248   }
24249 
24250   // pre-AVX2 256-bit extensions need to be split into 128-bit instructions.
24251   if (Subtarget.hasAVX()) {
24252     assert(VT.is256BitVector() && "256-bit vector expected");
24253     MVT HalfVT = VT.getHalfNumVectorElementsVT();
24254     int HalfNumElts = HalfVT.getVectorNumElements();
24255 
24256     unsigned NumSrcElts = InVT.getVectorNumElements();
24257     SmallVector<int, 16> HiMask(NumSrcElts, SM_SentinelUndef);
24258     for (int i = 0; i != HalfNumElts; ++i)
24259       HiMask[i] = HalfNumElts + i;
24260 
24261     SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
24262     SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
24263     Hi = DAG.getNode(Opc, dl, HalfVT, Hi);
24264     return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
24265   }
24266 
24267   // We should only get here for sign extend.
24268   assert(Opc == ISD::SIGN_EXTEND_VECTOR_INREG && "Unexpected opcode!");
24269   assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs");
24270   unsigned InNumElts = InVT.getVectorNumElements();
24271 
24272   // If the source elements are already all-signbits, we don't need to extend,
24273   // just splat the elements.
24274   APInt DemandedElts = APInt::getLowBitsSet(InNumElts, NumElts);
24275   if (DAG.ComputeNumSignBits(In, DemandedElts) == InVT.getScalarSizeInBits()) {
24276     unsigned Scale = InNumElts / NumElts;
24277     SmallVector<int, 16> ShuffleMask;
24278     for (unsigned I = 0; I != NumElts; ++I)
24279       ShuffleMask.append(Scale, I);
24280     return DAG.getBitcast(VT,
24281                           DAG.getVectorShuffle(InVT, dl, In, In, ShuffleMask));
24282   }
24283 
24284   // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI.
24285   SDValue Curr = In;
24286   SDValue SignExt = Curr;
24287 
24288   // As SRAI is only available on i16/i32 types, we expand only up to i32
24289   // and handle i64 separately.
24290   if (InVT != MVT::v4i32) {
24291     MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
24292 
24293     unsigned DestWidth = DestVT.getScalarSizeInBits();
24294     unsigned Scale = DestWidth / InSVT.getSizeInBits();
24295     unsigned DestElts = DestVT.getVectorNumElements();
24296 
24297     // Build a shuffle mask that takes each input element and places it in the
24298     // MSBs of the new element size.
24299     SmallVector<int, 16> Mask(InNumElts, SM_SentinelUndef);
24300     for (unsigned i = 0; i != DestElts; ++i)
24301       Mask[i * Scale + (Scale - 1)] = i;
24302 
24303     Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask);
24304     Curr = DAG.getBitcast(DestVT, Curr);
24305 
24306     unsigned SignExtShift = DestWidth - InSVT.getSizeInBits();
24307     SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
24308                           DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
24309   }
24310 
24311   if (VT == MVT::v2i64) {
24312     assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT");
24313     SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
24314     SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
24315     SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});
24316     SignExt = DAG.getBitcast(VT, SignExt);
24317   }
24318 
24319   return SignExt;
24320 }
24321 
24322 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24323                                 SelectionDAG &DAG) {
24324   MVT VT = Op->getSimpleValueType(0);
24325   SDValue In = Op->getOperand(0);
24326   MVT InVT = In.getSimpleValueType();
24327   SDLoc dl(Op);
24328 
24329   if (InVT.getVectorElementType() == MVT::i1)
24330     return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
24331 
24332   assert(VT.isVector() && InVT.isVector() && "Expected vector type");
24333   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
24334          "Expected same number of elements");
24335   assert((VT.getVectorElementType() == MVT::i16 ||
24336           VT.getVectorElementType() == MVT::i32 ||
24337           VT.getVectorElementType() == MVT::i64) &&
24338          "Unexpected element type");
24339   assert((InVT.getVectorElementType() == MVT::i8 ||
24340           InVT.getVectorElementType() == MVT::i16 ||
24341           InVT.getVectorElementType() == MVT::i32) &&
24342          "Unexpected element type");
24343 
24344   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
24345     assert(InVT == MVT::v32i8 && "Unexpected VT!");
24346     return splitVectorIntUnary(Op, DAG);
24347   }
24348 
24349   if (Subtarget.hasInt256())
24350     return Op;
24351 
24352   // Optimize vectors in AVX mode
24353   // Sign extend  v8i16 to v8i32 and
24354   //              v4i32 to v4i64
24355   //
24356   // Divide input vector into two parts
24357   // for v4i32 the high shuffle mask will be {2, 3, -1, -1}
24358   // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
24359   // concat the vectors to original VT
24360   MVT HalfVT = VT.getHalfNumVectorElementsVT();
24361   SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
24362 
24363   unsigned NumElems = InVT.getVectorNumElements();
24364   SmallVector<int,8> ShufMask(NumElems, -1);
24365   for (unsigned i = 0; i != NumElems/2; ++i)
24366     ShufMask[i] = i + NumElems/2;
24367 
24368   SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
24369   OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
24370 
24371   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
24372 }
24373 
24374 /// Change a vector store into a pair of half-size vector stores.
24375 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
24376   SDValue StoredVal = Store->getValue();
24377   assert((StoredVal.getValueType().is256BitVector() ||
24378           StoredVal.getValueType().is512BitVector()) &&
24379          "Expecting 256/512-bit op");
24380 
24381   // Splitting volatile memory ops is not allowed unless the operation was not
24382   // legal to begin with. Assume the input store is legal (this transform is
24383   // only used for targets with AVX). Note: It is possible that we have an
24384   // illegal type like v2i128, and so we could allow splitting a volatile store
24385   // in that case if that is important.
24386   if (!Store->isSimple())
24387     return SDValue();
24388 
24389   SDLoc DL(Store);
24390   SDValue Value0, Value1;
24391   std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL);
24392   unsigned HalfOffset = Value0.getValueType().getStoreSize();
24393   SDValue Ptr0 = Store->getBasePtr();
24394   SDValue Ptr1 =
24395       DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(HalfOffset), DL);
24396   SDValue Ch0 =
24397       DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
24398                    Store->getOriginalAlign(),
24399                    Store->getMemOperand()->getFlags());
24400   SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
24401                              Store->getPointerInfo().getWithOffset(HalfOffset),
24402                              Store->getOriginalAlign(),
24403                              Store->getMemOperand()->getFlags());
24404   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
24405 }
24406 
24407 /// Scalarize a vector store, bitcasting to TargetVT to determine the scalar
24408 /// type.
24409 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
24410                                     SelectionDAG &DAG) {
24411   SDValue StoredVal = Store->getValue();
24412   assert(StoreVT.is128BitVector() &&
24413          StoredVal.getValueType().is128BitVector() && "Expecting 128-bit op");
24414   StoredVal = DAG.getBitcast(StoreVT, StoredVal);
24415 
24416   // Splitting volatile memory ops is not allowed unless the operation was not
24417   // legal to begin with. We are assuming the input op is legal (this transform
24418   // is only used for targets with AVX).
24419   if (!Store->isSimple())
24420     return SDValue();
24421 
24422   MVT StoreSVT = StoreVT.getScalarType();
24423   unsigned NumElems = StoreVT.getVectorNumElements();
24424   unsigned ScalarSize = StoreSVT.getStoreSize();
24425 
24426   SDLoc DL(Store);
24427   SmallVector<SDValue, 4> Stores;
24428   for (unsigned i = 0; i != NumElems; ++i) {
24429     unsigned Offset = i * ScalarSize;
24430     SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(),
24431                                            TypeSize::getFixed(Offset), DL);
24432     SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
24433                               DAG.getIntPtrConstant(i, DL));
24434     SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
24435                               Store->getPointerInfo().getWithOffset(Offset),
24436                               Store->getOriginalAlign(),
24437                               Store->getMemOperand()->getFlags());
24438     Stores.push_back(Ch);
24439   }
24440   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
24441 }
24442 
24443 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
24444                           SelectionDAG &DAG) {
24445   StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
24446   SDLoc dl(St);
24447   SDValue StoredVal = St->getValue();
24448 
24449   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores.
24450   if (StoredVal.getValueType().isVector() &&
24451       StoredVal.getValueType().getVectorElementType() == MVT::i1) {
24452     unsigned NumElts = StoredVal.getValueType().getVectorNumElements();
24453     assert(NumElts <= 8 && "Unexpected VT");
24454     assert(!St->isTruncatingStore() && "Expected non-truncating store");
24455     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24456            "Expected AVX512F without AVX512DQI");
24457 
24458     // We must pad with zeros to ensure we store zeroes to any unused bits.
24459     StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
24460                             DAG.getUNDEF(MVT::v16i1), StoredVal,
24461                             DAG.getIntPtrConstant(0, dl));
24462     StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
24463     StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
24464     // Make sure we store zeros in the extra bits.
24465     if (NumElts < 8)
24466       StoredVal = DAG.getZeroExtendInReg(
24467           StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts));
24468 
24469     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24470                         St->getPointerInfo(), St->getOriginalAlign(),
24471                         St->getMemOperand()->getFlags());
24472   }
24473 
24474   if (St->isTruncatingStore())
24475     return SDValue();
24476 
24477   // If this is a 256-bit store of concatenated ops, we are better off splitting
24478   // that store into two 128-bit stores. This avoids spurious use of 256-bit ops
24479   // and each half can execute independently. Some cores would split the op into
24480   // halves anyway, so the concat (vinsertf128) is purely an extra op.
24481   MVT StoreVT = StoredVal.getSimpleValueType();
24482   if (StoreVT.is256BitVector() ||
24483       ((StoreVT == MVT::v32i16 || StoreVT == MVT::v64i8) &&
24484        !Subtarget.hasBWI())) {
24485     if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal.getNode(), DAG))
24486       return splitVectorStore(St, DAG);
24487     return SDValue();
24488   }
24489 
24490   if (StoreVT.is32BitVector())
24491     return SDValue();
24492 
24493   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24494   assert(StoreVT.is64BitVector() && "Unexpected VT");
24495   assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==
24496              TargetLowering::TypeWidenVector &&
24497          "Unexpected type action!");
24498 
24499   EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
24500   StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
24501                           DAG.getUNDEF(StoreVT));
24502 
24503   if (Subtarget.hasSSE2()) {
24504     // Widen the vector, cast to a v2x64 type, extract the single 64-bit element
24505     // and store it.
24506     MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64;
24507     MVT CastVT = MVT::getVectorVT(StVT, 2);
24508     StoredVal = DAG.getBitcast(CastVT, StoredVal);
24509     StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal,
24510                             DAG.getIntPtrConstant(0, dl));
24511 
24512     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24513                         St->getPointerInfo(), St->getOriginalAlign(),
24514                         St->getMemOperand()->getFlags());
24515   }
24516   assert(Subtarget.hasSSE1() && "Expected SSE");
24517   SDVTList Tys = DAG.getVTList(MVT::Other);
24518   SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
24519   return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,
24520                                  St->getMemOperand());
24521 }
24522 
24523 // Lower vector extended loads using a shuffle. If SSSE3 is not available we
24524 // may emit an illegal shuffle but the expansion is still better than scalar
24525 // code. We generate sext/sext_invec for SEXTLOADs if it's available, otherwise
24526 // we'll emit a shuffle and a arithmetic shift.
24527 // FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
24528 // TODO: It is possible to support ZExt by zeroing the undef values during
24529 // the shuffle phase or after the shuffle.
24530 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
24531                                  SelectionDAG &DAG) {
24532   MVT RegVT = Op.getSimpleValueType();
24533   assert(RegVT.isVector() && "We only custom lower vector loads.");
24534   assert(RegVT.isInteger() &&
24535          "We only custom lower integer vector loads.");
24536 
24537   LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
24538   SDLoc dl(Ld);
24539 
24540   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 loads.
24541   if (RegVT.getVectorElementType() == MVT::i1) {
24542     assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load");
24543     assert(RegVT.getVectorNumElements() <= 8 && "Unexpected VT");
24544     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24545            "Expected AVX512F without AVX512DQI");
24546 
24547     SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
24548                                 Ld->getPointerInfo(), Ld->getOriginalAlign(),
24549                                 Ld->getMemOperand()->getFlags());
24550 
24551     // Replace chain users with the new chain.
24552     assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!");
24553 
24554     SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
24555     Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
24556                       DAG.getBitcast(MVT::v16i1, Val),
24557                       DAG.getIntPtrConstant(0, dl));
24558     return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
24559   }
24560 
24561   return SDValue();
24562 }
24563 
24564 /// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
24565 /// each of which has no other use apart from the AND / OR.
24566 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
24567   Opc = Op.getOpcode();
24568   if (Opc != ISD::OR && Opc != ISD::AND)
24569     return false;
24570   return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
24571           Op.getOperand(0).hasOneUse() &&
24572           Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
24573           Op.getOperand(1).hasOneUse());
24574 }
24575 
24576 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
24577   SDValue Chain = Op.getOperand(0);
24578   SDValue Cond  = Op.getOperand(1);
24579   SDValue Dest  = Op.getOperand(2);
24580   SDLoc dl(Op);
24581 
24582   // Bail out when we don't have native compare instructions.
24583   if (Cond.getOpcode() == ISD::SETCC &&
24584       Cond.getOperand(0).getValueType() != MVT::f128 &&
24585       !isSoftF16(Cond.getOperand(0).getValueType(), Subtarget)) {
24586     SDValue LHS = Cond.getOperand(0);
24587     SDValue RHS = Cond.getOperand(1);
24588     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
24589 
24590     // Special case for
24591     // setcc([su]{add,sub,mul}o == 0)
24592     // setcc([su]{add,sub,mul}o != 1)
24593     if (ISD::isOverflowIntrOpRes(LHS) &&
24594         (CC == ISD::SETEQ || CC == ISD::SETNE) &&
24595         (isNullConstant(RHS) || isOneConstant(RHS))) {
24596       SDValue Value, Overflow;
24597       X86::CondCode X86Cond;
24598       std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG);
24599 
24600       if ((CC == ISD::SETEQ) == isNullConstant(RHS))
24601         X86Cond = X86::GetOppositeBranchCondition(X86Cond);
24602 
24603       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24604       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24605                          Overflow);
24606     }
24607 
24608     if (LHS.getSimpleValueType().isInteger()) {
24609       SDValue CCVal;
24610       SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
24611       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24612                          EFLAGS);
24613     }
24614 
24615     if (CC == ISD::SETOEQ) {
24616       // For FCMP_OEQ, we can emit
24617       // two branches instead of an explicit AND instruction with a
24618       // separate test. However, we only do this if this block doesn't
24619       // have a fall-through edge, because this requires an explicit
24620       // jmp when the condition is false.
24621       if (Op.getNode()->hasOneUse()) {
24622         SDNode *User = *Op.getNode()->use_begin();
24623         // Look for an unconditional branch following this conditional branch.
24624         // We need this because we need to reverse the successors in order
24625         // to implement FCMP_OEQ.
24626         if (User->getOpcode() == ISD::BR) {
24627           SDValue FalseBB = User->getOperand(1);
24628           SDNode *NewBR =
24629             DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
24630           assert(NewBR == User);
24631           (void)NewBR;
24632           Dest = FalseBB;
24633 
24634           SDValue Cmp =
24635               DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24636           SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24637           Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
24638                               CCVal, Cmp);
24639           CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24640           return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24641                              Cmp);
24642         }
24643       }
24644     } else if (CC == ISD::SETUNE) {
24645       // For FCMP_UNE, we can emit
24646       // two branches instead of an explicit OR instruction with a
24647       // separate test.
24648       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24649       SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24650       Chain =
24651           DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);
24652       CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24653       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24654                          Cmp);
24655     } else {
24656       X86::CondCode X86Cond =
24657           TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
24658       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24659       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24660       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24661                          Cmp);
24662     }
24663   }
24664 
24665   if (ISD::isOverflowIntrOpRes(Cond)) {
24666     SDValue Value, Overflow;
24667     X86::CondCode X86Cond;
24668     std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24669 
24670     SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24671     return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24672                        Overflow);
24673   }
24674 
24675   // Look past the truncate if the high bits are known zero.
24676   if (isTruncWithZeroHighBitsInput(Cond, DAG))
24677     Cond = Cond.getOperand(0);
24678 
24679   EVT CondVT = Cond.getValueType();
24680 
24681   // Add an AND with 1 if we don't already have one.
24682   if (!(Cond.getOpcode() == ISD::AND && isOneConstant(Cond.getOperand(1))))
24683     Cond =
24684         DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT));
24685 
24686   SDValue LHS = Cond;
24687   SDValue RHS = DAG.getConstant(0, dl, CondVT);
24688 
24689   SDValue CCVal;
24690   SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
24691   return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24692                      EFLAGS);
24693 }
24694 
24695 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
24696 // Calls to _alloca are needed to probe the stack when allocating more than 4k
24697 // bytes in one go. Touching the stack at 4K increments is necessary to ensure
24698 // that the guard pages used by the OS virtual memory manager are allocated in
24699 // correct sequence.
24700 SDValue
24701 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
24702                                            SelectionDAG &DAG) const {
24703   MachineFunction &MF = DAG.getMachineFunction();
24704   bool SplitStack = MF.shouldSplitStack();
24705   bool EmitStackProbeCall = hasStackProbeSymbol(MF);
24706   bool Lower = (Subtarget.isOSWindows() && !Subtarget.isTargetMachO()) ||
24707                SplitStack || EmitStackProbeCall;
24708   SDLoc dl(Op);
24709 
24710   // Get the inputs.
24711   SDNode *Node = Op.getNode();
24712   SDValue Chain = Op.getOperand(0);
24713   SDValue Size  = Op.getOperand(1);
24714   MaybeAlign Alignment(Op.getConstantOperandVal(2));
24715   EVT VT = Node->getValueType(0);
24716 
24717   // Chain the dynamic stack allocation so that it doesn't modify the stack
24718   // pointer when other instructions are using the stack.
24719   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
24720 
24721   bool Is64Bit = Subtarget.is64Bit();
24722   MVT SPTy = getPointerTy(DAG.getDataLayout());
24723 
24724   SDValue Result;
24725   if (!Lower) {
24726     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24727     Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
24728     assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
24729                     " not tell us which reg is the stack pointer!");
24730 
24731     const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
24732     const Align StackAlign = TFI.getStackAlign();
24733     if (hasInlineStackProbe(MF)) {
24734       MachineRegisterInfo &MRI = MF.getRegInfo();
24735 
24736       const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24737       Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24738       Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
24739       Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, SPTy, Chain,
24740                            DAG.getRegister(Vreg, SPTy));
24741     } else {
24742       SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
24743       Chain = SP.getValue(1);
24744       Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
24745     }
24746     if (Alignment && *Alignment > StackAlign)
24747       Result =
24748           DAG.getNode(ISD::AND, dl, VT, Result,
24749                       DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
24750     Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
24751   } else if (SplitStack) {
24752     MachineRegisterInfo &MRI = MF.getRegInfo();
24753 
24754     if (Is64Bit) {
24755       // The 64 bit implementation of segmented stacks needs to clobber both r10
24756       // r11. This makes it impossible to use it along with nested parameters.
24757       const Function &F = MF.getFunction();
24758       for (const auto &A : F.args()) {
24759         if (A.hasNestAttr())
24760           report_fatal_error("Cannot use segmented stacks with functions that "
24761                              "have nested arguments.");
24762       }
24763     }
24764 
24765     const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24766     Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24767     Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
24768     Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
24769                                 DAG.getRegister(Vreg, SPTy));
24770   } else {
24771     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
24772     Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size);
24773     MF.getInfo<X86MachineFunctionInfo>()->setHasDynAlloca(true);
24774 
24775     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
24776     Register SPReg = RegInfo->getStackRegister();
24777     SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
24778     Chain = SP.getValue(1);
24779 
24780     if (Alignment) {
24781       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
24782                        DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
24783       Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
24784     }
24785 
24786     Result = SP;
24787   }
24788 
24789   Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
24790 
24791   SDValue Ops[2] = {Result, Chain};
24792   return DAG.getMergeValues(Ops, dl);
24793 }
24794 
24795 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
24796   MachineFunction &MF = DAG.getMachineFunction();
24797   auto PtrVT = getPointerTy(MF.getDataLayout());
24798   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
24799 
24800   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
24801   SDLoc DL(Op);
24802 
24803   if (!Subtarget.is64Bit() ||
24804       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) {
24805     // vastart just stores the address of the VarArgsFrameIndex slot into the
24806     // memory location argument.
24807     SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
24808     return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
24809                         MachinePointerInfo(SV));
24810   }
24811 
24812   // __va_list_tag:
24813   //   gp_offset         (0 - 6 * 8)
24814   //   fp_offset         (48 - 48 + 8 * 16)
24815   //   overflow_arg_area (point to parameters coming in memory).
24816   //   reg_save_area
24817   SmallVector<SDValue, 8> MemOps;
24818   SDValue FIN = Op.getOperand(1);
24819   // Store gp_offset
24820   SDValue Store = DAG.getStore(
24821       Op.getOperand(0), DL,
24822       DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
24823       MachinePointerInfo(SV));
24824   MemOps.push_back(Store);
24825 
24826   // Store fp_offset
24827   FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(4), DL);
24828   Store = DAG.getStore(
24829       Op.getOperand(0), DL,
24830       DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
24831       MachinePointerInfo(SV, 4));
24832   MemOps.push_back(Store);
24833 
24834   // Store ptr to overflow_arg_area
24835   FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
24836   SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
24837   Store =
24838       DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
24839   MemOps.push_back(Store);
24840 
24841   // Store ptr to reg_save_area.
24842   FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
24843       Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
24844   SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
24845   Store = DAG.getStore(
24846       Op.getOperand(0), DL, RSFIN, FIN,
24847       MachinePointerInfo(SV, Subtarget.isTarget64BitLP64() ? 16 : 12));
24848   MemOps.push_back(Store);
24849   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
24850 }
24851 
24852 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
24853   assert(Subtarget.is64Bit() &&
24854          "LowerVAARG only handles 64-bit va_arg!");
24855   assert(Op.getNumOperands() == 4);
24856 
24857   MachineFunction &MF = DAG.getMachineFunction();
24858   if (Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()))
24859     // The Win64 ABI uses char* instead of a structure.
24860     return DAG.expandVAArg(Op.getNode());
24861 
24862   SDValue Chain = Op.getOperand(0);
24863   SDValue SrcPtr = Op.getOperand(1);
24864   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
24865   unsigned Align = Op.getConstantOperandVal(3);
24866   SDLoc dl(Op);
24867 
24868   EVT ArgVT = Op.getNode()->getValueType(0);
24869   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
24870   uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
24871   uint8_t ArgMode;
24872 
24873   // Decide which area this value should be read from.
24874   // TODO: Implement the AMD64 ABI in its entirety. This simple
24875   // selection mechanism works only for the basic types.
24876   assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");
24877   if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
24878     ArgMode = 2;  // Argument passed in XMM register. Use fp_offset.
24879   } else {
24880     assert(ArgVT.isInteger() && ArgSize <= 32 /*bytes*/ &&
24881            "Unhandled argument type in LowerVAARG");
24882     ArgMode = 1;  // Argument passed in GPR64 register(s). Use gp_offset.
24883   }
24884 
24885   if (ArgMode == 2) {
24886     // Make sure using fp_offset makes sense.
24887     assert(!Subtarget.useSoftFloat() &&
24888            !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) &&
24889            Subtarget.hasSSE1());
24890   }
24891 
24892   // Insert VAARG node into the DAG
24893   // VAARG returns two values: Variable Argument Address, Chain
24894   SDValue InstOps[] = {Chain, SrcPtr,
24895                        DAG.getTargetConstant(ArgSize, dl, MVT::i32),
24896                        DAG.getTargetConstant(ArgMode, dl, MVT::i8),
24897                        DAG.getTargetConstant(Align, dl, MVT::i32)};
24898   SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
24899   SDValue VAARG = DAG.getMemIntrinsicNode(
24900       Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl,
24901       VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
24902       /*Alignment=*/std::nullopt,
24903       MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
24904   Chain = VAARG.getValue(1);
24905 
24906   // Load the next argument and return it
24907   return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo());
24908 }
24909 
24910 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
24911                            SelectionDAG &DAG) {
24912   // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows,
24913   // where a va_list is still an i8*.
24914   assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!");
24915   if (Subtarget.isCallingConvWin64(
24916         DAG.getMachineFunction().getFunction().getCallingConv()))
24917     // Probably a Win64 va_copy.
24918     return DAG.expandVACopy(Op.getNode());
24919 
24920   SDValue Chain = Op.getOperand(0);
24921   SDValue DstPtr = Op.getOperand(1);
24922   SDValue SrcPtr = Op.getOperand(2);
24923   const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
24924   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
24925   SDLoc DL(Op);
24926 
24927   return DAG.getMemcpy(
24928       Chain, DL, DstPtr, SrcPtr,
24929       DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
24930       Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
24931       false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
24932 }
24933 
24934 // Helper to get immediate/variable SSE shift opcode from other shift opcodes.
24935 static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
24936   switch (Opc) {
24937   case ISD::SHL:
24938   case X86ISD::VSHL:
24939   case X86ISD::VSHLI:
24940     return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
24941   case ISD::SRL:
24942   case X86ISD::VSRL:
24943   case X86ISD::VSRLI:
24944     return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
24945   case ISD::SRA:
24946   case X86ISD::VSRA:
24947   case X86ISD::VSRAI:
24948     return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
24949   }
24950   llvm_unreachable("Unknown target vector shift node");
24951 }
24952 
24953 /// Handle vector element shifts where the shift amount is a constant.
24954 /// Takes immediate version of shift as input.
24955 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
24956                                           SDValue SrcOp, uint64_t ShiftAmt,
24957                                           SelectionDAG &DAG) {
24958   MVT ElementType = VT.getVectorElementType();
24959 
24960   // Bitcast the source vector to the output type, this is mainly necessary for
24961   // vXi8/vXi64 shifts.
24962   if (VT != SrcOp.getSimpleValueType())
24963     SrcOp = DAG.getBitcast(VT, SrcOp);
24964 
24965   // Fold this packed shift into its first operand if ShiftAmt is 0.
24966   if (ShiftAmt == 0)
24967     return SrcOp;
24968 
24969   // Check for ShiftAmt >= element width
24970   if (ShiftAmt >= ElementType.getSizeInBits()) {
24971     if (Opc == X86ISD::VSRAI)
24972       ShiftAmt = ElementType.getSizeInBits() - 1;
24973     else
24974       return DAG.getConstant(0, dl, VT);
24975   }
24976 
24977   assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
24978          && "Unknown target vector shift-by-constant node");
24979 
24980   // Fold this packed vector shift into a build vector if SrcOp is a
24981   // vector of Constants or UNDEFs.
24982   if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
24983     unsigned ShiftOpc;
24984     switch (Opc) {
24985     default: llvm_unreachable("Unknown opcode!");
24986     case X86ISD::VSHLI:
24987       ShiftOpc = ISD::SHL;
24988       break;
24989     case X86ISD::VSRLI:
24990       ShiftOpc = ISD::SRL;
24991       break;
24992     case X86ISD::VSRAI:
24993       ShiftOpc = ISD::SRA;
24994       break;
24995     }
24996 
24997     SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
24998     if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
24999       return C;
25000   }
25001 
25002   return DAG.getNode(Opc, dl, VT, SrcOp,
25003                      DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
25004 }
25005 
25006 /// Handle vector element shifts by a splat shift amount
25007 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
25008                                    SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
25009                                    const X86Subtarget &Subtarget,
25010                                    SelectionDAG &DAG) {
25011   MVT AmtVT = ShAmt.getSimpleValueType();
25012   assert(AmtVT.isVector() && "Vector shift type mismatch");
25013   assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
25014          "Illegal vector splat index");
25015 
25016   // Move the splat element to the bottom element.
25017   if (ShAmtIdx != 0) {
25018     SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
25019     Mask[0] = ShAmtIdx;
25020     ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
25021   }
25022 
25023   // Peek through any zext node if we can get back to a 128-bit source.
25024   if (AmtVT.getScalarSizeInBits() == 64 &&
25025       (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
25026        ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
25027       ShAmt.getOperand(0).getValueType().isSimple() &&
25028       ShAmt.getOperand(0).getValueType().is128BitVector()) {
25029     ShAmt = ShAmt.getOperand(0);
25030     AmtVT = ShAmt.getSimpleValueType();
25031   }
25032 
25033   // See if we can mask off the upper elements using the existing source node.
25034   // The shift uses the entire lower 64-bits of the amount vector, so no need to
25035   // do this for vXi64 types.
25036   bool IsMasked = false;
25037   if (AmtVT.getScalarSizeInBits() < 64) {
25038     if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
25039         ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25040       // If the shift amount has come from a scalar, then zero-extend the scalar
25041       // before moving to the vector.
25042       ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
25043       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25044       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
25045       AmtVT = MVT::v4i32;
25046       IsMasked = true;
25047     } else if (ShAmt.getOpcode() == ISD::AND) {
25048       // See if the shift amount is already masked (e.g. for rotation modulo),
25049       // then we can zero-extend it by setting all the other mask elements to
25050       // zero.
25051       SmallVector<SDValue> MaskElts(
25052           AmtVT.getVectorNumElements(),
25053           DAG.getConstant(0, dl, AmtVT.getScalarType()));
25054       MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
25055       SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
25056       if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
25057                                              {ShAmt.getOperand(1), Mask}))) {
25058         ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
25059         IsMasked = true;
25060       }
25061     }
25062   }
25063 
25064   // Extract if the shift amount vector is larger than 128-bits.
25065   if (AmtVT.getSizeInBits() > 128) {
25066     ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
25067     AmtVT = ShAmt.getSimpleValueType();
25068   }
25069 
25070   // Zero-extend bottom element to v2i64 vector type, either by extension or
25071   // shuffle masking.
25072   if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
25073     if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
25074                                 ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
25075       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
25076     } else if (Subtarget.hasSSE41()) {
25077       ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
25078                           MVT::v2i64, ShAmt);
25079     } else {
25080       SDValue ByteShift = DAG.getTargetConstant(
25081           (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
25082       ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
25083       ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25084                           ByteShift);
25085       ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25086                           ByteShift);
25087     }
25088   }
25089 
25090   // Change opcode to non-immediate version.
25091   Opc = getTargetVShiftUniformOpcode(Opc, true);
25092 
25093   // The return type has to be a 128-bit type with the same element
25094   // type as the input type.
25095   MVT EltVT = VT.getVectorElementType();
25096   MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
25097 
25098   ShAmt = DAG.getBitcast(ShVT, ShAmt);
25099   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
25100 }
25101 
25102 /// Return Mask with the necessary casting or extending
25103 /// for \p Mask according to \p MaskVT when lowering masking intrinsics
25104 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
25105                            const X86Subtarget &Subtarget, SelectionDAG &DAG,
25106                            const SDLoc &dl) {
25107 
25108   if (isAllOnesConstant(Mask))
25109     return DAG.getConstant(1, dl, MaskVT);
25110   if (X86::isZeroNode(Mask))
25111     return DAG.getConstant(0, dl, MaskVT);
25112 
25113   assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!");
25114 
25115   if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {
25116     assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!");
25117     assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
25118     // In case 32bit mode, bitcast i64 is illegal, extend/split it.
25119     SDValue Lo, Hi;
25120     std::tie(Lo, Hi) = DAG.SplitScalar(Mask, dl, MVT::i32, MVT::i32);
25121     Lo = DAG.getBitcast(MVT::v32i1, Lo);
25122     Hi = DAG.getBitcast(MVT::v32i1, Hi);
25123     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
25124   } else {
25125     MVT BitcastVT = MVT::getVectorVT(MVT::i1,
25126                                      Mask.getSimpleValueType().getSizeInBits());
25127     // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
25128     // are extracted by EXTRACT_SUBVECTOR.
25129     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
25130                        DAG.getBitcast(BitcastVT, Mask),
25131                        DAG.getIntPtrConstant(0, dl));
25132   }
25133 }
25134 
25135 /// Return (and \p Op, \p Mask) for compare instructions or
25136 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
25137 /// necessary casting or extending for \p Mask when lowering masking intrinsics
25138 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
25139                                     SDValue PreservedSrc,
25140                                     const X86Subtarget &Subtarget,
25141                                     SelectionDAG &DAG) {
25142   MVT VT = Op.getSimpleValueType();
25143   MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
25144   unsigned OpcodeSelect = ISD::VSELECT;
25145   SDLoc dl(Op);
25146 
25147   if (isAllOnesConstant(Mask))
25148     return Op;
25149 
25150   SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25151 
25152   if (PreservedSrc.isUndef())
25153     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25154   return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
25155 }
25156 
25157 /// Creates an SDNode for a predicated scalar operation.
25158 /// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
25159 /// The mask is coming as MVT::i8 and it should be transformed
25160 /// to MVT::v1i1 while lowering masking intrinsics.
25161 /// The main difference between ScalarMaskingNode and VectorMaskingNode is using
25162 /// "X86select" instead of "vselect". We just can't create the "vselect" node
25163 /// for a scalar instruction.
25164 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
25165                                     SDValue PreservedSrc,
25166                                     const X86Subtarget &Subtarget,
25167                                     SelectionDAG &DAG) {
25168 
25169   if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask))
25170     if (MaskConst->getZExtValue() & 0x1)
25171       return Op;
25172 
25173   MVT VT = Op.getSimpleValueType();
25174   SDLoc dl(Op);
25175 
25176   assert(Mask.getValueType() == MVT::i8 && "Unexpect type");
25177   SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
25178                               DAG.getBitcast(MVT::v8i1, Mask),
25179                               DAG.getIntPtrConstant(0, dl));
25180   if (Op.getOpcode() == X86ISD::FSETCCM ||
25181       Op.getOpcode() == X86ISD::FSETCCM_SAE ||
25182       Op.getOpcode() == X86ISD::VFPCLASSS)
25183     return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
25184 
25185   if (PreservedSrc.isUndef())
25186     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25187   return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
25188 }
25189 
25190 static int getSEHRegistrationNodeSize(const Function *Fn) {
25191   if (!Fn->hasPersonalityFn())
25192     report_fatal_error(
25193         "querying registration node size for function without personality");
25194   // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
25195   // WinEHStatePass for the full struct definition.
25196   switch (classifyEHPersonality(Fn->getPersonalityFn())) {
25197   case EHPersonality::MSVC_X86SEH: return 24;
25198   case EHPersonality::MSVC_CXX: return 16;
25199   default: break;
25200   }
25201   report_fatal_error(
25202       "can only recover FP for 32-bit MSVC EH personality functions");
25203 }
25204 
25205 /// When the MSVC runtime transfers control to us, either to an outlined
25206 /// function or when returning to a parent frame after catching an exception, we
25207 /// recover the parent frame pointer by doing arithmetic on the incoming EBP.
25208 /// Here's the math:
25209 ///   RegNodeBase = EntryEBP - RegNodeSize
25210 ///   ParentFP = RegNodeBase - ParentFrameOffset
25211 /// Subtracting RegNodeSize takes us to the offset of the registration node, and
25212 /// subtracting the offset (negative on x86) takes us back to the parent FP.
25213 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
25214                                    SDValue EntryEBP) {
25215   MachineFunction &MF = DAG.getMachineFunction();
25216   SDLoc dl;
25217 
25218   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25219   MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
25220 
25221   // It's possible that the parent function no longer has a personality function
25222   // if the exceptional code was optimized away, in which case we just return
25223   // the incoming EBP.
25224   if (!Fn->hasPersonalityFn())
25225     return EntryEBP;
25226 
25227   // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
25228   // registration, or the .set_setframe offset.
25229   MCSymbol *OffsetSym =
25230       MF.getMMI().getContext().getOrCreateParentFrameOffsetSymbol(
25231           GlobalValue::dropLLVMManglingEscape(Fn->getName()));
25232   SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
25233   SDValue ParentFrameOffset =
25234       DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
25235 
25236   // Return EntryEBP + ParentFrameOffset for x64. This adjusts from RSP after
25237   // prologue to RBP in the parent function.
25238   const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
25239   if (Subtarget.is64Bit())
25240     return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset);
25241 
25242   int RegNodeSize = getSEHRegistrationNodeSize(Fn);
25243   // RegNodeBase = EntryEBP - RegNodeSize
25244   // ParentFP = RegNodeBase - ParentFrameOffset
25245   SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
25246                                     DAG.getConstant(RegNodeSize, dl, PtrVT));
25247   return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset);
25248 }
25249 
25250 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
25251                                                    SelectionDAG &DAG) const {
25252   // Helper to detect if the operand is CUR_DIRECTION rounding mode.
25253   auto isRoundModeCurDirection = [](SDValue Rnd) {
25254     if (auto *C = dyn_cast<ConstantSDNode>(Rnd))
25255       return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION;
25256 
25257     return false;
25258   };
25259   auto isRoundModeSAE = [](SDValue Rnd) {
25260     if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25261       unsigned RC = C->getZExtValue();
25262       if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25263         // Clear the NO_EXC bit and check remaining bits.
25264         RC ^= X86::STATIC_ROUNDING::NO_EXC;
25265         // As a convenience we allow no other bits or explicitly
25266         // current direction.
25267         return RC == 0 || RC == X86::STATIC_ROUNDING::CUR_DIRECTION;
25268       }
25269     }
25270 
25271     return false;
25272   };
25273   auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
25274     if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25275       RC = C->getZExtValue();
25276       if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25277         // Clear the NO_EXC bit and check remaining bits.
25278         RC ^= X86::STATIC_ROUNDING::NO_EXC;
25279         return RC == X86::STATIC_ROUNDING::TO_NEAREST_INT ||
25280                RC == X86::STATIC_ROUNDING::TO_NEG_INF ||
25281                RC == X86::STATIC_ROUNDING::TO_POS_INF ||
25282                RC == X86::STATIC_ROUNDING::TO_ZERO;
25283       }
25284     }
25285 
25286     return false;
25287   };
25288 
25289   SDLoc dl(Op);
25290   unsigned IntNo = Op.getConstantOperandVal(0);
25291   MVT VT = Op.getSimpleValueType();
25292   const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
25293 
25294   // Propagate flags from original node to transformed node(s).
25295   SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
25296 
25297   if (IntrData) {
25298     switch(IntrData->Type) {
25299     case INTR_TYPE_1OP: {
25300       // We specify 2 possible opcodes for intrinsics with rounding modes.
25301       // First, we check if the intrinsic may have non-default rounding mode,
25302       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25303       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25304       if (IntrWithRoundingModeOpcode != 0) {
25305         SDValue Rnd = Op.getOperand(2);
25306         unsigned RC = 0;
25307         if (isRoundModeSAEToX(Rnd, RC))
25308           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25309                              Op.getOperand(1),
25310                              DAG.getTargetConstant(RC, dl, MVT::i32));
25311         if (!isRoundModeCurDirection(Rnd))
25312           return SDValue();
25313       }
25314       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25315                          Op.getOperand(1));
25316     }
25317     case INTR_TYPE_1OP_SAE: {
25318       SDValue Sae = Op.getOperand(2);
25319 
25320       unsigned Opc;
25321       if (isRoundModeCurDirection(Sae))
25322         Opc = IntrData->Opc0;
25323       else if (isRoundModeSAE(Sae))
25324         Opc = IntrData->Opc1;
25325       else
25326         return SDValue();
25327 
25328       return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1));
25329     }
25330     case INTR_TYPE_2OP: {
25331       SDValue Src2 = Op.getOperand(2);
25332 
25333       // We specify 2 possible opcodes for intrinsics with rounding modes.
25334       // First, we check if the intrinsic may have non-default rounding mode,
25335       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25336       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25337       if (IntrWithRoundingModeOpcode != 0) {
25338         SDValue Rnd = Op.getOperand(3);
25339         unsigned RC = 0;
25340         if (isRoundModeSAEToX(Rnd, RC))
25341           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25342                              Op.getOperand(1), Src2,
25343                              DAG.getTargetConstant(RC, dl, MVT::i32));
25344         if (!isRoundModeCurDirection(Rnd))
25345           return SDValue();
25346       }
25347 
25348       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25349                          Op.getOperand(1), Src2);
25350     }
25351     case INTR_TYPE_2OP_SAE: {
25352       SDValue Sae = Op.getOperand(3);
25353 
25354       unsigned Opc;
25355       if (isRoundModeCurDirection(Sae))
25356         Opc = IntrData->Opc0;
25357       else if (isRoundModeSAE(Sae))
25358         Opc = IntrData->Opc1;
25359       else
25360         return SDValue();
25361 
25362       return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
25363                          Op.getOperand(2));
25364     }
25365     case INTR_TYPE_3OP:
25366     case INTR_TYPE_3OP_IMM8: {
25367       SDValue Src1 = Op.getOperand(1);
25368       SDValue Src2 = Op.getOperand(2);
25369       SDValue Src3 = Op.getOperand(3);
25370 
25371       if (IntrData->Type == INTR_TYPE_3OP_IMM8 &&
25372           Src3.getValueType() != MVT::i8) {
25373         Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8);
25374       }
25375 
25376       // We specify 2 possible opcodes for intrinsics with rounding modes.
25377       // First, we check if the intrinsic may have non-default rounding mode,
25378       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25379       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25380       if (IntrWithRoundingModeOpcode != 0) {
25381         SDValue Rnd = Op.getOperand(4);
25382         unsigned RC = 0;
25383         if (isRoundModeSAEToX(Rnd, RC))
25384           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25385                              Src1, Src2, Src3,
25386                              DAG.getTargetConstant(RC, dl, MVT::i32));
25387         if (!isRoundModeCurDirection(Rnd))
25388           return SDValue();
25389       }
25390 
25391       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25392                          {Src1, Src2, Src3});
25393     }
25394     case INTR_TYPE_4OP_IMM8: {
25395       assert(Op.getOperand(4)->getOpcode() == ISD::TargetConstant);
25396       SDValue Src4 = Op.getOperand(4);
25397       if (Src4.getValueType() != MVT::i8) {
25398         Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8);
25399       }
25400 
25401       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25402                          Op.getOperand(1), Op.getOperand(2), Op.getOperand(3),
25403                          Src4);
25404     }
25405     case INTR_TYPE_1OP_MASK: {
25406       SDValue Src = Op.getOperand(1);
25407       SDValue PassThru = Op.getOperand(2);
25408       SDValue Mask = Op.getOperand(3);
25409       // We add rounding mode to the Node when
25410       //   - RC Opcode is specified and
25411       //   - RC is not "current direction".
25412       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25413       if (IntrWithRoundingModeOpcode != 0) {
25414         SDValue Rnd = Op.getOperand(4);
25415         unsigned RC = 0;
25416         if (isRoundModeSAEToX(Rnd, RC))
25417           return getVectorMaskingNode(
25418               DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25419                           Src, DAG.getTargetConstant(RC, dl, MVT::i32)),
25420               Mask, PassThru, Subtarget, DAG);
25421         if (!isRoundModeCurDirection(Rnd))
25422           return SDValue();
25423       }
25424       return getVectorMaskingNode(
25425           DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
25426           Subtarget, DAG);
25427     }
25428     case INTR_TYPE_1OP_MASK_SAE: {
25429       SDValue Src = Op.getOperand(1);
25430       SDValue PassThru = Op.getOperand(2);
25431       SDValue Mask = Op.getOperand(3);
25432       SDValue Rnd = Op.getOperand(4);
25433 
25434       unsigned Opc;
25435       if (isRoundModeCurDirection(Rnd))
25436         Opc = IntrData->Opc0;
25437       else if (isRoundModeSAE(Rnd))
25438         Opc = IntrData->Opc1;
25439       else
25440         return SDValue();
25441 
25442       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru,
25443                                   Subtarget, DAG);
25444     }
25445     case INTR_TYPE_SCALAR_MASK: {
25446       SDValue Src1 = Op.getOperand(1);
25447       SDValue Src2 = Op.getOperand(2);
25448       SDValue passThru = Op.getOperand(3);
25449       SDValue Mask = Op.getOperand(4);
25450       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25451       // There are 2 kinds of intrinsics in this group:
25452       // (1) With suppress-all-exceptions (sae) or rounding mode- 6 operands
25453       // (2) With rounding mode and sae - 7 operands.
25454       bool HasRounding = IntrWithRoundingModeOpcode != 0;
25455       if (Op.getNumOperands() == (5U + HasRounding)) {
25456         if (HasRounding) {
25457           SDValue Rnd = Op.getOperand(5);
25458           unsigned RC = 0;
25459           if (isRoundModeSAEToX(Rnd, RC))
25460             return getScalarMaskingNode(
25461                 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2,
25462                             DAG.getTargetConstant(RC, dl, MVT::i32)),
25463                 Mask, passThru, Subtarget, DAG);
25464           if (!isRoundModeCurDirection(Rnd))
25465             return SDValue();
25466         }
25467         return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
25468                                                 Src2),
25469                                     Mask, passThru, Subtarget, DAG);
25470       }
25471 
25472       assert(Op.getNumOperands() == (6U + HasRounding) &&
25473              "Unexpected intrinsic form");
25474       SDValue RoundingMode = Op.getOperand(5);
25475       unsigned Opc = IntrData->Opc0;
25476       if (HasRounding) {
25477         SDValue Sae = Op.getOperand(6);
25478         if (isRoundModeSAE(Sae))
25479           Opc = IntrWithRoundingModeOpcode;
25480         else if (!isRoundModeCurDirection(Sae))
25481           return SDValue();
25482       }
25483       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
25484                                               Src2, RoundingMode),
25485                                   Mask, passThru, Subtarget, DAG);
25486     }
25487     case INTR_TYPE_SCALAR_MASK_RND: {
25488       SDValue Src1 = Op.getOperand(1);
25489       SDValue Src2 = Op.getOperand(2);
25490       SDValue passThru = Op.getOperand(3);
25491       SDValue Mask = Op.getOperand(4);
25492       SDValue Rnd = Op.getOperand(5);
25493 
25494       SDValue NewOp;
25495       unsigned RC = 0;
25496       if (isRoundModeCurDirection(Rnd))
25497         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25498       else if (isRoundModeSAEToX(Rnd, RC))
25499         NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25500                             DAG.getTargetConstant(RC, dl, MVT::i32));
25501       else
25502         return SDValue();
25503 
25504       return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG);
25505     }
25506     case INTR_TYPE_SCALAR_MASK_SAE: {
25507       SDValue Src1 = Op.getOperand(1);
25508       SDValue Src2 = Op.getOperand(2);
25509       SDValue passThru = Op.getOperand(3);
25510       SDValue Mask = Op.getOperand(4);
25511       SDValue Sae = Op.getOperand(5);
25512       unsigned Opc;
25513       if (isRoundModeCurDirection(Sae))
25514         Opc = IntrData->Opc0;
25515       else if (isRoundModeSAE(Sae))
25516         Opc = IntrData->Opc1;
25517       else
25518         return SDValue();
25519 
25520       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25521                                   Mask, passThru, Subtarget, DAG);
25522     }
25523     case INTR_TYPE_2OP_MASK: {
25524       SDValue Src1 = Op.getOperand(1);
25525       SDValue Src2 = Op.getOperand(2);
25526       SDValue PassThru = Op.getOperand(3);
25527       SDValue Mask = Op.getOperand(4);
25528       SDValue NewOp;
25529       if (IntrData->Opc1 != 0) {
25530         SDValue Rnd = Op.getOperand(5);
25531         unsigned RC = 0;
25532         if (isRoundModeSAEToX(Rnd, RC))
25533           NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25534                               DAG.getTargetConstant(RC, dl, MVT::i32));
25535         else if (!isRoundModeCurDirection(Rnd))
25536           return SDValue();
25537       }
25538       if (!NewOp)
25539         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25540       return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25541     }
25542     case INTR_TYPE_2OP_MASK_SAE: {
25543       SDValue Src1 = Op.getOperand(1);
25544       SDValue Src2 = Op.getOperand(2);
25545       SDValue PassThru = Op.getOperand(3);
25546       SDValue Mask = Op.getOperand(4);
25547 
25548       unsigned Opc = IntrData->Opc0;
25549       if (IntrData->Opc1 != 0) {
25550         SDValue Sae = Op.getOperand(5);
25551         if (isRoundModeSAE(Sae))
25552           Opc = IntrData->Opc1;
25553         else if (!isRoundModeCurDirection(Sae))
25554           return SDValue();
25555       }
25556 
25557       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25558                                   Mask, PassThru, Subtarget, DAG);
25559     }
25560     case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
25561       SDValue Src1 = Op.getOperand(1);
25562       SDValue Src2 = Op.getOperand(2);
25563       SDValue Src3 = Op.getOperand(3);
25564       SDValue PassThru = Op.getOperand(4);
25565       SDValue Mask = Op.getOperand(5);
25566       SDValue Sae = Op.getOperand(6);
25567       unsigned Opc;
25568       if (isRoundModeCurDirection(Sae))
25569         Opc = IntrData->Opc0;
25570       else if (isRoundModeSAE(Sae))
25571         Opc = IntrData->Opc1;
25572       else
25573         return SDValue();
25574 
25575       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25576                                   Mask, PassThru, Subtarget, DAG);
25577     }
25578     case INTR_TYPE_3OP_MASK_SAE: {
25579       SDValue Src1 = Op.getOperand(1);
25580       SDValue Src2 = Op.getOperand(2);
25581       SDValue Src3 = Op.getOperand(3);
25582       SDValue PassThru = Op.getOperand(4);
25583       SDValue Mask = Op.getOperand(5);
25584 
25585       unsigned Opc = IntrData->Opc0;
25586       if (IntrData->Opc1 != 0) {
25587         SDValue Sae = Op.getOperand(6);
25588         if (isRoundModeSAE(Sae))
25589           Opc = IntrData->Opc1;
25590         else if (!isRoundModeCurDirection(Sae))
25591           return SDValue();
25592       }
25593       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25594                                   Mask, PassThru, Subtarget, DAG);
25595     }
25596     case BLENDV: {
25597       SDValue Src1 = Op.getOperand(1);
25598       SDValue Src2 = Op.getOperand(2);
25599       SDValue Src3 = Op.getOperand(3);
25600 
25601       EVT MaskVT = Src3.getValueType().changeVectorElementTypeToInteger();
25602       Src3 = DAG.getBitcast(MaskVT, Src3);
25603 
25604       // Reverse the operands to match VSELECT order.
25605       return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
25606     }
25607     case VPERM_2OP : {
25608       SDValue Src1 = Op.getOperand(1);
25609       SDValue Src2 = Op.getOperand(2);
25610 
25611       // Swap Src1 and Src2 in the node creation
25612       return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
25613     }
25614     case CFMA_OP_MASKZ:
25615     case CFMA_OP_MASK: {
25616       SDValue Src1 = Op.getOperand(1);
25617       SDValue Src2 = Op.getOperand(2);
25618       SDValue Src3 = Op.getOperand(3);
25619       SDValue Mask = Op.getOperand(4);
25620       MVT VT = Op.getSimpleValueType();
25621 
25622       SDValue PassThru = Src3;
25623       if (IntrData->Type == CFMA_OP_MASKZ)
25624         PassThru = getZeroVector(VT, Subtarget, DAG, dl);
25625 
25626       // We add rounding mode to the Node when
25627       //   - RC Opcode is specified and
25628       //   - RC is not "current direction".
25629       SDValue NewOp;
25630       if (IntrData->Opc1 != 0) {
25631         SDValue Rnd = Op.getOperand(5);
25632         unsigned RC = 0;
25633         if (isRoundModeSAEToX(Rnd, RC))
25634           NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3,
25635                               DAG.getTargetConstant(RC, dl, MVT::i32));
25636         else if (!isRoundModeCurDirection(Rnd))
25637           return SDValue();
25638       }
25639       if (!NewOp)
25640         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3);
25641       return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25642     }
25643     case IFMA_OP:
25644       // NOTE: We need to swizzle the operands to pass the multiply operands
25645       // first.
25646       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25647                          Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
25648     case FPCLASSS: {
25649       SDValue Src1 = Op.getOperand(1);
25650       SDValue Imm = Op.getOperand(2);
25651       SDValue Mask = Op.getOperand(3);
25652       SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
25653       SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
25654                                                  Subtarget, DAG);
25655       // Need to fill with zeros to ensure the bitcast will produce zeroes
25656       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25657       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25658                                 DAG.getConstant(0, dl, MVT::v8i1),
25659                                 FPclassMask, DAG.getIntPtrConstant(0, dl));
25660       return DAG.getBitcast(MVT::i8, Ins);
25661     }
25662 
25663     case CMP_MASK_CC: {
25664       MVT MaskVT = Op.getSimpleValueType();
25665       SDValue CC = Op.getOperand(3);
25666       SDValue Mask = Op.getOperand(4);
25667       // We specify 2 possible opcodes for intrinsics with rounding modes.
25668       // First, we check if the intrinsic may have non-default rounding mode,
25669       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25670       if (IntrData->Opc1 != 0) {
25671         SDValue Sae = Op.getOperand(5);
25672         if (isRoundModeSAE(Sae))
25673           return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
25674                              Op.getOperand(2), CC, Mask, Sae);
25675         if (!isRoundModeCurDirection(Sae))
25676           return SDValue();
25677       }
25678       //default rounding mode
25679       return DAG.getNode(IntrData->Opc0, dl, MaskVT,
25680                          {Op.getOperand(1), Op.getOperand(2), CC, Mask});
25681     }
25682     case CMP_MASK_SCALAR_CC: {
25683       SDValue Src1 = Op.getOperand(1);
25684       SDValue Src2 = Op.getOperand(2);
25685       SDValue CC = Op.getOperand(3);
25686       SDValue Mask = Op.getOperand(4);
25687 
25688       SDValue Cmp;
25689       if (IntrData->Opc1 != 0) {
25690         SDValue Sae = Op.getOperand(5);
25691         if (isRoundModeSAE(Sae))
25692           Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);
25693         else if (!isRoundModeCurDirection(Sae))
25694           return SDValue();
25695       }
25696       //default rounding mode
25697       if (!Cmp.getNode())
25698         Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
25699 
25700       SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
25701                                              Subtarget, DAG);
25702       // Need to fill with zeros to ensure the bitcast will produce zeroes
25703       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25704       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25705                                 DAG.getConstant(0, dl, MVT::v8i1),
25706                                 CmpMask, DAG.getIntPtrConstant(0, dl));
25707       return DAG.getBitcast(MVT::i8, Ins);
25708     }
25709     case COMI: { // Comparison intrinsics
25710       ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
25711       SDValue LHS = Op.getOperand(1);
25712       SDValue RHS = Op.getOperand(2);
25713       // Some conditions require the operands to be swapped.
25714       if (CC == ISD::SETLT || CC == ISD::SETLE)
25715         std::swap(LHS, RHS);
25716 
25717       SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
25718       SDValue SetCC;
25719       switch (CC) {
25720       case ISD::SETEQ: { // (ZF = 0 and PF = 0)
25721         SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
25722         SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
25723         SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);
25724         break;
25725       }
25726       case ISD::SETNE: { // (ZF = 1 or PF = 1)
25727         SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
25728         SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
25729         SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);
25730         break;
25731       }
25732       case ISD::SETGT: // (CF = 0 and ZF = 0)
25733       case ISD::SETLT: { // Condition opposite to GT. Operands swapped above.
25734         SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
25735         break;
25736       }
25737       case ISD::SETGE: // CF = 0
25738       case ISD::SETLE: // Condition opposite to GE. Operands swapped above.
25739         SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
25740         break;
25741       default:
25742         llvm_unreachable("Unexpected illegal condition!");
25743       }
25744       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
25745     }
25746     case COMI_RM: { // Comparison intrinsics with Sae
25747       SDValue LHS = Op.getOperand(1);
25748       SDValue RHS = Op.getOperand(2);
25749       unsigned CondVal = Op.getConstantOperandVal(3);
25750       SDValue Sae = Op.getOperand(4);
25751 
25752       SDValue FCmp;
25753       if (isRoundModeCurDirection(Sae))
25754         FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
25755                            DAG.getTargetConstant(CondVal, dl, MVT::i8));
25756       else if (isRoundModeSAE(Sae))
25757         FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
25758                            DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
25759       else
25760         return SDValue();
25761       // Need to fill with zeros to ensure the bitcast will produce zeroes
25762       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25763       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
25764                                 DAG.getConstant(0, dl, MVT::v16i1),
25765                                 FCmp, DAG.getIntPtrConstant(0, dl));
25766       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
25767                          DAG.getBitcast(MVT::i16, Ins));
25768     }
25769     case VSHIFT: {
25770       SDValue SrcOp = Op.getOperand(1);
25771       SDValue ShAmt = Op.getOperand(2);
25772       assert(ShAmt.getValueType() == MVT::i32 &&
25773              "Unexpected VSHIFT amount type");
25774 
25775       // Catch shift-by-constant.
25776       if (auto *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
25777         return getTargetVShiftByConstNode(IntrData->Opc0, dl,
25778                                           Op.getSimpleValueType(), SrcOp,
25779                                           CShAmt->getZExtValue(), DAG);
25780 
25781       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25782       return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
25783                                  SrcOp, ShAmt, 0, Subtarget, DAG);
25784     }
25785     case COMPRESS_EXPAND_IN_REG: {
25786       SDValue Mask = Op.getOperand(3);
25787       SDValue DataToCompress = Op.getOperand(1);
25788       SDValue PassThru = Op.getOperand(2);
25789       if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is
25790         return Op.getOperand(1);
25791 
25792       // Avoid false dependency.
25793       if (PassThru.isUndef())
25794         PassThru = getZeroVector(VT, Subtarget, DAG, dl);
25795 
25796       return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
25797                          Mask);
25798     }
25799     case FIXUPIMM:
25800     case FIXUPIMM_MASKZ: {
25801       SDValue Src1 = Op.getOperand(1);
25802       SDValue Src2 = Op.getOperand(2);
25803       SDValue Src3 = Op.getOperand(3);
25804       SDValue Imm = Op.getOperand(4);
25805       SDValue Mask = Op.getOperand(5);
25806       SDValue Passthru = (IntrData->Type == FIXUPIMM)
25807                              ? Src1
25808                              : getZeroVector(VT, Subtarget, DAG, dl);
25809 
25810       unsigned Opc = IntrData->Opc0;
25811       if (IntrData->Opc1 != 0) {
25812         SDValue Sae = Op.getOperand(6);
25813         if (isRoundModeSAE(Sae))
25814           Opc = IntrData->Opc1;
25815         else if (!isRoundModeCurDirection(Sae))
25816           return SDValue();
25817       }
25818 
25819       SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
25820 
25821       if (Opc == X86ISD::VFIXUPIMM || Opc == X86ISD::VFIXUPIMM_SAE)
25822         return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
25823 
25824       return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
25825     }
25826     case ROUNDP: {
25827       assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode");
25828       // Clear the upper bits of the rounding immediate so that the legacy
25829       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
25830       auto Round = cast<ConstantSDNode>(Op.getOperand(2));
25831       SDValue RoundingMode =
25832           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
25833       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25834                          Op.getOperand(1), RoundingMode);
25835     }
25836     case ROUNDS: {
25837       assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode");
25838       // Clear the upper bits of the rounding immediate so that the legacy
25839       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
25840       auto Round = cast<ConstantSDNode>(Op.getOperand(3));
25841       SDValue RoundingMode =
25842           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
25843       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25844                          Op.getOperand(1), Op.getOperand(2), RoundingMode);
25845     }
25846     case BEXTRI: {
25847       assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode");
25848 
25849       uint64_t Imm = Op.getConstantOperandVal(2);
25850       SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
25851                                               Op.getValueType());
25852       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25853                          Op.getOperand(1), Control);
25854     }
25855     // ADC/SBB
25856     case ADX: {
25857       SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
25858       SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);
25859 
25860       SDValue Res;
25861       // If the carry in is zero, then we should just use ADD/SUB instead of
25862       // ADC/SBB.
25863       if (isNullConstant(Op.getOperand(1))) {
25864         Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2),
25865                           Op.getOperand(3));
25866       } else {
25867         SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
25868                                     DAG.getConstant(-1, dl, MVT::i8));
25869         Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
25870                           Op.getOperand(3), GenCF.getValue(1));
25871       }
25872       SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
25873       SDValue Results[] = { SetCC, Res };
25874       return DAG.getMergeValues(Results, dl);
25875     }
25876     case CVTPD2PS_MASK:
25877     case CVTPD2DQ_MASK:
25878     case CVTQQ2PS_MASK:
25879     case TRUNCATE_TO_REG: {
25880       SDValue Src = Op.getOperand(1);
25881       SDValue PassThru = Op.getOperand(2);
25882       SDValue Mask = Op.getOperand(3);
25883 
25884       if (isAllOnesConstant(Mask))
25885         return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
25886 
25887       MVT SrcVT = Src.getSimpleValueType();
25888       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
25889       Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25890       return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
25891                          {Src, PassThru, Mask});
25892     }
25893     case CVTPS2PH_MASK: {
25894       SDValue Src = Op.getOperand(1);
25895       SDValue Rnd = Op.getOperand(2);
25896       SDValue PassThru = Op.getOperand(3);
25897       SDValue Mask = Op.getOperand(4);
25898 
25899       unsigned RC = 0;
25900       unsigned Opc = IntrData->Opc0;
25901       bool SAE = Src.getValueType().is512BitVector() &&
25902                  (isRoundModeSAEToX(Rnd, RC) || isRoundModeSAE(Rnd));
25903       if (SAE) {
25904         Opc = X86ISD::CVTPS2PH_SAE;
25905         Rnd = DAG.getTargetConstant(RC, dl, MVT::i32);
25906       }
25907 
25908       if (isAllOnesConstant(Mask))
25909         return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd);
25910 
25911       if (SAE)
25912         Opc = X86ISD::MCVTPS2PH_SAE;
25913       else
25914         Opc = IntrData->Opc1;
25915       MVT SrcVT = Src.getSimpleValueType();
25916       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
25917       Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25918       return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd, PassThru, Mask);
25919     }
25920     case CVTNEPS2BF16_MASK: {
25921       SDValue Src = Op.getOperand(1);
25922       SDValue PassThru = Op.getOperand(2);
25923       SDValue Mask = Op.getOperand(3);
25924 
25925       if (ISD::isBuildVectorAllOnes(Mask.getNode()))
25926         return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
25927 
25928       // Break false dependency.
25929       if (PassThru.isUndef())
25930         PassThru = DAG.getConstant(0, dl, PassThru.getValueType());
25931 
25932       return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
25933                          Mask);
25934     }
25935     default:
25936       break;
25937     }
25938   }
25939 
25940   switch (IntNo) {
25941   default: return SDValue();    // Don't custom lower most intrinsics.
25942 
25943   // ptest and testp intrinsics. The intrinsic these come from are designed to
25944   // return an integer value, not just an instruction so lower it to the ptest
25945   // or testp pattern and a setcc for the result.
25946   case Intrinsic::x86_avx512_ktestc_b:
25947   case Intrinsic::x86_avx512_ktestc_w:
25948   case Intrinsic::x86_avx512_ktestc_d:
25949   case Intrinsic::x86_avx512_ktestc_q:
25950   case Intrinsic::x86_avx512_ktestz_b:
25951   case Intrinsic::x86_avx512_ktestz_w:
25952   case Intrinsic::x86_avx512_ktestz_d:
25953   case Intrinsic::x86_avx512_ktestz_q:
25954   case Intrinsic::x86_sse41_ptestz:
25955   case Intrinsic::x86_sse41_ptestc:
25956   case Intrinsic::x86_sse41_ptestnzc:
25957   case Intrinsic::x86_avx_ptestz_256:
25958   case Intrinsic::x86_avx_ptestc_256:
25959   case Intrinsic::x86_avx_ptestnzc_256:
25960   case Intrinsic::x86_avx_vtestz_ps:
25961   case Intrinsic::x86_avx_vtestc_ps:
25962   case Intrinsic::x86_avx_vtestnzc_ps:
25963   case Intrinsic::x86_avx_vtestz_pd:
25964   case Intrinsic::x86_avx_vtestc_pd:
25965   case Intrinsic::x86_avx_vtestnzc_pd:
25966   case Intrinsic::x86_avx_vtestz_ps_256:
25967   case Intrinsic::x86_avx_vtestc_ps_256:
25968   case Intrinsic::x86_avx_vtestnzc_ps_256:
25969   case Intrinsic::x86_avx_vtestz_pd_256:
25970   case Intrinsic::x86_avx_vtestc_pd_256:
25971   case Intrinsic::x86_avx_vtestnzc_pd_256: {
25972     unsigned TestOpc = X86ISD::PTEST;
25973     X86::CondCode X86CC;
25974     switch (IntNo) {
25975     default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
25976     case Intrinsic::x86_avx512_ktestc_b:
25977     case Intrinsic::x86_avx512_ktestc_w:
25978     case Intrinsic::x86_avx512_ktestc_d:
25979     case Intrinsic::x86_avx512_ktestc_q:
25980       // CF = 1
25981       TestOpc = X86ISD::KTEST;
25982       X86CC = X86::COND_B;
25983       break;
25984     case Intrinsic::x86_avx512_ktestz_b:
25985     case Intrinsic::x86_avx512_ktestz_w:
25986     case Intrinsic::x86_avx512_ktestz_d:
25987     case Intrinsic::x86_avx512_ktestz_q:
25988       TestOpc = X86ISD::KTEST;
25989       X86CC = X86::COND_E;
25990       break;
25991     case Intrinsic::x86_avx_vtestz_ps:
25992     case Intrinsic::x86_avx_vtestz_pd:
25993     case Intrinsic::x86_avx_vtestz_ps_256:
25994     case Intrinsic::x86_avx_vtestz_pd_256:
25995       TestOpc = X86ISD::TESTP;
25996       [[fallthrough]];
25997     case Intrinsic::x86_sse41_ptestz:
25998     case Intrinsic::x86_avx_ptestz_256:
25999       // ZF = 1
26000       X86CC = X86::COND_E;
26001       break;
26002     case Intrinsic::x86_avx_vtestc_ps:
26003     case Intrinsic::x86_avx_vtestc_pd:
26004     case Intrinsic::x86_avx_vtestc_ps_256:
26005     case Intrinsic::x86_avx_vtestc_pd_256:
26006       TestOpc = X86ISD::TESTP;
26007       [[fallthrough]];
26008     case Intrinsic::x86_sse41_ptestc:
26009     case Intrinsic::x86_avx_ptestc_256:
26010       // CF = 1
26011       X86CC = X86::COND_B;
26012       break;
26013     case Intrinsic::x86_avx_vtestnzc_ps:
26014     case Intrinsic::x86_avx_vtestnzc_pd:
26015     case Intrinsic::x86_avx_vtestnzc_ps_256:
26016     case Intrinsic::x86_avx_vtestnzc_pd_256:
26017       TestOpc = X86ISD::TESTP;
26018       [[fallthrough]];
26019     case Intrinsic::x86_sse41_ptestnzc:
26020     case Intrinsic::x86_avx_ptestnzc_256:
26021       // ZF and CF = 0
26022       X86CC = X86::COND_A;
26023       break;
26024     }
26025 
26026     SDValue LHS = Op.getOperand(1);
26027     SDValue RHS = Op.getOperand(2);
26028     SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
26029     SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
26030     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26031   }
26032 
26033   case Intrinsic::x86_sse42_pcmpistria128:
26034   case Intrinsic::x86_sse42_pcmpestria128:
26035   case Intrinsic::x86_sse42_pcmpistric128:
26036   case Intrinsic::x86_sse42_pcmpestric128:
26037   case Intrinsic::x86_sse42_pcmpistrio128:
26038   case Intrinsic::x86_sse42_pcmpestrio128:
26039   case Intrinsic::x86_sse42_pcmpistris128:
26040   case Intrinsic::x86_sse42_pcmpestris128:
26041   case Intrinsic::x86_sse42_pcmpistriz128:
26042   case Intrinsic::x86_sse42_pcmpestriz128: {
26043     unsigned Opcode;
26044     X86::CondCode X86CC;
26045     switch (IntNo) {
26046     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
26047     case Intrinsic::x86_sse42_pcmpistria128:
26048       Opcode = X86ISD::PCMPISTR;
26049       X86CC = X86::COND_A;
26050       break;
26051     case Intrinsic::x86_sse42_pcmpestria128:
26052       Opcode = X86ISD::PCMPESTR;
26053       X86CC = X86::COND_A;
26054       break;
26055     case Intrinsic::x86_sse42_pcmpistric128:
26056       Opcode = X86ISD::PCMPISTR;
26057       X86CC = X86::COND_B;
26058       break;
26059     case Intrinsic::x86_sse42_pcmpestric128:
26060       Opcode = X86ISD::PCMPESTR;
26061       X86CC = X86::COND_B;
26062       break;
26063     case Intrinsic::x86_sse42_pcmpistrio128:
26064       Opcode = X86ISD::PCMPISTR;
26065       X86CC = X86::COND_O;
26066       break;
26067     case Intrinsic::x86_sse42_pcmpestrio128:
26068       Opcode = X86ISD::PCMPESTR;
26069       X86CC = X86::COND_O;
26070       break;
26071     case Intrinsic::x86_sse42_pcmpistris128:
26072       Opcode = X86ISD::PCMPISTR;
26073       X86CC = X86::COND_S;
26074       break;
26075     case Intrinsic::x86_sse42_pcmpestris128:
26076       Opcode = X86ISD::PCMPESTR;
26077       X86CC = X86::COND_S;
26078       break;
26079     case Intrinsic::x86_sse42_pcmpistriz128:
26080       Opcode = X86ISD::PCMPISTR;
26081       X86CC = X86::COND_E;
26082       break;
26083     case Intrinsic::x86_sse42_pcmpestriz128:
26084       Opcode = X86ISD::PCMPESTR;
26085       X86CC = X86::COND_E;
26086       break;
26087     }
26088     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26089     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26090     SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
26091     SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
26092     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26093   }
26094 
26095   case Intrinsic::x86_sse42_pcmpistri128:
26096   case Intrinsic::x86_sse42_pcmpestri128: {
26097     unsigned Opcode;
26098     if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
26099       Opcode = X86ISD::PCMPISTR;
26100     else
26101       Opcode = X86ISD::PCMPESTR;
26102 
26103     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26104     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26105     return DAG.getNode(Opcode, dl, VTs, NewOps);
26106   }
26107 
26108   case Intrinsic::x86_sse42_pcmpistrm128:
26109   case Intrinsic::x86_sse42_pcmpestrm128: {
26110     unsigned Opcode;
26111     if (IntNo == Intrinsic::x86_sse42_pcmpistrm128)
26112       Opcode = X86ISD::PCMPISTR;
26113     else
26114       Opcode = X86ISD::PCMPESTR;
26115 
26116     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26117     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26118     return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
26119   }
26120 
26121   case Intrinsic::eh_sjlj_lsda: {
26122     MachineFunction &MF = DAG.getMachineFunction();
26123     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26124     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
26125     auto &Context = MF.getMMI().getContext();
26126     MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
26127                                             Twine(MF.getFunctionNumber()));
26128     return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT,
26129                        DAG.getMCSymbol(S, PtrVT));
26130   }
26131 
26132   case Intrinsic::x86_seh_lsda: {
26133     // Compute the symbol for the LSDA. We know it'll get emitted later.
26134     MachineFunction &MF = DAG.getMachineFunction();
26135     SDValue Op1 = Op.getOperand(1);
26136     auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
26137     MCSymbol *LSDASym = MF.getMMI().getContext().getOrCreateLSDASymbol(
26138         GlobalValue::dropLLVMManglingEscape(Fn->getName()));
26139 
26140     // Generate a simple absolute symbol reference. This intrinsic is only
26141     // supported on 32-bit Windows, which isn't PIC.
26142     SDValue Result = DAG.getMCSymbol(LSDASym, VT);
26143     return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
26144   }
26145 
26146   case Intrinsic::eh_recoverfp: {
26147     SDValue FnOp = Op.getOperand(1);
26148     SDValue IncomingFPOp = Op.getOperand(2);
26149     GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
26150     auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
26151     if (!Fn)
26152       report_fatal_error(
26153           "llvm.eh.recoverfp must take a function as the first argument");
26154     return recoverFramePointer(DAG, Fn, IncomingFPOp);
26155   }
26156 
26157   case Intrinsic::localaddress: {
26158     // Returns one of the stack, base, or frame pointer registers, depending on
26159     // which is used to reference local variables.
26160     MachineFunction &MF = DAG.getMachineFunction();
26161     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
26162     unsigned Reg;
26163     if (RegInfo->hasBasePointer(MF))
26164       Reg = RegInfo->getBaseRegister();
26165     else { // Handles the SP or FP case.
26166       bool CantUseFP = RegInfo->hasStackRealignment(MF);
26167       if (CantUseFP)
26168         Reg = RegInfo->getPtrSizedStackRegister(MF);
26169       else
26170         Reg = RegInfo->getPtrSizedFrameRegister(MF);
26171     }
26172     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
26173   }
26174   case Intrinsic::x86_avx512_vp2intersect_q_512:
26175   case Intrinsic::x86_avx512_vp2intersect_q_256:
26176   case Intrinsic::x86_avx512_vp2intersect_q_128:
26177   case Intrinsic::x86_avx512_vp2intersect_d_512:
26178   case Intrinsic::x86_avx512_vp2intersect_d_256:
26179   case Intrinsic::x86_avx512_vp2intersect_d_128: {
26180     MVT MaskVT = Op.getSimpleValueType();
26181 
26182     SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
26183     SDLoc DL(Op);
26184 
26185     SDValue Operation =
26186         DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
26187                     Op->getOperand(1), Op->getOperand(2));
26188 
26189     SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26190                                                  MaskVT, Operation);
26191     SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
26192                                                  MaskVT, Operation);
26193     return DAG.getMergeValues({Result0, Result1}, DL);
26194   }
26195   case Intrinsic::x86_mmx_pslli_w:
26196   case Intrinsic::x86_mmx_pslli_d:
26197   case Intrinsic::x86_mmx_pslli_q:
26198   case Intrinsic::x86_mmx_psrli_w:
26199   case Intrinsic::x86_mmx_psrli_d:
26200   case Intrinsic::x86_mmx_psrli_q:
26201   case Intrinsic::x86_mmx_psrai_w:
26202   case Intrinsic::x86_mmx_psrai_d: {
26203     SDLoc DL(Op);
26204     SDValue ShAmt = Op.getOperand(2);
26205     // If the argument is a constant, convert it to a target constant.
26206     if (auto *C = dyn_cast<ConstantSDNode>(ShAmt)) {
26207       // Clamp out of bounds shift amounts since they will otherwise be masked
26208       // to 8-bits which may make it no longer out of bounds.
26209       unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255);
26210       if (ShiftAmount == 0)
26211         return Op.getOperand(1);
26212 
26213       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26214                          Op.getOperand(0), Op.getOperand(1),
26215                          DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
26216     }
26217 
26218     unsigned NewIntrinsic;
26219     switch (IntNo) {
26220     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
26221     case Intrinsic::x86_mmx_pslli_w:
26222       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
26223       break;
26224     case Intrinsic::x86_mmx_pslli_d:
26225       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
26226       break;
26227     case Intrinsic::x86_mmx_pslli_q:
26228       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
26229       break;
26230     case Intrinsic::x86_mmx_psrli_w:
26231       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
26232       break;
26233     case Intrinsic::x86_mmx_psrli_d:
26234       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
26235       break;
26236     case Intrinsic::x86_mmx_psrli_q:
26237       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
26238       break;
26239     case Intrinsic::x86_mmx_psrai_w:
26240       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
26241       break;
26242     case Intrinsic::x86_mmx_psrai_d:
26243       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
26244       break;
26245     }
26246 
26247     // The vector shift intrinsics with scalars uses 32b shift amounts but
26248     // the sse2/mmx shift instructions reads 64 bits. Copy the 32 bits to an
26249     // MMX register.
26250     ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
26251     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26252                        DAG.getTargetConstant(NewIntrinsic, DL,
26253                                              getPointerTy(DAG.getDataLayout())),
26254                        Op.getOperand(1), ShAmt);
26255   }
26256   case Intrinsic::thread_pointer: {
26257     if (Subtarget.isTargetELF()) {
26258       SDLoc dl(Op);
26259       EVT PtrVT = getPointerTy(DAG.getDataLayout());
26260       // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
26261       Value *Ptr = Constant::getNullValue(PointerType::get(
26262           *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
26263       return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
26264                          DAG.getIntPtrConstant(0, dl), MachinePointerInfo(Ptr));
26265     }
26266     report_fatal_error(
26267         "Target OS doesn't support __builtin_thread_pointer() yet.");
26268   }
26269   }
26270 }
26271 
26272 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26273                                  SDValue Src, SDValue Mask, SDValue Base,
26274                                  SDValue Index, SDValue ScaleOp, SDValue Chain,
26275                                  const X86Subtarget &Subtarget) {
26276   SDLoc dl(Op);
26277   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26278   // Scale must be constant.
26279   if (!C)
26280     return SDValue();
26281   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26282   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26283                                         TLI.getPointerTy(DAG.getDataLayout()));
26284   EVT MaskVT = Mask.getValueType().changeVectorElementTypeToInteger();
26285   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26286   // If source is undef or we know it won't be used, use a zero vector
26287   // to break register dependency.
26288   // TODO: use undef instead and let BreakFalseDeps deal with it?
26289   if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26290     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26291 
26292   // Cast mask to an integer type.
26293   Mask = DAG.getBitcast(MaskVT, Mask);
26294 
26295   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26296 
26297   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26298   SDValue Res =
26299       DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26300                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26301   return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26302 }
26303 
26304 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
26305                              SDValue Src, SDValue Mask, SDValue Base,
26306                              SDValue Index, SDValue ScaleOp, SDValue Chain,
26307                              const X86Subtarget &Subtarget) {
26308   MVT VT = Op.getSimpleValueType();
26309   SDLoc dl(Op);
26310   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26311   // Scale must be constant.
26312   if (!C)
26313     return SDValue();
26314   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26315   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26316                                         TLI.getPointerTy(DAG.getDataLayout()));
26317   unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26318                               VT.getVectorNumElements());
26319   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26320 
26321   // We support two versions of the gather intrinsics. One with scalar mask and
26322   // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26323   if (Mask.getValueType() != MaskVT)
26324     Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26325 
26326   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26327   // If source is undef or we know it won't be used, use a zero vector
26328   // to break register dependency.
26329   // TODO: use undef instead and let BreakFalseDeps deal with it?
26330   if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26331     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26332 
26333   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26334 
26335   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26336   SDValue Res =
26337       DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26338                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26339   return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26340 }
26341 
26342 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26343                                SDValue Src, SDValue Mask, SDValue Base,
26344                                SDValue Index, SDValue ScaleOp, SDValue Chain,
26345                                const X86Subtarget &Subtarget) {
26346   SDLoc dl(Op);
26347   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26348   // Scale must be constant.
26349   if (!C)
26350     return SDValue();
26351   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26352   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26353                                         TLI.getPointerTy(DAG.getDataLayout()));
26354   unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26355                               Src.getSimpleValueType().getVectorNumElements());
26356   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26357 
26358   // We support two versions of the scatter intrinsics. One with scalar mask and
26359   // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26360   if (Mask.getValueType() != MaskVT)
26361     Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26362 
26363   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26364 
26365   SDVTList VTs = DAG.getVTList(MVT::Other);
26366   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
26367   SDValue Res =
26368       DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
26369                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26370   return Res;
26371 }
26372 
26373 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26374                                SDValue Mask, SDValue Base, SDValue Index,
26375                                SDValue ScaleOp, SDValue Chain,
26376                                const X86Subtarget &Subtarget) {
26377   SDLoc dl(Op);
26378   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26379   // Scale must be constant.
26380   if (!C)
26381     return SDValue();
26382   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26383   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26384                                         TLI.getPointerTy(DAG.getDataLayout()));
26385   SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
26386   SDValue Segment = DAG.getRegister(0, MVT::i32);
26387   MVT MaskVT =
26388     MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
26389   SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26390   SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
26391   SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
26392   return SDValue(Res, 0);
26393 }
26394 
26395 /// Handles the lowering of builtin intrinsics with chain that return their
26396 /// value into registers EDX:EAX.
26397 /// If operand ScrReg is a valid register identifier, then operand 2 of N is
26398 /// copied to SrcReg. The assumption is that SrcReg is an implicit input to
26399 /// TargetOpcode.
26400 /// Returns a Glue value which can be used to add extra copy-from-reg if the
26401 /// expanded intrinsics implicitly defines extra registers (i.e. not just
26402 /// EDX:EAX).
26403 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
26404                                         SelectionDAG &DAG,
26405                                         unsigned TargetOpcode,
26406                                         unsigned SrcReg,
26407                                         const X86Subtarget &Subtarget,
26408                                         SmallVectorImpl<SDValue> &Results) {
26409   SDValue Chain = N->getOperand(0);
26410   SDValue Glue;
26411 
26412   if (SrcReg) {
26413     assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
26414     Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
26415     Glue = Chain.getValue(1);
26416   }
26417 
26418   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
26419   SDValue N1Ops[] = {Chain, Glue};
26420   SDNode *N1 = DAG.getMachineNode(
26421       TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
26422   Chain = SDValue(N1, 0);
26423 
26424   // Reads the content of XCR and returns it in registers EDX:EAX.
26425   SDValue LO, HI;
26426   if (Subtarget.is64Bit()) {
26427     LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
26428     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
26429                             LO.getValue(2));
26430   } else {
26431     LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
26432     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
26433                             LO.getValue(2));
26434   }
26435   Chain = HI.getValue(1);
26436   Glue = HI.getValue(2);
26437 
26438   if (Subtarget.is64Bit()) {
26439     // Merge the two 32-bit values into a 64-bit one.
26440     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
26441                               DAG.getConstant(32, DL, MVT::i8));
26442     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
26443     Results.push_back(Chain);
26444     return Glue;
26445   }
26446 
26447   // Use a buildpair to merge the two 32-bit values into a 64-bit one.
26448   SDValue Ops[] = { LO, HI };
26449   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
26450   Results.push_back(Pair);
26451   Results.push_back(Chain);
26452   return Glue;
26453 }
26454 
26455 /// Handles the lowering of builtin intrinsics that read the time stamp counter
26456 /// (x86_rdtsc and x86_rdtscp). This function is also used to custom lower
26457 /// READCYCLECOUNTER nodes.
26458 static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
26459                                     SelectionDAG &DAG,
26460                                     const X86Subtarget &Subtarget,
26461                                     SmallVectorImpl<SDValue> &Results) {
26462   // The processor's time-stamp counter (a 64-bit MSR) is stored into the
26463   // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
26464   // and the EAX register is loaded with the low-order 32 bits.
26465   SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
26466                                              /* NoRegister */0, Subtarget,
26467                                              Results);
26468   if (Opcode != X86::RDTSCP)
26469     return;
26470 
26471   SDValue Chain = Results[1];
26472   // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
26473   // the ECX register. Add 'ecx' explicitly to the chain.
26474   SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
26475   Results[1] = ecx;
26476   Results.push_back(ecx.getValue(1));
26477 }
26478 
26479 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
26480                                      SelectionDAG &DAG) {
26481   SmallVector<SDValue, 3> Results;
26482   SDLoc DL(Op);
26483   getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
26484                           Results);
26485   return DAG.getMergeValues(Results, DL);
26486 }
26487 
26488 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
26489   MachineFunction &MF = DAG.getMachineFunction();
26490   SDValue Chain = Op.getOperand(0);
26491   SDValue RegNode = Op.getOperand(2);
26492   WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26493   if (!EHInfo)
26494     report_fatal_error("EH registrations only live in functions using WinEH");
26495 
26496   // Cast the operand to an alloca, and remember the frame index.
26497   auto *FINode = dyn_cast<FrameIndexSDNode>(RegNode);
26498   if (!FINode)
26499     report_fatal_error("llvm.x86.seh.ehregnode expects a static alloca");
26500   EHInfo->EHRegNodeFrameIndex = FINode->getIndex();
26501 
26502   // Return the chain operand without making any DAG nodes.
26503   return Chain;
26504 }
26505 
26506 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
26507   MachineFunction &MF = DAG.getMachineFunction();
26508   SDValue Chain = Op.getOperand(0);
26509   SDValue EHGuard = Op.getOperand(2);
26510   WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26511   if (!EHInfo)
26512     report_fatal_error("EHGuard only live in functions using WinEH");
26513 
26514   // Cast the operand to an alloca, and remember the frame index.
26515   auto *FINode = dyn_cast<FrameIndexSDNode>(EHGuard);
26516   if (!FINode)
26517     report_fatal_error("llvm.x86.seh.ehguard expects a static alloca");
26518   EHInfo->EHGuardFrameIndex = FINode->getIndex();
26519 
26520   // Return the chain operand without making any DAG nodes.
26521   return Chain;
26522 }
26523 
26524 /// Emit Truncating Store with signed or unsigned saturation.
26525 static SDValue
26526 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
26527                 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
26528                 SelectionDAG &DAG) {
26529   SDVTList VTs = DAG.getVTList(MVT::Other);
26530   SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
26531   SDValue Ops[] = { Chain, Val, Ptr, Undef };
26532   unsigned Opc = SignedSat ? X86ISD::VTRUNCSTORES : X86ISD::VTRUNCSTOREUS;
26533   return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26534 }
26535 
26536 /// Emit Masked Truncating Store with signed or unsigned saturation.
26537 static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
26538                                      const SDLoc &DL,
26539                       SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
26540                       MachineMemOperand *MMO, SelectionDAG &DAG) {
26541   SDVTList VTs = DAG.getVTList(MVT::Other);
26542   SDValue Ops[] = { Chain, Val, Ptr, Mask };
26543   unsigned Opc = SignedSat ? X86ISD::VMTRUNCSTORES : X86ISD::VMTRUNCSTOREUS;
26544   return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26545 }
26546 
26547 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
26548                                       SelectionDAG &DAG) {
26549   unsigned IntNo = Op.getConstantOperandVal(1);
26550   const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
26551   if (!IntrData) {
26552     switch (IntNo) {
26553 
26554     case Intrinsic::swift_async_context_addr: {
26555       SDLoc dl(Op);
26556       auto &MF = DAG.getMachineFunction();
26557       auto X86FI = MF.getInfo<X86MachineFunctionInfo>();
26558       if (Subtarget.is64Bit()) {
26559         MF.getFrameInfo().setFrameAddressIsTaken(true);
26560         X86FI->setHasSwiftAsyncContext(true);
26561         SDValue Chain = Op->getOperand(0);
26562         SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
26563         SDValue Result =
26564             SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
26565                                        DAG.getTargetConstant(8, dl, MVT::i32)),
26566                     0);
26567         // Return { result, chain }.
26568         return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26569                            CopyRBP.getValue(1));
26570       } else {
26571         // 32-bit so no special extended frame, create or reuse an existing
26572         // stack slot.
26573         if (!X86FI->getSwiftAsyncContextFrameIdx())
26574           X86FI->setSwiftAsyncContextFrameIdx(
26575               MF.getFrameInfo().CreateStackObject(4, Align(4), false));
26576         SDValue Result =
26577             DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(), MVT::i32);
26578         // Return { result, chain }.
26579         return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26580                            Op->getOperand(0));
26581       }
26582     }
26583 
26584     case llvm::Intrinsic::x86_seh_ehregnode:
26585       return MarkEHRegistrationNode(Op, DAG);
26586     case llvm::Intrinsic::x86_seh_ehguard:
26587       return MarkEHGuard(Op, DAG);
26588     case llvm::Intrinsic::x86_rdpkru: {
26589       SDLoc dl(Op);
26590       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26591       // Create a RDPKRU node and pass 0 to the ECX parameter.
26592       return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0),
26593                          DAG.getConstant(0, dl, MVT::i32));
26594     }
26595     case llvm::Intrinsic::x86_wrpkru: {
26596       SDLoc dl(Op);
26597       // Create a WRPKRU node, pass the input to the EAX parameter,  and pass 0
26598       // to the EDX and ECX parameters.
26599       return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
26600                          Op.getOperand(0), Op.getOperand(2),
26601                          DAG.getConstant(0, dl, MVT::i32),
26602                          DAG.getConstant(0, dl, MVT::i32));
26603     }
26604     case llvm::Intrinsic::asan_check_memaccess: {
26605       // Mark this as adjustsStack because it will be lowered to a call.
26606       DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true);
26607       // Don't do anything here, we will expand these intrinsics out later.
26608       return Op;
26609     }
26610     case llvm::Intrinsic::x86_flags_read_u32:
26611     case llvm::Intrinsic::x86_flags_read_u64:
26612     case llvm::Intrinsic::x86_flags_write_u32:
26613     case llvm::Intrinsic::x86_flags_write_u64: {
26614       // We need a frame pointer because this will get lowered to a PUSH/POP
26615       // sequence.
26616       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
26617       MFI.setHasCopyImplyingStackAdjustment(true);
26618       // Don't do anything here, we will expand these intrinsics out later
26619       // during FinalizeISel in EmitInstrWithCustomInserter.
26620       return Op;
26621     }
26622     case Intrinsic::x86_lwpins32:
26623     case Intrinsic::x86_lwpins64:
26624     case Intrinsic::x86_umwait:
26625     case Intrinsic::x86_tpause: {
26626       SDLoc dl(Op);
26627       SDValue Chain = Op->getOperand(0);
26628       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26629       unsigned Opcode;
26630 
26631       switch (IntNo) {
26632       default: llvm_unreachable("Impossible intrinsic");
26633       case Intrinsic::x86_umwait:
26634         Opcode = X86ISD::UMWAIT;
26635         break;
26636       case Intrinsic::x86_tpause:
26637         Opcode = X86ISD::TPAUSE;
26638         break;
26639       case Intrinsic::x86_lwpins32:
26640       case Intrinsic::x86_lwpins64:
26641         Opcode = X86ISD::LWPINS;
26642         break;
26643       }
26644 
26645       SDValue Operation =
26646           DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
26647                       Op->getOperand(3), Op->getOperand(4));
26648       SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
26649       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26650                          Operation.getValue(1));
26651     }
26652     case Intrinsic::x86_enqcmd:
26653     case Intrinsic::x86_enqcmds: {
26654       SDLoc dl(Op);
26655       SDValue Chain = Op.getOperand(0);
26656       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26657       unsigned Opcode;
26658       switch (IntNo) {
26659       default: llvm_unreachable("Impossible intrinsic!");
26660       case Intrinsic::x86_enqcmd:
26661         Opcode = X86ISD::ENQCMD;
26662         break;
26663       case Intrinsic::x86_enqcmds:
26664         Opcode = X86ISD::ENQCMDS;
26665         break;
26666       }
26667       SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
26668                                       Op.getOperand(3));
26669       SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
26670       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26671                          Operation.getValue(1));
26672     }
26673     case Intrinsic::x86_aesenc128kl:
26674     case Intrinsic::x86_aesdec128kl:
26675     case Intrinsic::x86_aesenc256kl:
26676     case Intrinsic::x86_aesdec256kl: {
26677       SDLoc DL(Op);
26678       SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);
26679       SDValue Chain = Op.getOperand(0);
26680       unsigned Opcode;
26681 
26682       switch (IntNo) {
26683       default: llvm_unreachable("Impossible intrinsic");
26684       case Intrinsic::x86_aesenc128kl:
26685         Opcode = X86ISD::AESENC128KL;
26686         break;
26687       case Intrinsic::x86_aesdec128kl:
26688         Opcode = X86ISD::AESDEC128KL;
26689         break;
26690       case Intrinsic::x86_aesenc256kl:
26691         Opcode = X86ISD::AESENC256KL;
26692         break;
26693       case Intrinsic::x86_aesdec256kl:
26694         Opcode = X86ISD::AESDEC256KL;
26695         break;
26696       }
26697 
26698       MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26699       MachineMemOperand *MMO = MemIntr->getMemOperand();
26700       EVT MemVT = MemIntr->getMemoryVT();
26701       SDValue Operation = DAG.getMemIntrinsicNode(
26702           Opcode, DL, VTs, {Chain, Op.getOperand(2), Op.getOperand(3)}, MemVT,
26703           MMO);
26704       SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
26705 
26706       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
26707                          {ZF, Operation.getValue(0), Operation.getValue(2)});
26708     }
26709     case Intrinsic::x86_aesencwide128kl:
26710     case Intrinsic::x86_aesdecwide128kl:
26711     case Intrinsic::x86_aesencwide256kl:
26712     case Intrinsic::x86_aesdecwide256kl: {
26713       SDLoc DL(Op);
26714       SDVTList VTs = DAG.getVTList(
26715           {MVT::i32, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64,
26716            MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::Other});
26717       SDValue Chain = Op.getOperand(0);
26718       unsigned Opcode;
26719 
26720       switch (IntNo) {
26721       default: llvm_unreachable("Impossible intrinsic");
26722       case Intrinsic::x86_aesencwide128kl:
26723         Opcode = X86ISD::AESENCWIDE128KL;
26724         break;
26725       case Intrinsic::x86_aesdecwide128kl:
26726         Opcode = X86ISD::AESDECWIDE128KL;
26727         break;
26728       case Intrinsic::x86_aesencwide256kl:
26729         Opcode = X86ISD::AESENCWIDE256KL;
26730         break;
26731       case Intrinsic::x86_aesdecwide256kl:
26732         Opcode = X86ISD::AESDECWIDE256KL;
26733         break;
26734       }
26735 
26736       MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26737       MachineMemOperand *MMO = MemIntr->getMemOperand();
26738       EVT MemVT = MemIntr->getMemoryVT();
26739       SDValue Operation = DAG.getMemIntrinsicNode(
26740           Opcode, DL, VTs,
26741           {Chain, Op.getOperand(2), Op.getOperand(3), Op.getOperand(4),
26742            Op.getOperand(5), Op.getOperand(6), Op.getOperand(7),
26743            Op.getOperand(8), Op.getOperand(9), Op.getOperand(10)},
26744           MemVT, MMO);
26745       SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
26746 
26747       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
26748                          {ZF, Operation.getValue(1), Operation.getValue(2),
26749                           Operation.getValue(3), Operation.getValue(4),
26750                           Operation.getValue(5), Operation.getValue(6),
26751                           Operation.getValue(7), Operation.getValue(8),
26752                           Operation.getValue(9)});
26753     }
26754     case Intrinsic::x86_testui: {
26755       SDLoc dl(Op);
26756       SDValue Chain = Op.getOperand(0);
26757       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26758       SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain);
26759       SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
26760       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26761                          Operation.getValue(1));
26762     }
26763     case Intrinsic::x86_atomic_bts_rm:
26764     case Intrinsic::x86_atomic_btc_rm:
26765     case Intrinsic::x86_atomic_btr_rm: {
26766       SDLoc DL(Op);
26767       MVT VT = Op.getSimpleValueType();
26768       SDValue Chain = Op.getOperand(0);
26769       SDValue Op1 = Op.getOperand(2);
26770       SDValue Op2 = Op.getOperand(3);
26771       unsigned Opc = IntNo == Intrinsic::x86_atomic_bts_rm   ? X86ISD::LBTS_RM
26772                      : IntNo == Intrinsic::x86_atomic_btc_rm ? X86ISD::LBTC_RM
26773                                                              : X86ISD::LBTR_RM;
26774       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26775       SDValue Res =
26776           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26777                                   {Chain, Op1, Op2}, VT, MMO);
26778       Chain = Res.getValue(1);
26779       Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
26780       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
26781     }
26782     case Intrinsic::x86_atomic_bts:
26783     case Intrinsic::x86_atomic_btc:
26784     case Intrinsic::x86_atomic_btr: {
26785       SDLoc DL(Op);
26786       MVT VT = Op.getSimpleValueType();
26787       SDValue Chain = Op.getOperand(0);
26788       SDValue Op1 = Op.getOperand(2);
26789       SDValue Op2 = Op.getOperand(3);
26790       unsigned Opc = IntNo == Intrinsic::x86_atomic_bts   ? X86ISD::LBTS
26791                      : IntNo == Intrinsic::x86_atomic_btc ? X86ISD::LBTC
26792                                                           : X86ISD::LBTR;
26793       SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);
26794       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26795       SDValue Res =
26796           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26797                                   {Chain, Op1, Op2, Size}, VT, MMO);
26798       Chain = Res.getValue(1);
26799       Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
26800       unsigned Imm = Op2->getAsZExtVal();
26801       if (Imm)
26802         Res = DAG.getNode(ISD::SHL, DL, VT, Res,
26803                           DAG.getShiftAmountConstant(Imm, VT, DL));
26804       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
26805     }
26806     case Intrinsic::x86_cmpccxadd32:
26807     case Intrinsic::x86_cmpccxadd64: {
26808       SDLoc DL(Op);
26809       SDValue Chain = Op.getOperand(0);
26810       SDValue Addr = Op.getOperand(2);
26811       SDValue Src1 = Op.getOperand(3);
26812       SDValue Src2 = Op.getOperand(4);
26813       SDValue CC = Op.getOperand(5);
26814       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26815       SDValue Operation = DAG.getMemIntrinsicNode(
26816           X86ISD::CMPCCXADD, DL, Op->getVTList(), {Chain, Addr, Src1, Src2, CC},
26817           MVT::i32, MMO);
26818       return Operation;
26819     }
26820     case Intrinsic::x86_aadd32:
26821     case Intrinsic::x86_aadd64:
26822     case Intrinsic::x86_aand32:
26823     case Intrinsic::x86_aand64:
26824     case Intrinsic::x86_aor32:
26825     case Intrinsic::x86_aor64:
26826     case Intrinsic::x86_axor32:
26827     case Intrinsic::x86_axor64: {
26828       SDLoc DL(Op);
26829       SDValue Chain = Op.getOperand(0);
26830       SDValue Op1 = Op.getOperand(2);
26831       SDValue Op2 = Op.getOperand(3);
26832       MVT VT = Op2.getSimpleValueType();
26833       unsigned Opc = 0;
26834       switch (IntNo) {
26835       default:
26836         llvm_unreachable("Unknown Intrinsic");
26837       case Intrinsic::x86_aadd32:
26838       case Intrinsic::x86_aadd64:
26839         Opc = X86ISD::AADD;
26840         break;
26841       case Intrinsic::x86_aand32:
26842       case Intrinsic::x86_aand64:
26843         Opc = X86ISD::AAND;
26844         break;
26845       case Intrinsic::x86_aor32:
26846       case Intrinsic::x86_aor64:
26847         Opc = X86ISD::AOR;
26848         break;
26849       case Intrinsic::x86_axor32:
26850       case Intrinsic::x86_axor64:
26851         Opc = X86ISD::AXOR;
26852         break;
26853       }
26854       MachineMemOperand *MMO = cast<MemSDNode>(Op)->getMemOperand();
26855       return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(),
26856                                      {Chain, Op1, Op2}, VT, MMO);
26857     }
26858     case Intrinsic::x86_atomic_add_cc:
26859     case Intrinsic::x86_atomic_sub_cc:
26860     case Intrinsic::x86_atomic_or_cc:
26861     case Intrinsic::x86_atomic_and_cc:
26862     case Intrinsic::x86_atomic_xor_cc: {
26863       SDLoc DL(Op);
26864       SDValue Chain = Op.getOperand(0);
26865       SDValue Op1 = Op.getOperand(2);
26866       SDValue Op2 = Op.getOperand(3);
26867       X86::CondCode CC = (X86::CondCode)Op.getConstantOperandVal(4);
26868       MVT VT = Op2.getSimpleValueType();
26869       unsigned Opc = 0;
26870       switch (IntNo) {
26871       default:
26872         llvm_unreachable("Unknown Intrinsic");
26873       case Intrinsic::x86_atomic_add_cc:
26874         Opc = X86ISD::LADD;
26875         break;
26876       case Intrinsic::x86_atomic_sub_cc:
26877         Opc = X86ISD::LSUB;
26878         break;
26879       case Intrinsic::x86_atomic_or_cc:
26880         Opc = X86ISD::LOR;
26881         break;
26882       case Intrinsic::x86_atomic_and_cc:
26883         Opc = X86ISD::LAND;
26884         break;
26885       case Intrinsic::x86_atomic_xor_cc:
26886         Opc = X86ISD::LXOR;
26887         break;
26888       }
26889       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26890       SDValue LockArith =
26891           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26892                                   {Chain, Op1, Op2}, VT, MMO);
26893       Chain = LockArith.getValue(1);
26894       return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
26895     }
26896     }
26897     return SDValue();
26898   }
26899 
26900   SDLoc dl(Op);
26901   switch(IntrData->Type) {
26902   default: llvm_unreachable("Unknown Intrinsic Type");
26903   case RDSEED:
26904   case RDRAND: {
26905     // Emit the node with the right value type.
26906     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);
26907     SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
26908 
26909     // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
26910     // Otherwise return the value from Rand, which is always 0, casted to i32.
26911     SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
26912                      DAG.getConstant(1, dl, Op->getValueType(1)),
26913                      DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
26914                      SDValue(Result.getNode(), 1)};
26915     SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
26916 
26917     // Return { result, isValid, chain }.
26918     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
26919                        SDValue(Result.getNode(), 2));
26920   }
26921   case GATHER_AVX2: {
26922     SDValue Chain = Op.getOperand(0);
26923     SDValue Src   = Op.getOperand(2);
26924     SDValue Base  = Op.getOperand(3);
26925     SDValue Index = Op.getOperand(4);
26926     SDValue Mask  = Op.getOperand(5);
26927     SDValue Scale = Op.getOperand(6);
26928     return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
26929                              Scale, Chain, Subtarget);
26930   }
26931   case GATHER: {
26932   //gather(v1, mask, index, base, scale);
26933     SDValue Chain = Op.getOperand(0);
26934     SDValue Src   = Op.getOperand(2);
26935     SDValue Base  = Op.getOperand(3);
26936     SDValue Index = Op.getOperand(4);
26937     SDValue Mask  = Op.getOperand(5);
26938     SDValue Scale = Op.getOperand(6);
26939     return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
26940                          Chain, Subtarget);
26941   }
26942   case SCATTER: {
26943   //scatter(base, mask, index, v1, scale);
26944     SDValue Chain = Op.getOperand(0);
26945     SDValue Base  = Op.getOperand(2);
26946     SDValue Mask  = Op.getOperand(3);
26947     SDValue Index = Op.getOperand(4);
26948     SDValue Src   = Op.getOperand(5);
26949     SDValue Scale = Op.getOperand(6);
26950     return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
26951                           Scale, Chain, Subtarget);
26952   }
26953   case PREFETCH: {
26954     const APInt &HintVal = Op.getConstantOperandAPInt(6);
26955     assert((HintVal == 2 || HintVal == 3) &&
26956            "Wrong prefetch hint in intrinsic: should be 2 or 3");
26957     unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
26958     SDValue Chain = Op.getOperand(0);
26959     SDValue Mask  = Op.getOperand(2);
26960     SDValue Index = Op.getOperand(3);
26961     SDValue Base  = Op.getOperand(4);
26962     SDValue Scale = Op.getOperand(5);
26963     return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
26964                            Subtarget);
26965   }
26966   // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
26967   case RDTSC: {
26968     SmallVector<SDValue, 2> Results;
26969     getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
26970                             Results);
26971     return DAG.getMergeValues(Results, dl);
26972   }
26973   // Read Performance Monitoring Counters.
26974   case RDPMC:
26975   // Read Processor Register.
26976   case RDPRU:
26977   // GetExtended Control Register.
26978   case XGETBV: {
26979     SmallVector<SDValue, 2> Results;
26980 
26981     // RDPMC uses ECX to select the index of the performance counter to read.
26982     // RDPRU uses ECX to select the processor register to read.
26983     // XGETBV uses ECX to select the index of the XCR register to return.
26984     // The result is stored into registers EDX:EAX.
26985     expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
26986                                 Subtarget, Results);
26987     return DAG.getMergeValues(Results, dl);
26988   }
26989   // XTEST intrinsics.
26990   case XTEST: {
26991     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
26992     SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
26993 
26994     SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
26995     SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
26996     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
26997                        Ret, SDValue(InTrans.getNode(), 1));
26998   }
26999   case TRUNCATE_TO_MEM_VI8:
27000   case TRUNCATE_TO_MEM_VI16:
27001   case TRUNCATE_TO_MEM_VI32: {
27002     SDValue Mask = Op.getOperand(4);
27003     SDValue DataToTruncate = Op.getOperand(3);
27004     SDValue Addr = Op.getOperand(2);
27005     SDValue Chain = Op.getOperand(0);
27006 
27007     MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
27008     assert(MemIntr && "Expected MemIntrinsicSDNode!");
27009 
27010     EVT MemVT  = MemIntr->getMemoryVT();
27011 
27012     uint16_t TruncationOp = IntrData->Opc0;
27013     switch (TruncationOp) {
27014     case X86ISD::VTRUNC: {
27015       if (isAllOnesConstant(Mask)) // return just a truncate store
27016         return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT,
27017                                  MemIntr->getMemOperand());
27018 
27019       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27020       SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27021       SDValue Offset = DAG.getUNDEF(VMask.getValueType());
27022 
27023       return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
27024                                 MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED,
27025                                 true /* truncating */);
27026     }
27027     case X86ISD::VTRUNCUS:
27028     case X86ISD::VTRUNCS: {
27029       bool IsSigned = (TruncationOp == X86ISD::VTRUNCS);
27030       if (isAllOnesConstant(Mask))
27031         return EmitTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr, MemVT,
27032                                MemIntr->getMemOperand(), DAG);
27033 
27034       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27035       SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27036 
27037       return EmitMaskedTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr,
27038                                    VMask, MemVT, MemIntr->getMemOperand(), DAG);
27039     }
27040     default:
27041       llvm_unreachable("Unsupported truncstore intrinsic");
27042     }
27043   }
27044   }
27045 }
27046 
27047 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
27048                                            SelectionDAG &DAG) const {
27049   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
27050   MFI.setReturnAddressIsTaken(true);
27051 
27052   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
27053     return SDValue();
27054 
27055   unsigned Depth = Op.getConstantOperandVal(0);
27056   SDLoc dl(Op);
27057   EVT PtrVT = getPointerTy(DAG.getDataLayout());
27058 
27059   if (Depth > 0) {
27060     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
27061     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27062     SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
27063     return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
27064                        DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
27065                        MachinePointerInfo());
27066   }
27067 
27068   // Just load the return address.
27069   SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
27070   return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
27071                      MachinePointerInfo());
27072 }
27073 
27074 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
27075                                                  SelectionDAG &DAG) const {
27076   DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true);
27077   return getReturnAddressFrameIndex(DAG);
27078 }
27079 
27080 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
27081   MachineFunction &MF = DAG.getMachineFunction();
27082   MachineFrameInfo &MFI = MF.getFrameInfo();
27083   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
27084   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27085   EVT VT = Op.getValueType();
27086 
27087   MFI.setFrameAddressIsTaken(true);
27088 
27089   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
27090     // Depth > 0 makes no sense on targets which use Windows unwind codes.  It
27091     // is not possible to crawl up the stack without looking at the unwind codes
27092     // simultaneously.
27093     int FrameAddrIndex = FuncInfo->getFAIndex();
27094     if (!FrameAddrIndex) {
27095       // Set up a frame object for the return address.
27096       unsigned SlotSize = RegInfo->getSlotSize();
27097       FrameAddrIndex = MF.getFrameInfo().CreateFixedObject(
27098           SlotSize, /*SPOffset=*/0, /*IsImmutable=*/false);
27099       FuncInfo->setFAIndex(FrameAddrIndex);
27100     }
27101     return DAG.getFrameIndex(FrameAddrIndex, VT);
27102   }
27103 
27104   unsigned FrameReg =
27105       RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
27106   SDLoc dl(Op);  // FIXME probably not meaningful
27107   unsigned Depth = Op.getConstantOperandVal(0);
27108   assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
27109           (FrameReg == X86::EBP && VT == MVT::i32)) &&
27110          "Invalid Frame Register!");
27111   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
27112   while (Depth--)
27113     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
27114                             MachinePointerInfo());
27115   return FrameAddr;
27116 }
27117 
27118 // FIXME? Maybe this could be a TableGen attribute on some registers and
27119 // this table could be generated automatically from RegInfo.
27120 Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
27121                                               const MachineFunction &MF) const {
27122   const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
27123 
27124   Register Reg = StringSwitch<unsigned>(RegName)
27125                      .Case("esp", X86::ESP)
27126                      .Case("rsp", X86::RSP)
27127                      .Case("ebp", X86::EBP)
27128                      .Case("rbp", X86::RBP)
27129                      .Case("r14", X86::R14)
27130                      .Case("r15", X86::R15)
27131                      .Default(0);
27132 
27133   if (Reg == X86::EBP || Reg == X86::RBP) {
27134     if (!TFI.hasFP(MF))
27135       report_fatal_error("register " + StringRef(RegName) +
27136                          " is allocatable: function has no frame pointer");
27137 #ifndef NDEBUG
27138     else {
27139       const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27140       Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF);
27141       assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
27142              "Invalid Frame Register!");
27143     }
27144 #endif
27145   }
27146 
27147   if (Reg)
27148     return Reg;
27149 
27150   report_fatal_error("Invalid register name global variable");
27151 }
27152 
27153 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
27154                                                      SelectionDAG &DAG) const {
27155   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27156   return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
27157 }
27158 
27159 Register X86TargetLowering::getExceptionPointerRegister(
27160     const Constant *PersonalityFn) const {
27161   if (classifyEHPersonality(PersonalityFn) == EHPersonality::CoreCLR)
27162     return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27163 
27164   return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX;
27165 }
27166 
27167 Register X86TargetLowering::getExceptionSelectorRegister(
27168     const Constant *PersonalityFn) const {
27169   // Funclet personalities don't use selectors (the runtime does the selection).
27170   if (isFuncletEHPersonality(classifyEHPersonality(PersonalityFn)))
27171     return X86::NoRegister;
27172   return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27173 }
27174 
27175 bool X86TargetLowering::needsFixedCatchObjects() const {
27176   return Subtarget.isTargetWin64();
27177 }
27178 
27179 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
27180   SDValue Chain     = Op.getOperand(0);
27181   SDValue Offset    = Op.getOperand(1);
27182   SDValue Handler   = Op.getOperand(2);
27183   SDLoc dl      (Op);
27184 
27185   EVT PtrVT = getPointerTy(DAG.getDataLayout());
27186   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27187   Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
27188   assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
27189           (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
27190          "Invalid Frame Register!");
27191   SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
27192   Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
27193 
27194   SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
27195                                  DAG.getIntPtrConstant(RegInfo->getSlotSize(),
27196                                                        dl));
27197   StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
27198   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
27199   Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
27200 
27201   return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
27202                      DAG.getRegister(StoreAddrReg, PtrVT));
27203 }
27204 
27205 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
27206                                                SelectionDAG &DAG) const {
27207   SDLoc DL(Op);
27208   // If the subtarget is not 64bit, we may need the global base reg
27209   // after isel expand pseudo, i.e., after CGBR pass ran.
27210   // Therefore, ask for the GlobalBaseReg now, so that the pass
27211   // inserts the code for us in case we need it.
27212   // Otherwise, we will end up in a situation where we will
27213   // reference a virtual register that is not defined!
27214   if (!Subtarget.is64Bit()) {
27215     const X86InstrInfo *TII = Subtarget.getInstrInfo();
27216     (void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
27217   }
27218   return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
27219                      DAG.getVTList(MVT::i32, MVT::Other),
27220                      Op.getOperand(0), Op.getOperand(1));
27221 }
27222 
27223 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
27224                                                 SelectionDAG &DAG) const {
27225   SDLoc DL(Op);
27226   return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
27227                      Op.getOperand(0), Op.getOperand(1));
27228 }
27229 
27230 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
27231                                                        SelectionDAG &DAG) const {
27232   SDLoc DL(Op);
27233   return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
27234                      Op.getOperand(0));
27235 }
27236 
27237 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
27238   return Op.getOperand(0);
27239 }
27240 
27241 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
27242                                                 SelectionDAG &DAG) const {
27243   SDValue Root = Op.getOperand(0);
27244   SDValue Trmp = Op.getOperand(1); // trampoline
27245   SDValue FPtr = Op.getOperand(2); // nested function
27246   SDValue Nest = Op.getOperand(3); // 'nest' parameter value
27247   SDLoc dl (Op);
27248 
27249   const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
27250   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
27251 
27252   if (Subtarget.is64Bit()) {
27253     SDValue OutChains[6];
27254 
27255     // Large code-model.
27256     const unsigned char JMP64r  = 0xFF; // 64-bit jmp through register opcode.
27257     const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
27258 
27259     const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
27260     const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
27261 
27262     const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
27263 
27264     // Load the pointer to the nested function into R11.
27265     unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
27266     SDValue Addr = Trmp;
27267     OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27268                                 Addr, MachinePointerInfo(TrmpAddr));
27269 
27270     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27271                        DAG.getConstant(2, dl, MVT::i64));
27272     OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
27273                                 MachinePointerInfo(TrmpAddr, 2), Align(2));
27274 
27275     // Load the 'nest' parameter value into R10.
27276     // R10 is specified in X86CallingConv.td
27277     OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
27278     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27279                        DAG.getConstant(10, dl, MVT::i64));
27280     OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27281                                 Addr, MachinePointerInfo(TrmpAddr, 10));
27282 
27283     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27284                        DAG.getConstant(12, dl, MVT::i64));
27285     OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
27286                                 MachinePointerInfo(TrmpAddr, 12), Align(2));
27287 
27288     // Jump to the nested function.
27289     OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
27290     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27291                        DAG.getConstant(20, dl, MVT::i64));
27292     OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27293                                 Addr, MachinePointerInfo(TrmpAddr, 20));
27294 
27295     unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
27296     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27297                        DAG.getConstant(22, dl, MVT::i64));
27298     OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
27299                                 Addr, MachinePointerInfo(TrmpAddr, 22));
27300 
27301     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27302   } else {
27303     const Function *Func =
27304       cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
27305     CallingConv::ID CC = Func->getCallingConv();
27306     unsigned NestReg;
27307 
27308     switch (CC) {
27309     default:
27310       llvm_unreachable("Unsupported calling convention");
27311     case CallingConv::C:
27312     case CallingConv::X86_StdCall: {
27313       // Pass 'nest' parameter in ECX.
27314       // Must be kept in sync with X86CallingConv.td
27315       NestReg = X86::ECX;
27316 
27317       // Check that ECX wasn't needed by an 'inreg' parameter.
27318       FunctionType *FTy = Func->getFunctionType();
27319       const AttributeList &Attrs = Func->getAttributes();
27320 
27321       if (!Attrs.isEmpty() && !Func->isVarArg()) {
27322         unsigned InRegCount = 0;
27323         unsigned Idx = 0;
27324 
27325         for (FunctionType::param_iterator I = FTy->param_begin(),
27326              E = FTy->param_end(); I != E; ++I, ++Idx)
27327           if (Attrs.hasParamAttr(Idx, Attribute::InReg)) {
27328             const DataLayout &DL = DAG.getDataLayout();
27329             // FIXME: should only count parameters that are lowered to integers.
27330             InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
27331           }
27332 
27333         if (InRegCount > 2) {
27334           report_fatal_error("Nest register in use - reduce number of inreg"
27335                              " parameters!");
27336         }
27337       }
27338       break;
27339     }
27340     case CallingConv::X86_FastCall:
27341     case CallingConv::X86_ThisCall:
27342     case CallingConv::Fast:
27343     case CallingConv::Tail:
27344     case CallingConv::SwiftTail:
27345       // Pass 'nest' parameter in EAX.
27346       // Must be kept in sync with X86CallingConv.td
27347       NestReg = X86::EAX;
27348       break;
27349     }
27350 
27351     SDValue OutChains[4];
27352     SDValue Addr, Disp;
27353 
27354     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27355                        DAG.getConstant(10, dl, MVT::i32));
27356     Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
27357 
27358     // This is storing the opcode for MOV32ri.
27359     const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
27360     const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
27361     OutChains[0] =
27362         DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),
27363                      Trmp, MachinePointerInfo(TrmpAddr));
27364 
27365     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27366                        DAG.getConstant(1, dl, MVT::i32));
27367     OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
27368                                 MachinePointerInfo(TrmpAddr, 1), Align(1));
27369 
27370     const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
27371     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27372                        DAG.getConstant(5, dl, MVT::i32));
27373     OutChains[2] =
27374         DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr,
27375                      MachinePointerInfo(TrmpAddr, 5), Align(1));
27376 
27377     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27378                        DAG.getConstant(6, dl, MVT::i32));
27379     OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
27380                                 MachinePointerInfo(TrmpAddr, 6), Align(1));
27381 
27382     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27383   }
27384 }
27385 
27386 SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
27387                                              SelectionDAG &DAG) const {
27388   /*
27389    The rounding mode is in bits 11:10 of FPSR, and has the following
27390    settings:
27391      00 Round to nearest
27392      01 Round to -inf
27393      10 Round to +inf
27394      11 Round to 0
27395 
27396   GET_ROUNDING, on the other hand, expects the following:
27397     -1 Undefined
27398      0 Round to 0
27399      1 Round to nearest
27400      2 Round to +inf
27401      3 Round to -inf
27402 
27403   To perform the conversion, we use a packed lookup table of the four 2-bit
27404   values that we can index by FPSP[11:10]
27405     0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
27406 
27407     (0x2d >> ((FPSR & 0xc00) >> 9)) & 3
27408   */
27409 
27410   MachineFunction &MF = DAG.getMachineFunction();
27411   MVT VT = Op.getSimpleValueType();
27412   SDLoc DL(Op);
27413 
27414   // Save FP Control Word to stack slot
27415   int SSFI = MF.getFrameInfo().CreateStackObject(2, Align(2), false);
27416   SDValue StackSlot =
27417       DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
27418 
27419   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
27420 
27421   SDValue Chain = Op.getOperand(0);
27422   SDValue Ops[] = {Chain, StackSlot};
27423   Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
27424                                   DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,
27425                                   Align(2), MachineMemOperand::MOStore);
27426 
27427   // Load FP Control Word from stack slot
27428   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
27429   Chain = CWD.getValue(1);
27430 
27431   // Mask and turn the control bits into a shift for the lookup table.
27432   SDValue Shift =
27433     DAG.getNode(ISD::SRL, DL, MVT::i16,
27434                 DAG.getNode(ISD::AND, DL, MVT::i16,
27435                             CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
27436                 DAG.getConstant(9, DL, MVT::i8));
27437   Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
27438 
27439   SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
27440   SDValue RetVal =
27441     DAG.getNode(ISD::AND, DL, MVT::i32,
27442                 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
27443                 DAG.getConstant(3, DL, MVT::i32));
27444 
27445   RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
27446 
27447   return DAG.getMergeValues({RetVal, Chain}, DL);
27448 }
27449 
27450 SDValue X86TargetLowering::LowerSET_ROUNDING(SDValue Op,
27451                                              SelectionDAG &DAG) const {
27452   MachineFunction &MF = DAG.getMachineFunction();
27453   SDLoc DL(Op);
27454   SDValue Chain = Op.getNode()->getOperand(0);
27455 
27456   // FP control word may be set only from data in memory. So we need to allocate
27457   // stack space to save/load FP control word.
27458   int OldCWFrameIdx = MF.getFrameInfo().CreateStackObject(4, Align(4), false);
27459   SDValue StackSlot =
27460       DAG.getFrameIndex(OldCWFrameIdx, getPointerTy(DAG.getDataLayout()));
27461   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, OldCWFrameIdx);
27462   MachineMemOperand *MMO =
27463       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 2, Align(2));
27464 
27465   // Store FP control word into memory.
27466   SDValue Ops[] = {Chain, StackSlot};
27467   Chain = DAG.getMemIntrinsicNode(
27468       X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO);
27469 
27470   // Load FP Control Word from stack slot and clear RM field (bits 11:10).
27471   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);
27472   Chain = CWD.getValue(1);
27473   CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0),
27474                     DAG.getConstant(0xf3ff, DL, MVT::i16));
27475 
27476   // Calculate new rounding mode.
27477   SDValue NewRM = Op.getNode()->getOperand(1);
27478   SDValue RMBits;
27479   if (auto *CVal = dyn_cast<ConstantSDNode>(NewRM)) {
27480     uint64_t RM = CVal->getZExtValue();
27481     int FieldVal;
27482     switch (static_cast<RoundingMode>(RM)) {
27483     case RoundingMode::NearestTiesToEven: FieldVal = X86::rmToNearest; break;
27484     case RoundingMode::TowardNegative:    FieldVal = X86::rmDownward; break;
27485     case RoundingMode::TowardPositive:    FieldVal = X86::rmUpward; break;
27486     case RoundingMode::TowardZero:        FieldVal = X86::rmTowardZero; break;
27487     default:
27488       llvm_unreachable("rounding mode is not supported by X86 hardware");
27489     }
27490     RMBits = DAG.getConstant(FieldVal, DL, MVT::i16);
27491   } else {
27492     // Need to convert argument into bits of control word:
27493     //    0 Round to 0       -> 11
27494     //    1 Round to nearest -> 00
27495     //    2 Round to +inf    -> 10
27496     //    3 Round to -inf    -> 01
27497     // The 2-bit value needs then to be shifted so that it occupies bits 11:10.
27498     // To make the conversion, put all these values into a value 0xc9 and shift
27499     // it left depending on the rounding mode:
27500     //    (0xc9 << 4) & 0xc00 = X86::rmTowardZero
27501     //    (0xc9 << 6) & 0xc00 = X86::rmToNearest
27502     //    ...
27503     // (0xc9 << (2 * NewRM + 4)) & 0xc00
27504     SDValue ShiftValue =
27505         DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
27506                     DAG.getNode(ISD::ADD, DL, MVT::i32,
27507                                 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM,
27508                                             DAG.getConstant(1, DL, MVT::i8)),
27509                                 DAG.getConstant(4, DL, MVT::i32)));
27510     SDValue Shifted =
27511         DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16),
27512                     ShiftValue);
27513     RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted,
27514                          DAG.getConstant(0xc00, DL, MVT::i16));
27515   }
27516 
27517   // Update rounding mode bits and store the new FP Control Word into stack.
27518   CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits);
27519   Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(2));
27520 
27521   // Load FP control word from the slot.
27522   SDValue OpsLD[] = {Chain, StackSlot};
27523   MachineMemOperand *MMOL =
27524       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 2, Align(2));
27525   Chain = DAG.getMemIntrinsicNode(
27526       X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL);
27527 
27528   // If target supports SSE, set MXCSR as well. Rounding mode is encoded in the
27529   // same way but in bits 14:13.
27530   if (Subtarget.hasSSE1()) {
27531     // Store MXCSR into memory.
27532     Chain = DAG.getNode(
27533         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27534         DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27535         StackSlot);
27536 
27537     // Load MXCSR from stack slot and clear RM field (bits 14:13).
27538     SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);
27539     Chain = CWD.getValue(1);
27540     CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0),
27541                       DAG.getConstant(0xffff9fff, DL, MVT::i32));
27542 
27543     // Shift X87 RM bits from 11:10 to 14:13.
27544     RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits);
27545     RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits,
27546                          DAG.getConstant(3, DL, MVT::i8));
27547 
27548     // Update rounding mode bits and store the new FP Control Word into stack.
27549     CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits);
27550     Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(4));
27551 
27552     // Load MXCSR from the slot.
27553     Chain = DAG.getNode(
27554         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27555         DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27556         StackSlot);
27557   }
27558 
27559   return Chain;
27560 }
27561 
27562 const unsigned X87StateSize = 28;
27563 const unsigned FPStateSize = 32;
27564 [[maybe_unused]] const unsigned FPStateSizeInBits = FPStateSize * 8;
27565 
27566 SDValue X86TargetLowering::LowerGET_FPENV_MEM(SDValue Op,
27567                                               SelectionDAG &DAG) const {
27568   MachineFunction &MF = DAG.getMachineFunction();
27569   SDLoc DL(Op);
27570   SDValue Chain = Op->getOperand(0);
27571   SDValue Ptr = Op->getOperand(1);
27572   auto *Node = cast<FPStateAccessSDNode>(Op);
27573   EVT MemVT = Node->getMemoryVT();
27574   assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27575   MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27576 
27577   // Get x87 state, if it presents.
27578   if (Subtarget.hasX87()) {
27579     Chain =
27580         DAG.getMemIntrinsicNode(X86ISD::FNSTENVm, DL, DAG.getVTList(MVT::Other),
27581                                 {Chain, Ptr}, MemVT, MMO);
27582 
27583     // FNSTENV changes the exception mask, so load back the stored environment.
27584     MachineMemOperand::Flags NewFlags =
27585         MachineMemOperand::MOLoad |
27586         (MMO->getFlags() & ~MachineMemOperand::MOStore);
27587     MMO = MF.getMachineMemOperand(MMO, NewFlags);
27588     Chain =
27589         DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27590                                 {Chain, Ptr}, MemVT, MMO);
27591   }
27592 
27593   // If target supports SSE, get MXCSR as well.
27594   if (Subtarget.hasSSE1()) {
27595     // Get pointer to the MXCSR location in memory.
27596     MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27597     SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27598                                     DAG.getConstant(X87StateSize, DL, PtrVT));
27599     // Store MXCSR into memory.
27600     Chain = DAG.getNode(
27601         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27602         DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27603         MXCSRAddr);
27604   }
27605 
27606   return Chain;
27607 }
27608 
27609 static SDValue createSetFPEnvNodes(SDValue Ptr, SDValue Chain, SDLoc DL,
27610                                    EVT MemVT, MachineMemOperand *MMO,
27611                                    SelectionDAG &DAG,
27612                                    const X86Subtarget &Subtarget) {
27613   // Set x87 state, if it presents.
27614   if (Subtarget.hasX87())
27615     Chain =
27616         DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27617                                 {Chain, Ptr}, MemVT, MMO);
27618   // If target supports SSE, set MXCSR as well.
27619   if (Subtarget.hasSSE1()) {
27620     // Get pointer to the MXCSR location in memory.
27621     MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27622     SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27623                                     DAG.getConstant(X87StateSize, DL, PtrVT));
27624     // Load MXCSR from memory.
27625     Chain = DAG.getNode(
27626         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27627         DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27628         MXCSRAddr);
27629   }
27630   return Chain;
27631 }
27632 
27633 SDValue X86TargetLowering::LowerSET_FPENV_MEM(SDValue Op,
27634                                               SelectionDAG &DAG) const {
27635   SDLoc DL(Op);
27636   SDValue Chain = Op->getOperand(0);
27637   SDValue Ptr = Op->getOperand(1);
27638   auto *Node = cast<FPStateAccessSDNode>(Op);
27639   EVT MemVT = Node->getMemoryVT();
27640   assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27641   MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27642   return createSetFPEnvNodes(Ptr, Chain, DL, MemVT, MMO, DAG, Subtarget);
27643 }
27644 
27645 SDValue X86TargetLowering::LowerRESET_FPENV(SDValue Op,
27646                                             SelectionDAG &DAG) const {
27647   MachineFunction &MF = DAG.getMachineFunction();
27648   SDLoc DL(Op);
27649   SDValue Chain = Op.getNode()->getOperand(0);
27650 
27651   IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext());
27652   ArrayType *FPEnvTy = ArrayType::get(ItemTy, 8);
27653   SmallVector<Constant *, 8> FPEnvVals;
27654 
27655   // x87 FPU Control Word: mask all floating-point exceptions, sets rounding to
27656   // nearest. FPU precision is set to 53 bits on Windows and 64 bits otherwise
27657   // for compatibility with glibc.
27658   unsigned X87CW = Subtarget.isTargetWindowsMSVC() ? 0x27F : 0x37F;
27659   FPEnvVals.push_back(ConstantInt::get(ItemTy, X87CW));
27660   Constant *Zero = ConstantInt::get(ItemTy, 0);
27661   for (unsigned I = 0; I < 6; ++I)
27662     FPEnvVals.push_back(Zero);
27663 
27664   // MXCSR: mask all floating-point exceptions, sets rounding to nearest, clear
27665   // all exceptions, sets DAZ and FTZ to 0.
27666   FPEnvVals.push_back(ConstantInt::get(ItemTy, 0x1F80));
27667   Constant *FPEnvBits = ConstantArray::get(FPEnvTy, FPEnvVals);
27668   MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27669   SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT);
27670   MachinePointerInfo MPI =
27671       MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
27672   MachineMemOperand *MMO = MF.getMachineMemOperand(
27673       MPI, MachineMemOperand::MOStore, X87StateSize, Align(4));
27674 
27675   return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget);
27676 }
27677 
27678 /// Lower a vector CTLZ using native supported vector CTLZ instruction.
27679 //
27680 // i8/i16 vector implemented using dword LZCNT vector instruction
27681 // ( sub(trunc(lzcnt(zext32(x)))) ). In case zext32(x) is illegal,
27682 // split the vector, perform operation on it's Lo a Hi part and
27683 // concatenate the results.
27684 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
27685                                          const X86Subtarget &Subtarget) {
27686   assert(Op.getOpcode() == ISD::CTLZ);
27687   SDLoc dl(Op);
27688   MVT VT = Op.getSimpleValueType();
27689   MVT EltVT = VT.getVectorElementType();
27690   unsigned NumElems = VT.getVectorNumElements();
27691 
27692   assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&
27693           "Unsupported element type");
27694 
27695   // Split vector, it's Lo and Hi parts will be handled in next iteration.
27696   if (NumElems > 16 ||
27697       (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
27698     return splitVectorIntUnary(Op, DAG);
27699 
27700   MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
27701   assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
27702           "Unsupported value type for operation");
27703 
27704   // Use native supported vector instruction vplzcntd.
27705   Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
27706   SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
27707   SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
27708   SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
27709 
27710   return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);
27711 }
27712 
27713 // Lower CTLZ using a PSHUFB lookup table implementation.
27714 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
27715                                        const X86Subtarget &Subtarget,
27716                                        SelectionDAG &DAG) {
27717   MVT VT = Op.getSimpleValueType();
27718   int NumElts = VT.getVectorNumElements();
27719   int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);
27720   MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);
27721 
27722   // Per-nibble leading zero PSHUFB lookup table.
27723   const int LUT[16] = {/* 0 */ 4, /* 1 */ 3, /* 2 */ 2, /* 3 */ 2,
27724                        /* 4 */ 1, /* 5 */ 1, /* 6 */ 1, /* 7 */ 1,
27725                        /* 8 */ 0, /* 9 */ 0, /* a */ 0, /* b */ 0,
27726                        /* c */ 0, /* d */ 0, /* e */ 0, /* f */ 0};
27727 
27728   SmallVector<SDValue, 64> LUTVec;
27729   for (int i = 0; i < NumBytes; ++i)
27730     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
27731   SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
27732 
27733   // Begin by bitcasting the input to byte vector, then split those bytes
27734   // into lo/hi nibbles and use the PSHUFB LUT to perform CLTZ on each of them.
27735   // If the hi input nibble is zero then we add both results together, otherwise
27736   // we just take the hi result (by masking the lo result to zero before the
27737   // add).
27738   SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
27739   SDValue Zero = DAG.getConstant(0, DL, CurrVT);
27740 
27741   SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
27742   SDValue Lo = Op0;
27743   SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
27744   SDValue HiZ;
27745   if (CurrVT.is512BitVector()) {
27746     MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
27747     HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
27748     HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
27749   } else {
27750     HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
27751   }
27752 
27753   Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
27754   Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
27755   Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
27756   SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
27757 
27758   // Merge result back from vXi8 back to VT, working on the lo/hi halves
27759   // of the current vector width in the same way we did for the nibbles.
27760   // If the upper half of the input element is zero then add the halves'
27761   // leading zero counts together, otherwise just use the upper half's.
27762   // Double the width of the result until we are at target width.
27763   while (CurrVT != VT) {
27764     int CurrScalarSizeInBits = CurrVT.getScalarSizeInBits();
27765     int CurrNumElts = CurrVT.getVectorNumElements();
27766     MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);
27767     MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);
27768     SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
27769 
27770     // Check if the upper half of the input element is zero.
27771     if (CurrVT.is512BitVector()) {
27772       MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
27773       HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
27774                          DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
27775       HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
27776     } else {
27777       HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
27778                          DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
27779     }
27780     HiZ = DAG.getBitcast(NextVT, HiZ);
27781 
27782     // Move the upper/lower halves to the lower bits as we'll be extending to
27783     // NextVT. Mask the lower result to zero if HiZ is true and add the results
27784     // together.
27785     SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
27786     SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
27787     SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
27788     R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
27789     Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
27790     CurrVT = NextVT;
27791   }
27792 
27793   return Res;
27794 }
27795 
27796 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
27797                                const X86Subtarget &Subtarget,
27798                                SelectionDAG &DAG) {
27799   MVT VT = Op.getSimpleValueType();
27800 
27801   if (Subtarget.hasCDI() &&
27802       // vXi8 vectors need to be promoted to 512-bits for vXi32.
27803       (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))
27804     return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget);
27805 
27806   // Decompose 256-bit ops into smaller 128-bit ops.
27807   if (VT.is256BitVector() && !Subtarget.hasInt256())
27808     return splitVectorIntUnary(Op, DAG);
27809 
27810   // Decompose 512-bit ops into smaller 256-bit ops.
27811   if (VT.is512BitVector() && !Subtarget.hasBWI())
27812     return splitVectorIntUnary(Op, DAG);
27813 
27814   assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB");
27815   return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
27816 }
27817 
27818 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
27819                          SelectionDAG &DAG) {
27820   MVT VT = Op.getSimpleValueType();
27821   MVT OpVT = VT;
27822   unsigned NumBits = VT.getSizeInBits();
27823   SDLoc dl(Op);
27824   unsigned Opc = Op.getOpcode();
27825 
27826   if (VT.isVector())
27827     return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
27828 
27829   Op = Op.getOperand(0);
27830   if (VT == MVT::i8) {
27831     // Zero extend to i32 since there is not an i8 bsr.
27832     OpVT = MVT::i32;
27833     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
27834   }
27835 
27836   // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
27837   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
27838   Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
27839 
27840   if (Opc == ISD::CTLZ) {
27841     // If src is zero (i.e. bsr sets ZF), returns NumBits.
27842     SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
27843                      DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
27844                      Op.getValue(1)};
27845     Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
27846   }
27847 
27848   // Finally xor with NumBits-1.
27849   Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
27850                    DAG.getConstant(NumBits - 1, dl, OpVT));
27851 
27852   if (VT == MVT::i8)
27853     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
27854   return Op;
27855 }
27856 
27857 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
27858                          SelectionDAG &DAG) {
27859   MVT VT = Op.getSimpleValueType();
27860   unsigned NumBits = VT.getScalarSizeInBits();
27861   SDValue N0 = Op.getOperand(0);
27862   SDLoc dl(Op);
27863 
27864   assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&
27865          "Only scalar CTTZ requires custom lowering");
27866 
27867   // Issue a bsf (scan bits forward) which also sets EFLAGS.
27868   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
27869   Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0);
27870 
27871   // If src is known never zero we can skip the CMOV.
27872   if (DAG.isKnownNeverZero(N0))
27873     return Op;
27874 
27875   // If src is zero (i.e. bsf sets ZF), returns NumBits.
27876   SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
27877                    DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
27878                    Op.getValue(1)};
27879   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
27880 }
27881 
27882 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
27883                            const X86Subtarget &Subtarget) {
27884   MVT VT = Op.getSimpleValueType();
27885   if (VT == MVT::i16 || VT == MVT::i32)
27886     return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
27887 
27888   if (VT == MVT::v32i16 || VT == MVT::v64i8)
27889     return splitVectorIntBinary(Op, DAG);
27890 
27891   assert(Op.getSimpleValueType().is256BitVector() &&
27892          Op.getSimpleValueType().isInteger() &&
27893          "Only handle AVX 256-bit vector integer operation");
27894   return splitVectorIntBinary(Op, DAG);
27895 }
27896 
27897 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
27898                                   const X86Subtarget &Subtarget) {
27899   MVT VT = Op.getSimpleValueType();
27900   SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
27901   unsigned Opcode = Op.getOpcode();
27902   SDLoc DL(Op);
27903 
27904   if (VT == MVT::v32i16 || VT == MVT::v64i8 ||
27905       (VT.is256BitVector() && !Subtarget.hasInt256())) {
27906     assert(Op.getSimpleValueType().isInteger() &&
27907            "Only handle AVX vector integer operation");
27908     return splitVectorIntBinary(Op, DAG);
27909   }
27910 
27911   // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*.
27912   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27913   EVT SetCCResultType =
27914       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
27915 
27916   unsigned BitWidth = VT.getScalarSizeInBits();
27917   if (Opcode == ISD::USUBSAT) {
27918     if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) {
27919       // Handle a special-case with a bit-hack instead of cmp+select:
27920       // usubsat X, SMIN --> (X ^ SMIN) & (X s>> BW-1)
27921       // If the target can use VPTERNLOG, DAGToDAG will match this as
27922       // "vpsra + vpternlog" which is better than "vpmax + vpsub" with a
27923       // "broadcast" constant load.
27924       ConstantSDNode *C = isConstOrConstSplat(Y, true);
27925       if (C && C->getAPIntValue().isSignMask()) {
27926         SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT);
27927         SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT);
27928         SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask);
27929         SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt);
27930         return DAG.getNode(ISD::AND, DL, VT, Xor, Sra);
27931       }
27932     }
27933     if (!TLI.isOperationLegal(ISD::UMAX, VT)) {
27934       // usubsat X, Y --> (X >u Y) ? X - Y : 0
27935       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
27936       SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
27937       // TODO: Move this to DAGCombiner?
27938       if (SetCCResultType == VT &&
27939           DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits())
27940         return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub);
27941       return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
27942     }
27943   }
27944 
27945   if ((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
27946       (!VT.isVector() || VT == MVT::v2i64)) {
27947     APInt MinVal = APInt::getSignedMinValue(BitWidth);
27948     APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
27949     SDValue Zero = DAG.getConstant(0, DL, VT);
27950     SDValue Result =
27951         DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL,
27952                     DAG.getVTList(VT, SetCCResultType), X, Y);
27953     SDValue SumDiff = Result.getValue(0);
27954     SDValue Overflow = Result.getValue(1);
27955     SDValue SatMin = DAG.getConstant(MinVal, DL, VT);
27956     SDValue SatMax = DAG.getConstant(MaxVal, DL, VT);
27957     SDValue SumNeg =
27958         DAG.getSetCC(DL, SetCCResultType, SumDiff, Zero, ISD::SETLT);
27959     Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin);
27960     return DAG.getSelect(DL, VT, Overflow, Result, SumDiff);
27961   }
27962 
27963   // Use default expansion.
27964   return SDValue();
27965 }
27966 
27967 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
27968                         SelectionDAG &DAG) {
27969   MVT VT = Op.getSimpleValueType();
27970   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {
27971     // Since X86 does not have CMOV for 8-bit integer, we don't convert
27972     // 8-bit integer abs to NEG and CMOV.
27973     SDLoc DL(Op);
27974     SDValue N0 = Op.getOperand(0);
27975     SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
27976                               DAG.getConstant(0, DL, VT), N0);
27977     SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
27978                      SDValue(Neg.getNode(), 1)};
27979     return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
27980   }
27981 
27982   // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X).
27983   if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {
27984     SDLoc DL(Op);
27985     SDValue Src = Op.getOperand(0);
27986     SDValue Sub =
27987         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
27988     return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);
27989   }
27990 
27991   if (VT.is256BitVector() && !Subtarget.hasInt256()) {
27992     assert(VT.isInteger() &&
27993            "Only handle AVX 256-bit vector integer operation");
27994     return splitVectorIntUnary(Op, DAG);
27995   }
27996 
27997   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
27998     return splitVectorIntUnary(Op, DAG);
27999 
28000   // Default to expand.
28001   return SDValue();
28002 }
28003 
28004 static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget,
28005                         SelectionDAG &DAG) {
28006   MVT VT = Op.getSimpleValueType();
28007 
28008   // For AVX1 cases, split to use legal ops.
28009   if (VT.is256BitVector() && !Subtarget.hasInt256())
28010     return splitVectorIntBinary(Op, DAG);
28011 
28012   if (VT == MVT::v32i16 || VT == MVT::v64i8)
28013     return splitVectorIntBinary(Op, DAG);
28014 
28015   // Default to expand.
28016   return SDValue();
28017 }
28018 
28019 static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
28020                            SelectionDAG &DAG) {
28021   MVT VT = Op.getSimpleValueType();
28022 
28023   // For AVX1 cases, split to use legal ops.
28024   if (VT.is256BitVector() && !Subtarget.hasInt256())
28025     return splitVectorIntBinary(Op, DAG);
28026 
28027   if (VT == MVT::v32i16 || VT == MVT::v64i8)
28028     return splitVectorIntBinary(Op, DAG);
28029 
28030   // Default to expand.
28031   return SDValue();
28032 }
28033 
28034 static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
28035                                       SelectionDAG &DAG) {
28036   assert((Op.getOpcode() == ISD::FMAXIMUM || Op.getOpcode() == ISD::FMINIMUM) &&
28037          "Expected FMAXIMUM or FMINIMUM opcode");
28038   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28039   EVT VT = Op.getValueType();
28040   SDValue X = Op.getOperand(0);
28041   SDValue Y = Op.getOperand(1);
28042   SDLoc DL(Op);
28043   uint64_t SizeInBits = VT.getScalarSizeInBits();
28044   APInt PreferredZero = APInt::getZero(SizeInBits);
28045   APInt OppositeZero = PreferredZero;
28046   EVT IVT = VT.changeTypeToInteger();
28047   X86ISD::NodeType MinMaxOp;
28048   if (Op.getOpcode() == ISD::FMAXIMUM) {
28049     MinMaxOp = X86ISD::FMAX;
28050     OppositeZero.setSignBit();
28051   } else {
28052     PreferredZero.setSignBit();
28053     MinMaxOp = X86ISD::FMIN;
28054   }
28055   EVT SetCCType =
28056       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28057 
28058   // The tables below show the expected result of Max in cases of NaN and
28059   // signed zeros.
28060   //
28061   //                 Y                       Y
28062   //             Num   xNaN              +0     -0
28063   //          ---------------         ---------------
28064   //     Num  |  Max |   Y  |     +0  |  +0  |  +0  |
28065   // X        ---------------  X      ---------------
28066   //    xNaN  |   X  |  X/Y |     -0  |  +0  |  -0  |
28067   //          ---------------         ---------------
28068   //
28069   // It is achieved by means of FMAX/FMIN with preliminary checks and operand
28070   // reordering.
28071   //
28072   // We check if any of operands is NaN and return NaN. Then we check if any of
28073   // operands is zero or negative zero (for fmaximum and fminimum respectively)
28074   // to ensure the correct zero is returned.
28075   auto MatchesZero = [](SDValue Op, APInt Zero) {
28076     Op = peekThroughBitcasts(Op);
28077     if (auto *CstOp = dyn_cast<ConstantFPSDNode>(Op))
28078       return CstOp->getValueAPF().bitcastToAPInt() == Zero;
28079     if (auto *CstOp = dyn_cast<ConstantSDNode>(Op))
28080       return CstOp->getAPIntValue() == Zero;
28081     if (Op->getOpcode() == ISD::BUILD_VECTOR ||
28082         Op->getOpcode() == ISD::SPLAT_VECTOR) {
28083       for (const SDValue &OpVal : Op->op_values()) {
28084         if (OpVal.isUndef())
28085           continue;
28086         auto *CstOp = dyn_cast<ConstantFPSDNode>(OpVal);
28087         if (!CstOp)
28088           return false;
28089         if (!CstOp->getValueAPF().isZero())
28090           continue;
28091         if (CstOp->getValueAPF().bitcastToAPInt() != Zero)
28092           return false;
28093       }
28094       return true;
28095     }
28096     return false;
28097   };
28098 
28099   bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
28100   bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
28101   bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
28102                           Op->getFlags().hasNoSignedZeros() ||
28103                           DAG.isKnownNeverZeroFloat(X) ||
28104                           DAG.isKnownNeverZeroFloat(Y);
28105   SDValue NewX, NewY;
28106   if (IgnoreSignedZero || MatchesZero(Y, PreferredZero) ||
28107       MatchesZero(X, OppositeZero)) {
28108     // Operands are already in right order or order does not matter.
28109     NewX = X;
28110     NewY = Y;
28111   } else if (MatchesZero(X, PreferredZero) || MatchesZero(Y, OppositeZero)) {
28112     NewX = Y;
28113     NewY = X;
28114   } else if (!VT.isVector() && (VT == MVT::f16 || Subtarget.hasDQI()) &&
28115              (Op->getFlags().hasNoNaNs() || IsXNeverNaN || IsYNeverNaN)) {
28116     if (IsXNeverNaN)
28117       std::swap(X, Y);
28118     // VFPCLASSS consumes a vector type. So provide a minimal one corresponded
28119     // xmm register.
28120     MVT VectorType = MVT::getVectorVT(VT.getSimpleVT(), 128 / SizeInBits);
28121     SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
28122     // Bits of classes:
28123     // Bits  Imm8[0] Imm8[1] Imm8[2] Imm8[3] Imm8[4]  Imm8[5]  Imm8[6] Imm8[7]
28124     // Class    QNAN PosZero NegZero  PosINF  NegINF Denormal Negative    SNAN
28125     SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
28126                                         DL, MVT::i32);
28127     SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
28128     SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
28129                               DAG.getConstant(0, DL, MVT::v8i1), IsNanZero,
28130                               DAG.getIntPtrConstant(0, DL));
28131     SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
28132     NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X);
28133     NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
28134     return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28135   } else {
28136     SDValue IsXSigned;
28137     if (Subtarget.is64Bit() || VT != MVT::f64) {
28138       SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
28139       SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
28140       IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
28141     } else {
28142       assert(VT == MVT::f64);
28143       SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
28144                                 DAG.getConstantFP(0, DL, MVT::v2f64), X,
28145                                 DAG.getIntPtrConstant(0, DL));
28146       SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
28147       SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
28148                                DAG.getIntPtrConstant(1, DL));
28149       Hi = DAG.getBitcast(MVT::i32, Hi);
28150       SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
28151       EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
28152                                              *DAG.getContext(), MVT::i32);
28153       IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
28154     }
28155     if (MinMaxOp == X86ISD::FMAX) {
28156       NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28157       NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28158     } else {
28159       NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28160       NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28161     }
28162   }
28163 
28164   bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
28165                    Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
28166 
28167   // If we did no ordering operands for signed zero handling and we need
28168   // to process NaN and we know that the second operand is not NaN then put
28169   // it in first operand and we will not need to post handle NaN after max/min.
28170   if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
28171     std::swap(NewX, NewY);
28172 
28173   SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28174 
28175   if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
28176     return MinMax;
28177 
28178   SDValue IsNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO);
28179   return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax);
28180 }
28181 
28182 static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
28183                         SelectionDAG &DAG) {
28184   MVT VT = Op.getSimpleValueType();
28185 
28186   // For AVX1 cases, split to use legal ops.
28187   if (VT.is256BitVector() && !Subtarget.hasInt256())
28188     return splitVectorIntBinary(Op, DAG);
28189 
28190   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.useBWIRegs())
28191     return splitVectorIntBinary(Op, DAG);
28192 
28193   SDLoc dl(Op);
28194   bool IsSigned = Op.getOpcode() == ISD::ABDS;
28195   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28196 
28197   // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
28198   if (VT.isScalarInteger()) {
28199     unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u);
28200     MVT WideVT = MVT::getIntegerVT(WideBits);
28201     if (TLI.isTypeLegal(WideVT)) {
28202       // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
28203       // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
28204       unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28205       SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
28206       SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
28207       SDValue Diff = DAG.getNode(ISD::SUB, dl, WideVT, LHS, RHS);
28208       SDValue AbsDiff = DAG.getNode(ISD::ABS, dl, WideVT, Diff);
28209       return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff);
28210     }
28211   }
28212 
28213   // TODO: Move to TargetLowering expandABD().
28214   if (!Subtarget.hasSSE41() &&
28215       ((IsSigned && VT == MVT::v16i8) || VT == MVT::v4i32)) {
28216     SDValue LHS = DAG.getFreeze(Op.getOperand(0));
28217     SDValue RHS = DAG.getFreeze(Op.getOperand(1));
28218     ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
28219     SDValue Cmp = DAG.getSetCC(dl, VT, LHS, RHS, CC);
28220     SDValue Diff0 = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
28221     SDValue Diff1 = DAG.getNode(ISD::SUB, dl, VT, RHS, LHS);
28222     return getBitSelect(dl, VT, Diff0, Diff1, Cmp, DAG);
28223   }
28224 
28225   // Default to expand.
28226   return SDValue();
28227 }
28228 
28229 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
28230                         SelectionDAG &DAG) {
28231   SDLoc dl(Op);
28232   MVT VT = Op.getSimpleValueType();
28233 
28234   // Decompose 256-bit ops into 128-bit ops.
28235   if (VT.is256BitVector() && !Subtarget.hasInt256())
28236     return splitVectorIntBinary(Op, DAG);
28237 
28238   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28239     return splitVectorIntBinary(Op, DAG);
28240 
28241   SDValue A = Op.getOperand(0);
28242   SDValue B = Op.getOperand(1);
28243 
28244   // Lower v16i8/v32i8/v64i8 mul as sign-extension to v8i16/v16i16/v32i16
28245   // vector pairs, multiply and truncate.
28246   if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {
28247     unsigned NumElts = VT.getVectorNumElements();
28248 
28249     if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28250         (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28251       MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
28252       return DAG.getNode(
28253           ISD::TRUNCATE, dl, VT,
28254           DAG.getNode(ISD::MUL, dl, ExVT,
28255                       DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A),
28256                       DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B)));
28257     }
28258 
28259     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28260 
28261     // Extract the lo/hi parts to any extend to i16.
28262     // We're going to mask off the low byte of each result element of the
28263     // pmullw, so it doesn't matter what's in the high byte of each 16-bit
28264     // element.
28265     SDValue Undef = DAG.getUNDEF(VT);
28266     SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
28267     SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
28268 
28269     SDValue BLo, BHi;
28270     if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28271       // If the RHS is a constant, manually unpackl/unpackh.
28272       SmallVector<SDValue, 16> LoOps, HiOps;
28273       for (unsigned i = 0; i != NumElts; i += 16) {
28274         for (unsigned j = 0; j != 8; ++j) {
28275           LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl,
28276                                                MVT::i16));
28277           HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl,
28278                                                MVT::i16));
28279         }
28280       }
28281 
28282       BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28283       BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28284     } else {
28285       BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));
28286       BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));
28287     }
28288 
28289     // Multiply, mask the lower 8bits of the lo/hi results and pack.
28290     SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
28291     SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
28292     return getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28293   }
28294 
28295   // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
28296   if (VT == MVT::v4i32) {
28297     assert(Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&
28298            "Should not custom lower when pmulld is available!");
28299 
28300     // Extract the odd parts.
28301     static const int UnpackMask[] = { 1, -1, 3, -1 };
28302     SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
28303     SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
28304 
28305     // Multiply the even parts.
28306     SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28307                                 DAG.getBitcast(MVT::v2i64, A),
28308                                 DAG.getBitcast(MVT::v2i64, B));
28309     // Now multiply odd parts.
28310     SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28311                                DAG.getBitcast(MVT::v2i64, Aodds),
28312                                DAG.getBitcast(MVT::v2i64, Bodds));
28313 
28314     Evens = DAG.getBitcast(VT, Evens);
28315     Odds = DAG.getBitcast(VT, Odds);
28316 
28317     // Merge the two vectors back together with a shuffle. This expands into 2
28318     // shuffles.
28319     static const int ShufMask[] = { 0, 4, 2, 6 };
28320     return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
28321   }
28322 
28323   assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
28324          "Only know how to lower V2I64/V4I64/V8I64 multiply");
28325   assert(!Subtarget.hasDQI() && "DQI should use MULLQ");
28326 
28327   //  Ahi = psrlqi(a, 32);
28328   //  Bhi = psrlqi(b, 32);
28329   //
28330   //  AloBlo = pmuludq(a, b);
28331   //  AloBhi = pmuludq(a, Bhi);
28332   //  AhiBlo = pmuludq(Ahi, b);
28333   //
28334   //  Hi = psllqi(AloBhi + AhiBlo, 32);
28335   //  return AloBlo + Hi;
28336   KnownBits AKnown = DAG.computeKnownBits(A);
28337   KnownBits BKnown = DAG.computeKnownBits(B);
28338 
28339   APInt LowerBitsMask = APInt::getLowBitsSet(64, 32);
28340   bool ALoIsZero = LowerBitsMask.isSubsetOf(AKnown.Zero);
28341   bool BLoIsZero = LowerBitsMask.isSubsetOf(BKnown.Zero);
28342 
28343   APInt UpperBitsMask = APInt::getHighBitsSet(64, 32);
28344   bool AHiIsZero = UpperBitsMask.isSubsetOf(AKnown.Zero);
28345   bool BHiIsZero = UpperBitsMask.isSubsetOf(BKnown.Zero);
28346 
28347   SDValue Zero = DAG.getConstant(0, dl, VT);
28348 
28349   // Only multiply lo/hi halves that aren't known to be zero.
28350   SDValue AloBlo = Zero;
28351   if (!ALoIsZero && !BLoIsZero)
28352     AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
28353 
28354   SDValue AloBhi = Zero;
28355   if (!ALoIsZero && !BHiIsZero) {
28356     SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
28357     AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
28358   }
28359 
28360   SDValue AhiBlo = Zero;
28361   if (!AHiIsZero && !BLoIsZero) {
28362     SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
28363     AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
28364   }
28365 
28366   SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
28367   Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
28368 
28369   return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);
28370 }
28371 
28372 static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
28373                                      MVT VT, bool IsSigned,
28374                                      const X86Subtarget &Subtarget,
28375                                      SelectionDAG &DAG,
28376                                      SDValue *Low = nullptr) {
28377   unsigned NumElts = VT.getVectorNumElements();
28378 
28379   // For vXi8 we will unpack the low and high half of each 128 bit lane to widen
28380   // to a vXi16 type. Do the multiplies, shift the results and pack the half
28381   // lane results back together.
28382 
28383   // We'll take different approaches for signed and unsigned.
28384   // For unsigned we'll use punpcklbw/punpckhbw to put zero extend the bytes
28385   // and use pmullw to calculate the full 16-bit product.
28386   // For signed we'll use punpcklbw/punpckbw to extend the bytes to words and
28387   // shift them left into the upper byte of each word. This allows us to use
28388   // pmulhw to calculate the full 16-bit product. This trick means we don't
28389   // need to sign extend the bytes to use pmullw.
28390 
28391   MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28392   SDValue Zero = DAG.getConstant(0, dl, VT);
28393 
28394   SDValue ALo, AHi;
28395   if (IsSigned) {
28396     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
28397     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
28398   } else {
28399     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
28400     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
28401   }
28402 
28403   SDValue BLo, BHi;
28404   if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28405     // If the RHS is a constant, manually unpackl/unpackh and extend.
28406     SmallVector<SDValue, 16> LoOps, HiOps;
28407     for (unsigned i = 0; i != NumElts; i += 16) {
28408       for (unsigned j = 0; j != 8; ++j) {
28409         SDValue LoOp = B.getOperand(i + j);
28410         SDValue HiOp = B.getOperand(i + j + 8);
28411 
28412         if (IsSigned) {
28413           LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
28414           HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
28415           LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
28416                              DAG.getConstant(8, dl, MVT::i16));
28417           HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
28418                              DAG.getConstant(8, dl, MVT::i16));
28419         } else {
28420           LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
28421           HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
28422         }
28423 
28424         LoOps.push_back(LoOp);
28425         HiOps.push_back(HiOp);
28426       }
28427     }
28428 
28429     BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28430     BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28431   } else if (IsSigned) {
28432     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
28433     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
28434   } else {
28435     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
28436     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
28437   }
28438 
28439   // Multiply, lshr the upper 8bits to the lower 8bits of the lo/hi results and
28440   // pack back to vXi8.
28441   unsigned MulOpc = IsSigned ? ISD::MULHS : ISD::MUL;
28442   SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
28443   SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
28444 
28445   if (Low)
28446     *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28447 
28448   return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
28449 }
28450 
28451 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
28452                          SelectionDAG &DAG) {
28453   SDLoc dl(Op);
28454   MVT VT = Op.getSimpleValueType();
28455   bool IsSigned = Op->getOpcode() == ISD::MULHS;
28456   unsigned NumElts = VT.getVectorNumElements();
28457   SDValue A = Op.getOperand(0);
28458   SDValue B = Op.getOperand(1);
28459 
28460   // Decompose 256-bit ops into 128-bit ops.
28461   if (VT.is256BitVector() && !Subtarget.hasInt256())
28462     return splitVectorIntBinary(Op, DAG);
28463 
28464   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28465     return splitVectorIntBinary(Op, DAG);
28466 
28467   if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {
28468     assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||
28469            (VT == MVT::v8i32 && Subtarget.hasInt256()) ||
28470            (VT == MVT::v16i32 && Subtarget.hasAVX512()));
28471 
28472     // PMULxD operations multiply each even value (starting at 0) of LHS with
28473     // the related value of RHS and produce a widen result.
28474     // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28475     // => <2 x i64> <ae|cg>
28476     //
28477     // In other word, to have all the results, we need to perform two PMULxD:
28478     // 1. one with the even values.
28479     // 2. one with the odd values.
28480     // To achieve #2, with need to place the odd values at an even position.
28481     //
28482     // Place the odd value at an even position (basically, shift all values 1
28483     // step to the left):
28484     const int Mask[] = {1, -1,  3, -1,  5, -1,  7, -1,
28485                         9, -1, 11, -1, 13, -1, 15, -1};
28486     // <a|b|c|d> => <b|undef|d|undef>
28487     SDValue Odd0 =
28488         DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
28489     // <e|f|g|h> => <f|undef|h|undef>
28490     SDValue Odd1 =
28491         DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
28492 
28493     // Emit two multiplies, one for the lower 2 ints and one for the higher 2
28494     // ints.
28495     MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
28496     unsigned Opcode =
28497         (IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ;
28498     // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28499     // => <2 x i64> <ae|cg>
28500     SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28501                                                   DAG.getBitcast(MulVT, A),
28502                                                   DAG.getBitcast(MulVT, B)));
28503     // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
28504     // => <2 x i64> <bf|dh>
28505     SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28506                                                   DAG.getBitcast(MulVT, Odd0),
28507                                                   DAG.getBitcast(MulVT, Odd1)));
28508 
28509     // Shuffle it back into the right order.
28510     SmallVector<int, 16> ShufMask(NumElts);
28511     for (int i = 0; i != (int)NumElts; ++i)
28512       ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
28513 
28514     SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
28515 
28516     // If we have a signed multiply but no PMULDQ fix up the result of an
28517     // unsigned multiply.
28518     if (IsSigned && !Subtarget.hasSSE41()) {
28519       SDValue Zero = DAG.getConstant(0, dl, VT);
28520       SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
28521                                DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);
28522       SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
28523                                DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);
28524 
28525       SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
28526       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);
28527     }
28528 
28529     return Res;
28530   }
28531 
28532   // Only i8 vectors should need custom lowering after this.
28533   assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
28534          (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
28535          "Unsupported vector type");
28536 
28537   // Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
28538   // logical shift down the upper half and pack back to i8.
28539 
28540   // With SSE41 we can use sign/zero extend, but for pre-SSE41 we unpack
28541   // and then ashr/lshr the upper bits down to the lower bits before multiply.
28542 
28543   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28544       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28545     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28546     unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28547     SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28548     SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28549     SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28550     Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28551     return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28552   }
28553 
28554   return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG);
28555 }
28556 
28557 // Custom lowering for SMULO/UMULO.
28558 static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
28559                          SelectionDAG &DAG) {
28560   MVT VT = Op.getSimpleValueType();
28561 
28562   // Scalars defer to LowerXALUO.
28563   if (!VT.isVector())
28564     return LowerXALUO(Op, DAG);
28565 
28566   SDLoc dl(Op);
28567   bool IsSigned = Op->getOpcode() == ISD::SMULO;
28568   SDValue A = Op.getOperand(0);
28569   SDValue B = Op.getOperand(1);
28570   EVT OvfVT = Op->getValueType(1);
28571 
28572   if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) ||
28573       (VT == MVT::v64i8 && !Subtarget.hasBWI())) {
28574     // Extract the LHS Lo/Hi vectors
28575     SDValue LHSLo, LHSHi;
28576     std::tie(LHSLo, LHSHi) = splitVector(A, DAG, dl);
28577 
28578     // Extract the RHS Lo/Hi vectors
28579     SDValue RHSLo, RHSHi;
28580     std::tie(RHSLo, RHSHi) = splitVector(B, DAG, dl);
28581 
28582     EVT LoOvfVT, HiOvfVT;
28583     std::tie(LoOvfVT, HiOvfVT) = DAG.GetSplitDestVTs(OvfVT);
28584     SDVTList LoVTs = DAG.getVTList(LHSLo.getValueType(), LoOvfVT);
28585     SDVTList HiVTs = DAG.getVTList(LHSHi.getValueType(), HiOvfVT);
28586 
28587     // Issue the split operations.
28588     SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo);
28589     SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi);
28590 
28591     // Join the separate data results and the overflow results.
28592     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28593     SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1),
28594                               Hi.getValue(1));
28595 
28596     return DAG.getMergeValues({Res, Ovf}, dl);
28597   }
28598 
28599   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28600   EVT SetccVT =
28601       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28602 
28603   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28604       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28605     unsigned NumElts = VT.getVectorNumElements();
28606     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28607     unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28608     SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28609     SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28610     SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28611 
28612     SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28613 
28614     SDValue Ovf;
28615     if (IsSigned) {
28616       SDValue High, LowSign;
28617       if (OvfVT.getVectorElementType() == MVT::i1 &&
28618           (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28619         // Rather the truncating try to do the compare on vXi16 or vXi32.
28620         // Shift the high down filling with sign bits.
28621         High = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Mul, 8, DAG);
28622         // Fill all 16 bits with the sign bit from the low.
28623         LowSign =
28624             getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExVT, Mul, 8, DAG);
28625         LowSign = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, LowSign,
28626                                              15, DAG);
28627         SetccVT = OvfVT;
28628         if (!Subtarget.hasBWI()) {
28629           // We can't do a vXi16 compare so sign extend to v16i32.
28630           High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High);
28631           LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign);
28632         }
28633       } else {
28634         // Otherwise do the compare at vXi8.
28635         High = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28636         High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28637         LowSign =
28638             DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28639       }
28640 
28641       Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28642     } else {
28643       SDValue High =
28644           getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28645       if (OvfVT.getVectorElementType() == MVT::i1 &&
28646           (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28647         // Rather the truncating try to do the compare on vXi16 or vXi32.
28648         SetccVT = OvfVT;
28649         if (!Subtarget.hasBWI()) {
28650           // We can't do a vXi16 compare so sign extend to v16i32.
28651           High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High);
28652         }
28653       } else {
28654         // Otherwise do the compare at vXi8.
28655         High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28656       }
28657 
28658       Ovf =
28659           DAG.getSetCC(dl, SetccVT, High,
28660                        DAG.getConstant(0, dl, High.getValueType()), ISD::SETNE);
28661     }
28662 
28663     Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28664 
28665     return DAG.getMergeValues({Low, Ovf}, dl);
28666   }
28667 
28668   SDValue Low;
28669   SDValue High =
28670       LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low);
28671 
28672   SDValue Ovf;
28673   if (IsSigned) {
28674     // SMULO overflows if the high bits don't match the sign of the low.
28675     SDValue LowSign =
28676         DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28677     Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28678   } else {
28679     // UMULO overflows if the high bits are non-zero.
28680     Ovf =
28681         DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE);
28682   }
28683 
28684   Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28685 
28686   return DAG.getMergeValues({Low, Ovf}, dl);
28687 }
28688 
28689 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
28690   assert(Subtarget.isTargetWin64() && "Unexpected target");
28691   EVT VT = Op.getValueType();
28692   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
28693          "Unexpected return type for lowering");
28694 
28695   if (isa<ConstantSDNode>(Op->getOperand(1))) {
28696     SmallVector<SDValue> Result;
28697     if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i64, DAG))
28698       return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]);
28699   }
28700 
28701   RTLIB::Libcall LC;
28702   bool isSigned;
28703   switch (Op->getOpcode()) {
28704   default: llvm_unreachable("Unexpected request for libcall!");
28705   case ISD::SDIV:      isSigned = true;  LC = RTLIB::SDIV_I128;    break;
28706   case ISD::UDIV:      isSigned = false; LC = RTLIB::UDIV_I128;    break;
28707   case ISD::SREM:      isSigned = true;  LC = RTLIB::SREM_I128;    break;
28708   case ISD::UREM:      isSigned = false; LC = RTLIB::UREM_I128;    break;
28709   }
28710 
28711   SDLoc dl(Op);
28712   SDValue InChain = DAG.getEntryNode();
28713 
28714   TargetLowering::ArgListTy Args;
28715   TargetLowering::ArgListEntry Entry;
28716   for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
28717     EVT ArgVT = Op->getOperand(i).getValueType();
28718     assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
28719            "Unexpected argument type for lowering");
28720     SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
28721     int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
28722     MachinePointerInfo MPI =
28723         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
28724     Entry.Node = StackPtr;
28725     InChain =
28726         DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
28727     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
28728     Entry.Ty = PointerType::get(ArgTy,0);
28729     Entry.IsSExt = false;
28730     Entry.IsZExt = false;
28731     Args.push_back(Entry);
28732   }
28733 
28734   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
28735                                          getPointerTy(DAG.getDataLayout()));
28736 
28737   TargetLowering::CallLoweringInfo CLI(DAG);
28738   CLI.setDebugLoc(dl)
28739       .setChain(InChain)
28740       .setLibCallee(
28741           getLibcallCallingConv(LC),
28742           static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
28743           std::move(Args))
28744       .setInRegister()
28745       .setSExtResult(isSigned)
28746       .setZExtResult(!isSigned);
28747 
28748   std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
28749   return DAG.getBitcast(VT, CallInfo.first);
28750 }
28751 
28752 SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
28753                                                    SelectionDAG &DAG,
28754                                                    SDValue &Chain) const {
28755   assert(Subtarget.isTargetWin64() && "Unexpected target");
28756   EVT VT = Op.getValueType();
28757   bool IsStrict = Op->isStrictFPOpcode();
28758 
28759   SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
28760   EVT ArgVT = Arg.getValueType();
28761 
28762   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
28763          "Unexpected return type for lowering");
28764 
28765   RTLIB::Libcall LC;
28766   if (Op->getOpcode() == ISD::FP_TO_SINT ||
28767       Op->getOpcode() == ISD::STRICT_FP_TO_SINT)
28768     LC = RTLIB::getFPTOSINT(ArgVT, VT);
28769   else
28770     LC = RTLIB::getFPTOUINT(ArgVT, VT);
28771   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
28772 
28773   SDLoc dl(Op);
28774   MakeLibCallOptions CallOptions;
28775   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
28776 
28777   SDValue Result;
28778   // Expect the i128 argument returned as a v2i64 in xmm0, cast back to the
28779   // expected VT (i128).
28780   std::tie(Result, Chain) =
28781       makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
28782   Result = DAG.getBitcast(VT, Result);
28783   return Result;
28784 }
28785 
28786 SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
28787                                                    SelectionDAG &DAG) const {
28788   assert(Subtarget.isTargetWin64() && "Unexpected target");
28789   EVT VT = Op.getValueType();
28790   bool IsStrict = Op->isStrictFPOpcode();
28791 
28792   SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
28793   EVT ArgVT = Arg.getValueType();
28794 
28795   assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
28796          "Unexpected argument type for lowering");
28797 
28798   RTLIB::Libcall LC;
28799   if (Op->getOpcode() == ISD::SINT_TO_FP ||
28800       Op->getOpcode() == ISD::STRICT_SINT_TO_FP)
28801     LC = RTLIB::getSINTTOFP(ArgVT, VT);
28802   else
28803     LC = RTLIB::getUINTTOFP(ArgVT, VT);
28804   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
28805 
28806   SDLoc dl(Op);
28807   MakeLibCallOptions CallOptions;
28808   SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
28809 
28810   // Pass the i128 argument as an indirect argument on the stack.
28811   SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
28812   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
28813   MachinePointerInfo MPI =
28814       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
28815   Chain = DAG.getStore(Chain, dl, Arg, StackPtr, MPI, Align(16));
28816 
28817   SDValue Result;
28818   std::tie(Result, Chain) =
28819       makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain);
28820   return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
28821 }
28822 
28823 // Return true if the required (according to Opcode) shift-imm form is natively
28824 // supported by the Subtarget
28825 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
28826                                         unsigned Opcode) {
28827   if (!VT.isSimple())
28828     return false;
28829 
28830   if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
28831     return false;
28832 
28833   if (VT.getScalarSizeInBits() < 16)
28834     return false;
28835 
28836   if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
28837       (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))
28838     return true;
28839 
28840   bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) ||
28841                 (VT.is256BitVector() && Subtarget.hasInt256());
28842 
28843   bool AShift = LShift && (Subtarget.hasAVX512() ||
28844                            (VT != MVT::v2i64 && VT != MVT::v4i64));
28845   return (Opcode == ISD::SRA) ? AShift : LShift;
28846 }
28847 
28848 // The shift amount is a variable, but it is the same for all vector lanes.
28849 // These instructions are defined together with shift-immediate.
28850 static
28851 bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget,
28852                                       unsigned Opcode) {
28853   return supportedVectorShiftWithImm(VT, Subtarget, Opcode);
28854 }
28855 
28856 // Return true if the required (according to Opcode) variable-shift form is
28857 // natively supported by the Subtarget
28858 static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget,
28859                                     unsigned Opcode) {
28860   if (!VT.isSimple())
28861     return false;
28862 
28863   if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
28864     return false;
28865 
28866   if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)
28867     return false;
28868 
28869   // vXi16 supported only on AVX-512, BWI
28870   if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())
28871     return false;
28872 
28873   if (Subtarget.hasAVX512() &&
28874       (Subtarget.useAVX512Regs() || !VT.is512BitVector()))
28875     return true;
28876 
28877   bool LShift = VT.is128BitVector() || VT.is256BitVector();
28878   bool AShift = LShift &&  VT != MVT::v2i64 && VT != MVT::v4i64;
28879   return (Opcode == ISD::SRA) ? AShift : LShift;
28880 }
28881 
28882 static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
28883                                            const X86Subtarget &Subtarget) {
28884   MVT VT = Op.getSimpleValueType();
28885   SDLoc dl(Op);
28886   SDValue R = Op.getOperand(0);
28887   SDValue Amt = Op.getOperand(1);
28888   unsigned X86Opc = getTargetVShiftUniformOpcode(Op.getOpcode(), false);
28889 
28890   auto ArithmeticShiftRight64 = [&](uint64_t ShiftAmt) {
28891     assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");
28892     MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
28893     SDValue Ex = DAG.getBitcast(ExVT, R);
28894 
28895     // ashr(R, 63) === cmp_slt(R, 0)
28896     if (ShiftAmt == 63 && Subtarget.hasSSE42()) {
28897       assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&
28898              "Unsupported PCMPGT op");
28899       return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);
28900     }
28901 
28902     if (ShiftAmt >= 32) {
28903       // Splat sign to upper i32 dst, and SRA upper i32 src to lower i32.
28904       SDValue Upper =
28905           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
28906       SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
28907                                                  ShiftAmt - 32, DAG);
28908       if (VT == MVT::v2i64)
28909         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
28910       if (VT == MVT::v4i64)
28911         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
28912                                   {9, 1, 11, 3, 13, 5, 15, 7});
28913     } else {
28914       // SRA upper i32, SRL whole i64 and select lower i32.
28915       SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
28916                                                  ShiftAmt, DAG);
28917       SDValue Lower =
28918           getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
28919       Lower = DAG.getBitcast(ExVT, Lower);
28920       if (VT == MVT::v2i64)
28921         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
28922       if (VT == MVT::v4i64)
28923         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
28924                                   {8, 1, 10, 3, 12, 5, 14, 7});
28925     }
28926     return DAG.getBitcast(VT, Ex);
28927   };
28928 
28929   // Optimize shl/srl/sra with constant shift amount.
28930   APInt APIntShiftAmt;
28931   if (!X86::isConstantSplat(Amt, APIntShiftAmt))
28932     return SDValue();
28933 
28934   // If the shift amount is out of range, return undef.
28935   if (APIntShiftAmt.uge(VT.getScalarSizeInBits()))
28936     return DAG.getUNDEF(VT);
28937 
28938   uint64_t ShiftAmt = APIntShiftAmt.getZExtValue();
28939 
28940   if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) {
28941     // Hardware support for vector shifts is sparse which makes us scalarize the
28942     // vector operations in many cases. Also, on sandybridge ADD is faster than
28943     // shl: (shl V, 1) -> (add (freeze V), (freeze V))
28944     if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
28945       // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
28946       // must be 0). (add undef, undef) however can be any value. To make this
28947       // safe, we must freeze R to ensure that register allocation uses the same
28948       // register for an undefined value. This ensures that the result will
28949       // still be even and preserves the original semantics.
28950       R = DAG.getFreeze(R);
28951       return DAG.getNode(ISD::ADD, dl, VT, R, R);
28952     }
28953 
28954     return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
28955   }
28956 
28957   // i64 SRA needs to be performed as partial shifts.
28958   if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||
28959        (Subtarget.hasInt256() && VT == MVT::v4i64)) &&
28960       Op.getOpcode() == ISD::SRA)
28961     return ArithmeticShiftRight64(ShiftAmt);
28962 
28963   if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||
28964       (Subtarget.hasBWI() && VT == MVT::v64i8)) {
28965     unsigned NumElts = VT.getVectorNumElements();
28966     MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28967 
28968     // Simple i8 add case
28969     if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
28970       // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
28971       // must be 0). (add undef, undef) however can be any value. To make this
28972       // safe, we must freeze R to ensure that register allocation uses the same
28973       // register for an undefined value. This ensures that the result will
28974       // still be even and preserves the original semantics.
28975       R = DAG.getFreeze(R);
28976       return DAG.getNode(ISD::ADD, dl, VT, R, R);
28977     }
28978 
28979     // ashr(R, 7)  === cmp_slt(R, 0)
28980     if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
28981       SDValue Zeros = DAG.getConstant(0, dl, VT);
28982       if (VT.is512BitVector()) {
28983         assert(VT == MVT::v64i8 && "Unexpected element type!");
28984         SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
28985         return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);
28986       }
28987       return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
28988     }
28989 
28990     // XOP can shift v16i8 directly instead of as shift v8i16 + mask.
28991     if (VT == MVT::v16i8 && Subtarget.hasXOP())
28992       return SDValue();
28993 
28994     if (Op.getOpcode() == ISD::SHL) {
28995       // Make a large shift.
28996       SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
28997                                                ShiftAmt, DAG);
28998       SHL = DAG.getBitcast(VT, SHL);
28999       // Zero out the rightmost bits.
29000       APInt Mask = APInt::getHighBitsSet(8, 8 - ShiftAmt);
29001       return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT));
29002     }
29003     if (Op.getOpcode() == ISD::SRL) {
29004       // Make a large shift.
29005       SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
29006                                                ShiftAmt, DAG);
29007       SRL = DAG.getBitcast(VT, SRL);
29008       // Zero out the leftmost bits.
29009       APInt Mask = APInt::getLowBitsSet(8, 8 - ShiftAmt);
29010       return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT));
29011     }
29012     if (Op.getOpcode() == ISD::SRA) {
29013       // ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
29014       SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29015 
29016       SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
29017       Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
29018       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
29019       return Res;
29020     }
29021     llvm_unreachable("Unknown shift opcode.");
29022   }
29023 
29024   return SDValue();
29025 }
29026 
29027 static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
29028                                           const X86Subtarget &Subtarget) {
29029   MVT VT = Op.getSimpleValueType();
29030   SDLoc dl(Op);
29031   SDValue R = Op.getOperand(0);
29032   SDValue Amt = Op.getOperand(1);
29033   unsigned Opcode = Op.getOpcode();
29034   unsigned X86OpcI = getTargetVShiftUniformOpcode(Opcode, false);
29035 
29036   int BaseShAmtIdx = -1;
29037   if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) {
29038     if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode))
29039       return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, BaseShAmtIdx,
29040                                  Subtarget, DAG);
29041 
29042     // vXi8 shifts - shift as v8i16 + mask result.
29043     if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||
29044          (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||
29045          VT == MVT::v64i8) &&
29046         !Subtarget.hasXOP()) {
29047       unsigned NumElts = VT.getVectorNumElements();
29048       MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
29049       if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, Opcode)) {
29050         unsigned LogicalOp = (Opcode == ISD::SHL ? ISD::SHL : ISD::SRL);
29051         unsigned LogicalX86Op = getTargetVShiftUniformOpcode(LogicalOp, false);
29052 
29053         // Create the mask using vXi16 shifts. For shift-rights we need to move
29054         // the upper byte down before splatting the vXi8 mask.
29055         SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
29056         BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
29057                                       BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
29058         if (Opcode != ISD::SHL)
29059           BitMask = getTargetVShiftByConstNode(LogicalX86Op, dl, ExtVT, BitMask,
29060                                                8, DAG);
29061         BitMask = DAG.getBitcast(VT, BitMask);
29062         BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,
29063                                        SmallVector<int, 64>(NumElts, 0));
29064 
29065         SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
29066                                           DAG.getBitcast(ExtVT, R), BaseShAmt,
29067                                           BaseShAmtIdx, Subtarget, DAG);
29068         Res = DAG.getBitcast(VT, Res);
29069         Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);
29070 
29071         if (Opcode == ISD::SRA) {
29072           // ashr(R, Amt) === sub(xor(lshr(R, Amt), SignMask), SignMask)
29073           // SignMask = lshr(SignBit, Amt) - safe to do this with PSRLW.
29074           SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
29075           SignMask =
29076               getTargetVShiftNode(LogicalX86Op, dl, ExtVT, SignMask, BaseShAmt,
29077                                   BaseShAmtIdx, Subtarget, DAG);
29078           SignMask = DAG.getBitcast(VT, SignMask);
29079           Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);
29080           Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);
29081         }
29082         return Res;
29083       }
29084     }
29085   }
29086 
29087   return SDValue();
29088 }
29089 
29090 // Convert a shift/rotate left amount to a multiplication scale factor.
29091 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
29092                                        const X86Subtarget &Subtarget,
29093                                        SelectionDAG &DAG) {
29094   MVT VT = Amt.getSimpleValueType();
29095   if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||
29096         (Subtarget.hasInt256() && VT == MVT::v16i16) ||
29097         (Subtarget.hasAVX512() && VT == MVT::v32i16) ||
29098         (!Subtarget.hasAVX512() && VT == MVT::v16i8) ||
29099         (Subtarget.hasInt256() && VT == MVT::v32i8) ||
29100         (Subtarget.hasBWI() && VT == MVT::v64i8)))
29101     return SDValue();
29102 
29103   MVT SVT = VT.getVectorElementType();
29104   unsigned SVTBits = SVT.getSizeInBits();
29105   unsigned NumElems = VT.getVectorNumElements();
29106 
29107   APInt UndefElts;
29108   SmallVector<APInt> EltBits;
29109   if (getTargetConstantBitsFromNode(Amt, SVTBits, UndefElts, EltBits)) {
29110     APInt One(SVTBits, 1);
29111     SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
29112     for (unsigned I = 0; I != NumElems; ++I) {
29113       if (UndefElts[I] || EltBits[I].uge(SVTBits))
29114         continue;
29115       uint64_t ShAmt = EltBits[I].getZExtValue();
29116       Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT);
29117     }
29118     return DAG.getBuildVector(VT, dl, Elts);
29119   }
29120 
29121   // If the target doesn't support variable shifts, use either FP conversion
29122   // or integer multiplication to avoid shifting each element individually.
29123   if (VT == MVT::v4i32) {
29124     Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
29125     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,
29126                       DAG.getConstant(0x3f800000U, dl, VT));
29127     Amt = DAG.getBitcast(MVT::v4f32, Amt);
29128     return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);
29129   }
29130 
29131   // AVX2 can more effectively perform this as a zext/trunc to/from v8i32.
29132   if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {
29133     SDValue Z = DAG.getConstant(0, dl, VT);
29134     SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
29135     SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
29136     Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG);
29137     Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG);
29138     if (Subtarget.hasSSE41())
29139       return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29140     return getPack(DAG, Subtarget, dl, VT, Lo, Hi);
29141   }
29142 
29143   return SDValue();
29144 }
29145 
29146 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
29147                           SelectionDAG &DAG) {
29148   MVT VT = Op.getSimpleValueType();
29149   SDLoc dl(Op);
29150   SDValue R = Op.getOperand(0);
29151   SDValue Amt = Op.getOperand(1);
29152   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29153   bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29154 
29155   unsigned Opc = Op.getOpcode();
29156   unsigned X86OpcV = getTargetVShiftUniformOpcode(Opc, true);
29157   unsigned X86OpcI = getTargetVShiftUniformOpcode(Opc, false);
29158 
29159   assert(VT.isVector() && "Custom lowering only for vector shifts!");
29160   assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!");
29161 
29162   if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
29163     return V;
29164 
29165   if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
29166     return V;
29167 
29168   if (supportedVectorVarShift(VT, Subtarget, Opc))
29169     return Op;
29170 
29171   // i64 vector arithmetic shift can be emulated with the transform:
29172   // M = lshr(SIGN_MASK, Amt)
29173   // ashr(R, Amt) === sub(xor(lshr(R, Amt), M), M)
29174   if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) ||
29175        (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
29176       Opc == ISD::SRA) {
29177     SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
29178     SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
29179     R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29180     R = DAG.getNode(ISD::XOR, dl, VT, R, M);
29181     R = DAG.getNode(ISD::SUB, dl, VT, R, M);
29182     return R;
29183   }
29184 
29185   // XOP has 128-bit variable logical/arithmetic shifts.
29186   // +ve/-ve Amt = shift left/right.
29187   if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
29188                              VT == MVT::v8i16 || VT == MVT::v16i8)) {
29189     if (Opc == ISD::SRL || Opc == ISD::SRA) {
29190       SDValue Zero = DAG.getConstant(0, dl, VT);
29191       Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt);
29192     }
29193     if (Opc == ISD::SHL || Opc == ISD::SRL)
29194       return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
29195     if (Opc == ISD::SRA)
29196       return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
29197   }
29198 
29199   // 2i64 vector logical shifts can efficiently avoid scalarization - do the
29200   // shifts per-lane and then shuffle the partial results back together.
29201   if (VT == MVT::v2i64 && Opc != ISD::SRA) {
29202     // Splat the shift amounts so the scalar shifts above will catch it.
29203     SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
29204     SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
29205     SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
29206     SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
29207     return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
29208   }
29209 
29210   // If possible, lower this shift as a sequence of two shifts by
29211   // constant plus a BLENDing shuffle instead of scalarizing it.
29212   // Example:
29213   //   (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
29214   //
29215   // Could be rewritten as:
29216   //   (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
29217   //
29218   // The advantage is that the two shifts from the example would be
29219   // lowered as X86ISD::VSRLI nodes in parallel before blending.
29220   if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
29221                       (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29222     SDValue Amt1, Amt2;
29223     unsigned NumElts = VT.getVectorNumElements();
29224     SmallVector<int, 8> ShuffleMask;
29225     for (unsigned i = 0; i != NumElts; ++i) {
29226       SDValue A = Amt->getOperand(i);
29227       if (A.isUndef()) {
29228         ShuffleMask.push_back(SM_SentinelUndef);
29229         continue;
29230       }
29231       if (!Amt1 || Amt1 == A) {
29232         ShuffleMask.push_back(i);
29233         Amt1 = A;
29234         continue;
29235       }
29236       if (!Amt2 || Amt2 == A) {
29237         ShuffleMask.push_back(i + NumElts);
29238         Amt2 = A;
29239         continue;
29240       }
29241       break;
29242     }
29243 
29244     // Only perform this blend if we can perform it without loading a mask.
29245     if (ShuffleMask.size() == NumElts && Amt1 && Amt2 &&
29246         (VT != MVT::v16i16 ||
29247          is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&
29248         (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||
29249          canWidenShuffleElements(ShuffleMask))) {
29250       auto *Cst1 = dyn_cast<ConstantSDNode>(Amt1);
29251       auto *Cst2 = dyn_cast<ConstantSDNode>(Amt2);
29252       if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) &&
29253           Cst2->getAPIntValue().ult(EltSizeInBits)) {
29254         SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29255                                                     Cst1->getZExtValue(), DAG);
29256         SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29257                                                     Cst2->getZExtValue(), DAG);
29258         return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
29259       }
29260     }
29261   }
29262 
29263   // If possible, lower this packed shift into a vector multiply instead of
29264   // expanding it into a sequence of scalar shifts.
29265   // For v32i8 cases, it might be quicker to split/extend to vXi16 shifts.
29266   if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() ||
29267                                                 Subtarget.canExtendTo512BW())))
29268     if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
29269       return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
29270 
29271   // Constant ISD::SRL can be performed efficiently on vXi16 vectors as we
29272   // can replace with ISD::MULHU, creating scale factor from (NumEltBits - Amt).
29273   if (Opc == ISD::SRL && ConstantAmt &&
29274       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29275     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29276     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29277     if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29278       SDValue Zero = DAG.getConstant(0, dl, VT);
29279       SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
29280       SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
29281       return DAG.getSelect(dl, VT, ZAmt, R, Res);
29282     }
29283   }
29284 
29285   // Constant ISD::SRA can be performed efficiently on vXi16 vectors as we
29286   // can replace with ISD::MULHS, creating scale factor from (NumEltBits - Amt).
29287   // TODO: Special case handling for shift by 0/1, really we can afford either
29288   // of these cases in pre-SSE41/XOP/AVX512 but not both.
29289   if (Opc == ISD::SRA && ConstantAmt &&
29290       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
29291       ((Subtarget.hasSSE41() && !Subtarget.hasXOP() &&
29292         !Subtarget.hasAVX512()) ||
29293        DAG.isKnownNeverZero(Amt))) {
29294     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29295     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29296     if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29297       SDValue Amt0 =
29298           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
29299       SDValue Amt1 =
29300           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
29301       SDValue Sra1 =
29302           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
29303       SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
29304       Res = DAG.getSelect(dl, VT, Amt0, R, Res);
29305       return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
29306     }
29307   }
29308 
29309   // v4i32 Non Uniform Shifts.
29310   // If the shift amount is constant we can shift each lane using the SSE2
29311   // immediate shifts, else we need to zero-extend each lane to the lower i64
29312   // and shift using the SSE2 variable shifts.
29313   // The separate results can then be blended together.
29314   if (VT == MVT::v4i32) {
29315     SDValue Amt0, Amt1, Amt2, Amt3;
29316     if (ConstantAmt) {
29317       Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
29318       Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
29319       Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
29320       Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
29321     } else {
29322       // The SSE2 shifts use the lower i64 as the same shift amount for
29323       // all lanes and the upper i64 is ignored. On AVX we're better off
29324       // just zero-extending, but for SSE just duplicating the top 16-bits is
29325       // cheaper and has the same effect for out of range values.
29326       if (Subtarget.hasAVX()) {
29327         SDValue Z = DAG.getConstant(0, dl, VT);
29328         Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
29329         Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
29330         Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
29331         Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
29332       } else {
29333         SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
29334         SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
29335                                              {4, 5, 6, 7, -1, -1, -1, -1});
29336         SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG);
29337         SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG);
29338         Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02);
29339         Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13);
29340         Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02);
29341         Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13);
29342       }
29343     }
29344 
29345     unsigned ShOpc = ConstantAmt ? Opc : X86OpcV;
29346     SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
29347     SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
29348     SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
29349     SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
29350 
29351     // Merge the shifted lane results optimally with/without PBLENDW.
29352     // TODO - ideally shuffle combining would handle this.
29353     if (Subtarget.hasSSE41()) {
29354       SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
29355       SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
29356       return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
29357     }
29358     SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
29359     SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
29360     return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
29361   }
29362 
29363   // It's worth extending once and using the vXi16/vXi32 shifts for smaller
29364   // types, but without AVX512 the extra overheads to get from vXi8 to vXi32
29365   // make the existing SSE solution better.
29366   // NOTE: We honor prefered vector width before promoting to 512-bits.
29367   if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||
29368       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||
29369       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||
29370       (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||
29371       (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {
29372     assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&
29373            "Unexpected vector type");
29374     MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32;
29375     MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());
29376     unsigned ExtOpc = Opc == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
29377     R = DAG.getNode(ExtOpc, dl, ExtVT, R);
29378     Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt);
29379     return DAG.getNode(ISD::TRUNCATE, dl, VT,
29380                        DAG.getNode(Opc, dl, ExtVT, R, Amt));
29381   }
29382 
29383   // Constant ISD::SRA/SRL can be performed efficiently on vXi8 vectors as we
29384   // extend to vXi16 to perform a MUL scale effectively as a MUL_LOHI.
29385   if (ConstantAmt && (Opc == ISD::SRA || Opc == ISD::SRL) &&
29386       (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
29387        (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
29388       !Subtarget.hasXOP()) {
29389     int NumElts = VT.getVectorNumElements();
29390     SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
29391 
29392     // Extend constant shift amount to vXi16 (it doesn't matter if the type
29393     // isn't legal).
29394     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
29395     Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT);
29396     Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt);
29397     Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt);
29398     assert(ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&
29399            "Constant build vector expected");
29400 
29401     if (VT == MVT::v16i8 && Subtarget.hasInt256()) {
29402       bool IsSigned = Opc == ISD::SRA;
29403       R = DAG.getExtOrTrunc(IsSigned, R, dl, ExVT);
29404       R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt);
29405       R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8);
29406       return DAG.getZExtOrTrunc(R, dl, VT);
29407     }
29408 
29409     SmallVector<SDValue, 16> LoAmt, HiAmt;
29410     for (int i = 0; i != NumElts; i += 16) {
29411       for (int j = 0; j != 8; ++j) {
29412         LoAmt.push_back(Amt.getOperand(i + j));
29413         HiAmt.push_back(Amt.getOperand(i + j + 8));
29414       }
29415     }
29416 
29417     MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2);
29418     SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
29419     SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
29420 
29421     SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
29422     SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
29423     LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8);
29424     HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8);
29425     LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA);
29426     HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA);
29427     LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8);
29428     HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8);
29429     return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
29430   }
29431 
29432   if (VT == MVT::v16i8 ||
29433       (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
29434       (VT == MVT::v64i8 && Subtarget.hasBWI())) {
29435     MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
29436 
29437     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
29438       if (VT.is512BitVector()) {
29439         // On AVX512BW targets we make use of the fact that VSELECT lowers
29440         // to a masked blend which selects bytes based just on the sign bit
29441         // extracted to a mask.
29442         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
29443         V0 = DAG.getBitcast(VT, V0);
29444         V1 = DAG.getBitcast(VT, V1);
29445         Sel = DAG.getBitcast(VT, Sel);
29446         Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
29447                            ISD::SETGT);
29448         return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
29449       } else if (Subtarget.hasSSE41()) {
29450         // On SSE41 targets we can use PBLENDVB which selects bytes based just
29451         // on the sign bit.
29452         V0 = DAG.getBitcast(VT, V0);
29453         V1 = DAG.getBitcast(VT, V1);
29454         Sel = DAG.getBitcast(VT, Sel);
29455         return DAG.getBitcast(SelVT,
29456                               DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1));
29457       }
29458       // On pre-SSE41 targets we test for the sign bit by comparing to
29459       // zero - a negative value will set all bits of the lanes to true
29460       // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
29461       SDValue Z = DAG.getConstant(0, dl, SelVT);
29462       SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
29463       return DAG.getSelect(dl, SelVT, C, V0, V1);
29464     };
29465 
29466     // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
29467     // We can safely do this using i16 shifts as we're only interested in
29468     // the 3 lower bits of each byte.
29469     Amt = DAG.getBitcast(ExtVT, Amt);
29470     Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG);
29471     Amt = DAG.getBitcast(VT, Amt);
29472 
29473     if (Opc == ISD::SHL || Opc == ISD::SRL) {
29474       // r = VSELECT(r, shift(r, 4), a);
29475       SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
29476       R = SignBitSelect(VT, Amt, M, R);
29477 
29478       // a += a
29479       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29480 
29481       // r = VSELECT(r, shift(r, 2), a);
29482       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
29483       R = SignBitSelect(VT, Amt, M, R);
29484 
29485       // a += a
29486       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29487 
29488       // return VSELECT(r, shift(r, 1), a);
29489       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
29490       R = SignBitSelect(VT, Amt, M, R);
29491       return R;
29492     }
29493 
29494     if (Opc == ISD::SRA) {
29495       // For SRA we need to unpack each byte to the higher byte of a i16 vector
29496       // so we can correctly sign extend. We don't care what happens to the
29497       // lower byte.
29498       SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29499       SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29500       SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
29501       SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
29502       ALo = DAG.getBitcast(ExtVT, ALo);
29503       AHi = DAG.getBitcast(ExtVT, AHi);
29504       RLo = DAG.getBitcast(ExtVT, RLo);
29505       RHi = DAG.getBitcast(ExtVT, RHi);
29506 
29507       // r = VSELECT(r, shift(r, 4), a);
29508       SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
29509       SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
29510       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29511       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29512 
29513       // a += a
29514       ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29515       AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29516 
29517       // r = VSELECT(r, shift(r, 2), a);
29518       MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG);
29519       MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG);
29520       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29521       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29522 
29523       // a += a
29524       ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29525       AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29526 
29527       // r = VSELECT(r, shift(r, 1), a);
29528       MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG);
29529       MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG);
29530       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29531       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29532 
29533       // Logical shift the result back to the lower byte, leaving a zero upper
29534       // byte meaning that we can safely pack with PACKUSWB.
29535       RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG);
29536       RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG);
29537       return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
29538     }
29539   }
29540 
29541   if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {
29542     MVT ExtVT = MVT::v8i32;
29543     SDValue Z = DAG.getConstant(0, dl, VT);
29544     SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
29545     SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
29546     SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
29547     SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
29548     ALo = DAG.getBitcast(ExtVT, ALo);
29549     AHi = DAG.getBitcast(ExtVT, AHi);
29550     RLo = DAG.getBitcast(ExtVT, RLo);
29551     RHi = DAG.getBitcast(ExtVT, RHi);
29552     SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
29553     SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
29554     Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG);
29555     Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG);
29556     return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29557   }
29558 
29559   if (VT == MVT::v8i16) {
29560     // If we have a constant shift amount, the non-SSE41 path is best as
29561     // avoiding bitcasts make it easier to constant fold and reduce to PBLENDW.
29562     bool UseSSE41 = Subtarget.hasSSE41() &&
29563                     !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29564 
29565     auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
29566       // On SSE41 targets we can use PBLENDVB which selects bytes based just on
29567       // the sign bit.
29568       if (UseSSE41) {
29569         MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
29570         V0 = DAG.getBitcast(ExtVT, V0);
29571         V1 = DAG.getBitcast(ExtVT, V1);
29572         Sel = DAG.getBitcast(ExtVT, Sel);
29573         return DAG.getBitcast(
29574             VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1));
29575       }
29576       // On pre-SSE41 targets we splat the sign bit - a negative value will
29577       // set all bits of the lanes to true and VSELECT uses that in
29578       // its OR(AND(V0,C),AND(V1,~C)) lowering.
29579       SDValue C =
29580           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
29581       return DAG.getSelect(dl, VT, C, V0, V1);
29582     };
29583 
29584     // Turn 'a' into a mask suitable for VSELECT: a = a << 12;
29585     if (UseSSE41) {
29586       // On SSE41 targets we need to replicate the shift mask in both
29587       // bytes for PBLENDVB.
29588       Amt = DAG.getNode(
29589           ISD::OR, dl, VT,
29590           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
29591           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
29592     } else {
29593       Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
29594     }
29595 
29596     // r = VSELECT(r, shift(r, 8), a);
29597     SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
29598     R = SignBitSelect(Amt, M, R);
29599 
29600     // a += a
29601     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29602 
29603     // r = VSELECT(r, shift(r, 4), a);
29604     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
29605     R = SignBitSelect(Amt, M, R);
29606 
29607     // a += a
29608     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29609 
29610     // r = VSELECT(r, shift(r, 2), a);
29611     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
29612     R = SignBitSelect(Amt, M, R);
29613 
29614     // a += a
29615     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29616 
29617     // return VSELECT(r, shift(r, 1), a);
29618     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
29619     R = SignBitSelect(Amt, M, R);
29620     return R;
29621   }
29622 
29623   // Decompose 256-bit shifts into 128-bit shifts.
29624   if (VT.is256BitVector())
29625     return splitVectorIntBinary(Op, DAG);
29626 
29627   if (VT == MVT::v32i16 || VT == MVT::v64i8)
29628     return splitVectorIntBinary(Op, DAG);
29629 
29630   return SDValue();
29631 }
29632 
29633 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
29634                                 SelectionDAG &DAG) {
29635   MVT VT = Op.getSimpleValueType();
29636   assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
29637          "Unexpected funnel shift opcode!");
29638 
29639   SDLoc DL(Op);
29640   SDValue Op0 = Op.getOperand(0);
29641   SDValue Op1 = Op.getOperand(1);
29642   SDValue Amt = Op.getOperand(2);
29643   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29644   bool IsFSHR = Op.getOpcode() == ISD::FSHR;
29645 
29646   if (VT.isVector()) {
29647     APInt APIntShiftAmt;
29648     bool IsCstSplat = X86::isConstantSplat(Amt, APIntShiftAmt);
29649 
29650     if (Subtarget.hasVBMI2() && EltSizeInBits > 8) {
29651       if (IsFSHR)
29652         std::swap(Op0, Op1);
29653 
29654       if (IsCstSplat) {
29655         uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
29656         SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);
29657         return getAVX512Node(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,
29658                              {Op0, Op1, Imm}, DAG, Subtarget);
29659       }
29660       return getAVX512Node(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
29661                            {Op0, Op1, Amt}, DAG, Subtarget);
29662     }
29663     assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 ||
29664             VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 ||
29665             VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) &&
29666            "Unexpected funnel shift type!");
29667 
29668     // fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw.
29669     // fshr(x,y,z) -> unpack(y,x) >> (z & (bw-1))).
29670     if (IsCstSplat) {
29671       // TODO: Can't use generic expansion as UNDEF amt elements can be
29672       // converted to other values when folded to shift amounts, losing the
29673       // splat.
29674       uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
29675       uint64_t ShXAmt = IsFSHR ? (EltSizeInBits - ShiftAmt) : ShiftAmt;
29676       uint64_t ShYAmt = IsFSHR ? ShiftAmt : (EltSizeInBits - ShiftAmt);
29677       SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0,
29678                                 DAG.getShiftAmountConstant(ShXAmt, VT, DL));
29679       SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1,
29680                                 DAG.getShiftAmountConstant(ShYAmt, VT, DL));
29681       return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
29682     }
29683 
29684     SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
29685     SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
29686     bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode());
29687 
29688     // Constant vXi16 funnel shifts can be efficiently handled by default.
29689     if (IsCst && EltSizeInBits == 16)
29690       return SDValue();
29691 
29692     unsigned ShiftOpc = IsFSHR ? ISD::SRL : ISD::SHL;
29693     unsigned NumElts = VT.getVectorNumElements();
29694     MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
29695     MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
29696 
29697     // Split 256-bit integers on XOP/pre-AVX2 targets.
29698     // Split 512-bit integers on non 512-bit BWI targets.
29699     if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) ||
29700                                  !Subtarget.hasAVX2())) ||
29701         (VT.is512BitVector() && !Subtarget.useBWIRegs() &&
29702          EltSizeInBits < 32)) {
29703       // Pre-mask the amount modulo using the wider vector.
29704       Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod);
29705       return splitVectorOp(Op, DAG);
29706     }
29707 
29708     // Attempt to fold scalar shift as unpack(y,x) << zext(splat(z))
29709     if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) {
29710       int ScalarAmtIdx = -1;
29711       if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) {
29712         // Uniform vXi16 funnel shifts can be efficiently handled by default.
29713         if (EltSizeInBits == 16)
29714           return SDValue();
29715 
29716         SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
29717         SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
29718         Lo = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Lo, ScalarAmt,
29719                                  ScalarAmtIdx, Subtarget, DAG);
29720         Hi = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Hi, ScalarAmt,
29721                                  ScalarAmtIdx, Subtarget, DAG);
29722         return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
29723       }
29724     }
29725 
29726     MVT WideSVT = MVT::getIntegerVT(
29727         std::min<unsigned>(EltSizeInBits * 2, Subtarget.hasBWI() ? 16 : 32));
29728     MVT WideVT = MVT::getVectorVT(WideSVT, NumElts);
29729 
29730     // If per-element shifts are legal, fallback to generic expansion.
29731     if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP())
29732       return SDValue();
29733 
29734     // Attempt to fold as:
29735     // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
29736     // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
29737     if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
29738         supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
29739       Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0);
29740       Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1);
29741       AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
29742       Op0 = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, Op0,
29743                                        EltSizeInBits, DAG);
29744       SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1);
29745       Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod);
29746       if (!IsFSHR)
29747         Res = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, Res,
29748                                          EltSizeInBits, DAG);
29749       return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
29750     }
29751 
29752     // Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z)
29753     if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits <= 16) ||
29754         supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) {
29755       SDValue Z = DAG.getConstant(0, DL, VT);
29756       SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
29757       SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
29758       SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
29759       SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
29760       SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
29761       SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
29762       return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
29763     }
29764 
29765     // Fallback to generic expansion.
29766     return SDValue();
29767   }
29768   assert(
29769       (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
29770       "Unexpected funnel shift type!");
29771 
29772   // Expand slow SHLD/SHRD cases if we are not optimizing for size.
29773   bool OptForSize = DAG.shouldOptForSize();
29774   bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow();
29775 
29776   // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
29777   // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
29778   if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&
29779       !isa<ConstantSDNode>(Amt)) {
29780     SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
29781     SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
29782     Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
29783     Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
29784     Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
29785     SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
29786     Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
29787     if (IsFSHR) {
29788       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
29789     } else {
29790       Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
29791       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
29792     }
29793     return DAG.getZExtOrTrunc(Res, DL, VT);
29794   }
29795 
29796   if (VT == MVT::i8 || ExpandFunnel)
29797     return SDValue();
29798 
29799   // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
29800   if (VT == MVT::i16) {
29801     Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
29802                       DAG.getConstant(15, DL, Amt.getValueType()));
29803     unsigned FSHOp = (IsFSHR ? X86ISD::FSHR : X86ISD::FSHL);
29804     return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
29805   }
29806 
29807   return Op;
29808 }
29809 
29810 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
29811                            SelectionDAG &DAG) {
29812   MVT VT = Op.getSimpleValueType();
29813   assert(VT.isVector() && "Custom lowering only for vector rotates!");
29814 
29815   SDLoc DL(Op);
29816   SDValue R = Op.getOperand(0);
29817   SDValue Amt = Op.getOperand(1);
29818   unsigned Opcode = Op.getOpcode();
29819   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29820   int NumElts = VT.getVectorNumElements();
29821   bool IsROTL = Opcode == ISD::ROTL;
29822 
29823   // Check for constant splat rotation amount.
29824   APInt CstSplatValue;
29825   bool IsCstSplat = X86::isConstantSplat(Amt, CstSplatValue);
29826 
29827   // Check for splat rotate by zero.
29828   if (IsCstSplat && CstSplatValue.urem(EltSizeInBits) == 0)
29829     return R;
29830 
29831   // AVX512 implicitly uses modulo rotation amounts.
29832   if (Subtarget.hasAVX512() && 32 <= EltSizeInBits) {
29833     // Attempt to rotate by immediate.
29834     if (IsCstSplat) {
29835       unsigned RotOpc = IsROTL ? X86ISD::VROTLI : X86ISD::VROTRI;
29836       uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29837       return DAG.getNode(RotOpc, DL, VT, R,
29838                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));
29839     }
29840 
29841     // Else, fall-back on VPROLV/VPRORV.
29842     return Op;
29843   }
29844 
29845   // AVX512 VBMI2 vXi16 - lower to funnel shifts.
29846   if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) {
29847     unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
29848     return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
29849   }
29850 
29851   SDValue Z = DAG.getConstant(0, DL, VT);
29852 
29853   if (!IsROTL) {
29854     // If the ISD::ROTR amount is constant, we're always better converting to
29855     // ISD::ROTL.
29856     if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
29857       return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt);
29858 
29859     // XOP targets always prefers ISD::ROTL.
29860     if (Subtarget.hasXOP())
29861       return DAG.getNode(ISD::ROTL, DL, VT, R,
29862                          DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
29863   }
29864 
29865   // Split 256-bit integers on XOP/pre-AVX2 targets.
29866   if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
29867     return splitVectorIntBinary(Op, DAG);
29868 
29869   // XOP has 128-bit vector variable + immediate rotates.
29870   // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
29871   // XOP implicitly uses modulo rotation amounts.
29872   if (Subtarget.hasXOP()) {
29873     assert(IsROTL && "Only ROTL expected");
29874     assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");
29875 
29876     // Attempt to rotate by immediate.
29877     if (IsCstSplat) {
29878       uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29879       return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
29880                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));
29881     }
29882 
29883     // Use general rotate by variable (per-element).
29884     return Op;
29885   }
29886 
29887   // Rotate by an uniform constant - expand back to shifts.
29888   // TODO: Can't use generic expansion as UNDEF amt elements can be converted
29889   // to other values when folded to shift amounts, losing the splat.
29890   if (IsCstSplat) {
29891     uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29892     uint64_t ShlAmt = IsROTL ? RotAmt : (EltSizeInBits - RotAmt);
29893     uint64_t SrlAmt = IsROTL ? (EltSizeInBits - RotAmt) : RotAmt;
29894     SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R,
29895                               DAG.getShiftAmountConstant(ShlAmt, VT, DL));
29896     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R,
29897                               DAG.getShiftAmountConstant(SrlAmt, VT, DL));
29898     return DAG.getNode(ISD::OR, DL, VT, Shl, Srl);
29899   }
29900 
29901   // Split 512-bit integers on non 512-bit BWI targets.
29902   if (VT.is512BitVector() && !Subtarget.useBWIRegs())
29903     return splitVectorIntBinary(Op, DAG);
29904 
29905   assert(
29906       (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
29907        ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&
29908         Subtarget.hasAVX2()) ||
29909        ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) &&
29910       "Only vXi32/vXi16/vXi8 vector rotates supported");
29911 
29912   MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
29913   MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
29914 
29915   SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
29916   SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
29917 
29918   // Attempt to fold as unpack(x,x) << zext(splat(y)):
29919   // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
29920   // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
29921   if (EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) {
29922     int BaseRotAmtIdx = -1;
29923     if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) {
29924       if (EltSizeInBits == 16 && Subtarget.hasSSE41()) {
29925         unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
29926         return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
29927       }
29928       unsigned ShiftX86Opc = IsROTL ? X86ISD::VSHLI : X86ISD::VSRLI;
29929       SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
29930       SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
29931       Lo = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Lo, BaseRotAmt,
29932                                BaseRotAmtIdx, Subtarget, DAG);
29933       Hi = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Hi, BaseRotAmt,
29934                                BaseRotAmtIdx, Subtarget, DAG);
29935       return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
29936     }
29937   }
29938 
29939   bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29940   unsigned ShiftOpc = IsROTL ? ISD::SHL : ISD::SRL;
29941 
29942   // Attempt to fold as unpack(x,x) << zext(y):
29943   // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
29944   // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
29945   // Const vXi16/vXi32 are excluded in favor of MUL-based lowering.
29946   if (!(ConstantAmt && EltSizeInBits != 8) &&
29947       !supportedVectorVarShift(VT, Subtarget, ShiftOpc) &&
29948       (ConstantAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc))) {
29949     SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
29950     SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
29951     SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
29952     SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
29953     SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
29954     SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
29955     return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
29956   }
29957 
29958   // v16i8/v32i8/v64i8: Split rotation into rot4/rot2/rot1 stages and select by
29959   // the amount bit.
29960   // TODO: We're doing nothing here that we couldn't do for funnel shifts.
29961   if (EltSizeInBits == 8) {
29962     MVT WideVT =
29963         MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts);
29964 
29965     // Attempt to fold as:
29966     // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw.
29967     // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))).
29968     if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
29969         supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
29970       // If we're rotating by constant, just use default promotion.
29971       if (ConstantAmt)
29972         return SDValue();
29973       // See if we can perform this by widening to vXi16 or vXi32.
29974       R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R);
29975       R = DAG.getNode(
29976           ISD::OR, DL, WideVT, R,
29977           getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG));
29978       Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
29979       R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt);
29980       if (IsROTL)
29981         R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG);
29982       return DAG.getNode(ISD::TRUNCATE, DL, VT, R);
29983     }
29984 
29985     // We don't need ModuloAmt here as we just peek at individual bits.
29986     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
29987       if (Subtarget.hasSSE41()) {
29988         // On SSE41 targets we can use PBLENDVB which selects bytes based just
29989         // on the sign bit.
29990         V0 = DAG.getBitcast(VT, V0);
29991         V1 = DAG.getBitcast(VT, V1);
29992         Sel = DAG.getBitcast(VT, Sel);
29993         return DAG.getBitcast(SelVT,
29994                               DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
29995       }
29996       // On pre-SSE41 targets we test for the sign bit by comparing to
29997       // zero - a negative value will set all bits of the lanes to true
29998       // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
29999       SDValue Z = DAG.getConstant(0, DL, SelVT);
30000       SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
30001       return DAG.getSelect(DL, SelVT, C, V0, V1);
30002     };
30003 
30004     // ISD::ROTR is currently only profitable on AVX512 targets with VPTERNLOG.
30005     if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) {
30006       Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30007       IsROTL = true;
30008     }
30009 
30010     unsigned ShiftLHS = IsROTL ? ISD::SHL : ISD::SRL;
30011     unsigned ShiftRHS = IsROTL ? ISD::SRL : ISD::SHL;
30012 
30013     // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
30014     // We can safely do this using i16 shifts as we're only interested in
30015     // the 3 lower bits of each byte.
30016     Amt = DAG.getBitcast(ExtVT, Amt);
30017     Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
30018     Amt = DAG.getBitcast(VT, Amt);
30019 
30020     // r = VSELECT(r, rot(r, 4), a);
30021     SDValue M;
30022     M = DAG.getNode(
30023         ISD::OR, DL, VT,
30024         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)),
30025         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT)));
30026     R = SignBitSelect(VT, Amt, M, R);
30027 
30028     // a += a
30029     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30030 
30031     // r = VSELECT(r, rot(r, 2), a);
30032     M = DAG.getNode(
30033         ISD::OR, DL, VT,
30034         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)),
30035         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT)));
30036     R = SignBitSelect(VT, Amt, M, R);
30037 
30038     // a += a
30039     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30040 
30041     // return VSELECT(r, rot(r, 1), a);
30042     M = DAG.getNode(
30043         ISD::OR, DL, VT,
30044         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)),
30045         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT)));
30046     return SignBitSelect(VT, Amt, M, R);
30047   }
30048 
30049   bool IsSplatAmt = DAG.isSplatValue(Amt);
30050   bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) &&
30051                         supportedVectorVarShift(VT, Subtarget, ISD::SRL);
30052 
30053   // Fallback for splats + all supported variable shifts.
30054   // Fallback for non-constants AVX2 vXi16 as well.
30055   if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) {
30056     Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30057     SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
30058     AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
30059     SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
30060     SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
30061     return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
30062   }
30063 
30064   // Everything below assumes ISD::ROTL.
30065   if (!IsROTL) {
30066     Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30067     IsROTL = true;
30068   }
30069 
30070   // ISD::ROT* uses modulo rotate amounts.
30071   Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30072 
30073   assert(IsROTL && "Only ROTL supported");
30074 
30075   // As with shifts, attempt to convert the rotation amount to a multiplication
30076   // factor, fallback to general expansion.
30077   SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
30078   if (!Scale)
30079     return SDValue();
30080 
30081   // v8i16/v16i16: perform unsigned multiply hi/lo and OR the results.
30082   if (EltSizeInBits == 16) {
30083     SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
30084     SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
30085     return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
30086   }
30087 
30088   // v4i32: make use of the PMULUDQ instruction to multiply 2 lanes of v4i32
30089   // to v2i64 results at a time. The upper 32-bits contain the wrapped bits
30090   // that can then be OR'd with the lower 32-bits.
30091   assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected");
30092   static const int OddMask[] = {1, -1, 3, -1};
30093   SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
30094   SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
30095 
30096   SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30097                               DAG.getBitcast(MVT::v2i64, R),
30098                               DAG.getBitcast(MVT::v2i64, Scale));
30099   SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30100                               DAG.getBitcast(MVT::v2i64, R13),
30101                               DAG.getBitcast(MVT::v2i64, Scale13));
30102   Res02 = DAG.getBitcast(VT, Res02);
30103   Res13 = DAG.getBitcast(VT, Res13);
30104 
30105   return DAG.getNode(ISD::OR, DL, VT,
30106                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
30107                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
30108 }
30109 
30110 /// Returns true if the operand type is exactly twice the native width, and
30111 /// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
30112 /// Used to know whether to use cmpxchg8/16b when expanding atomic operations
30113 /// (otherwise we leave them alone to become __sync_fetch_and_... calls).
30114 bool X86TargetLowering::needsCmpXchgNb(Type *MemType) const {
30115   unsigned OpWidth = MemType->getPrimitiveSizeInBits();
30116 
30117   if (OpWidth == 64)
30118     return Subtarget.canUseCMPXCHG8B() && !Subtarget.is64Bit();
30119   if (OpWidth == 128)
30120     return Subtarget.canUseCMPXCHG16B();
30121 
30122   return false;
30123 }
30124 
30125 TargetLoweringBase::AtomicExpansionKind
30126 X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
30127   Type *MemType = SI->getValueOperand()->getType();
30128 
30129   bool NoImplicitFloatOps =
30130       SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
30131   if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30132       !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
30133       (Subtarget.hasSSE1() || Subtarget.hasX87()))
30134     return AtomicExpansionKind::None;
30135 
30136   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::Expand
30137                                  : AtomicExpansionKind::None;
30138 }
30139 
30140 // Note: this turns large loads into lock cmpxchg8b/16b.
30141 // TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
30142 TargetLowering::AtomicExpansionKind
30143 X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
30144   Type *MemType = LI->getType();
30145 
30146   // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
30147   // can use movq to do the load. If we have X87 we can load into an 80-bit
30148   // X87 register and store it to a stack temporary.
30149   bool NoImplicitFloatOps =
30150       LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
30151   if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30152       !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
30153       (Subtarget.hasSSE1() || Subtarget.hasX87()))
30154     return AtomicExpansionKind::None;
30155 
30156   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30157                                  : AtomicExpansionKind::None;
30158 }
30159 
30160 enum BitTestKind : unsigned {
30161   UndefBit,
30162   ConstantBit,
30163   NotConstantBit,
30164   ShiftBit,
30165   NotShiftBit
30166 };
30167 
30168 static std::pair<Value *, BitTestKind> FindSingleBitChange(Value *V) {
30169   using namespace llvm::PatternMatch;
30170   BitTestKind BTK = UndefBit;
30171   auto *C = dyn_cast<ConstantInt>(V);
30172   if (C) {
30173     // Check if V is a power of 2 or NOT power of 2.
30174     if (isPowerOf2_64(C->getZExtValue()))
30175       BTK = ConstantBit;
30176     else if (isPowerOf2_64((~C->getValue()).getZExtValue()))
30177       BTK = NotConstantBit;
30178     return {V, BTK};
30179   }
30180 
30181   // Check if V is some power of 2 pattern known to be non-zero
30182   auto *I = dyn_cast<Instruction>(V);
30183   if (I) {
30184     bool Not = false;
30185     // Check if we have a NOT
30186     Value *PeekI;
30187     if (match(I, m_c_Xor(m_Value(PeekI), m_AllOnes())) ||
30188         match(I, m_Sub(m_AllOnes(), m_Value(PeekI)))) {
30189       Not = true;
30190       I = dyn_cast<Instruction>(PeekI);
30191 
30192       // If I is constant, it will fold and we can evaluate later. If its an
30193       // argument or something of that nature, we can't analyze.
30194       if (I == nullptr)
30195         return {nullptr, UndefBit};
30196     }
30197     // We can only use 1 << X without more sophisticated analysis. C << X where
30198     // C is a power of 2 but not 1 can result in zero which cannot be translated
30199     // to bittest. Likewise any C >> X (either arith or logical) can be zero.
30200     if (I->getOpcode() == Instruction::Shl) {
30201       // Todo(1): The cmpxchg case is pretty costly so matching `BLSI(X)`, `X &
30202       // -X` and some other provable power of 2 patterns that we can use CTZ on
30203       // may be profitable.
30204       // Todo(2): It may be possible in some cases to prove that Shl(C, X) is
30205       // non-zero even where C != 1. Likewise LShr(C, X) and AShr(C, X) may also
30206       // be provably a non-zero power of 2.
30207       // Todo(3): ROTL and ROTR patterns on a power of 2 C should also be
30208       // transformable to bittest.
30209       auto *ShiftVal = dyn_cast<ConstantInt>(I->getOperand(0));
30210       if (!ShiftVal)
30211         return {nullptr, UndefBit};
30212       if (ShiftVal->equalsInt(1))
30213         BTK = Not ? NotShiftBit : ShiftBit;
30214 
30215       if (BTK == UndefBit)
30216         return {nullptr, UndefBit};
30217 
30218       Value *BitV = I->getOperand(1);
30219 
30220       Value *AndOp;
30221       const APInt *AndC;
30222       if (match(BitV, m_c_And(m_Value(AndOp), m_APInt(AndC)))) {
30223         // Read past a shiftmask instruction to find count
30224         if (*AndC == (I->getType()->getPrimitiveSizeInBits() - 1))
30225           BitV = AndOp;
30226       }
30227       return {BitV, BTK};
30228     }
30229   }
30230   return {nullptr, UndefBit};
30231 }
30232 
30233 TargetLowering::AtomicExpansionKind
30234 X86TargetLowering::shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const {
30235   using namespace llvm::PatternMatch;
30236   // If the atomicrmw's result isn't actually used, we can just add a "lock"
30237   // prefix to a normal instruction for these operations.
30238   if (AI->use_empty())
30239     return AtomicExpansionKind::None;
30240 
30241   if (AI->getOperation() == AtomicRMWInst::Xor) {
30242     // A ^ SignBit -> A + SignBit. This allows us to use `xadd` which is
30243     // preferable to both `cmpxchg` and `btc`.
30244     if (match(AI->getOperand(1), m_SignMask()))
30245       return AtomicExpansionKind::None;
30246   }
30247 
30248   // If the atomicrmw's result is used by a single bit AND, we may use
30249   // bts/btr/btc instruction for these operations.
30250   // Note: InstCombinePass can cause a de-optimization here. It replaces the
30251   // SETCC(And(AtomicRMW(P, power_of_2), power_of_2)) with LShr and Xor
30252   // (depending on CC). This pattern can only use bts/btr/btc but we don't
30253   // detect it.
30254   Instruction *I = AI->user_back();
30255   auto BitChange = FindSingleBitChange(AI->getValOperand());
30256   if (BitChange.second == UndefBit || !AI->hasOneUse() ||
30257       I->getOpcode() != Instruction::And ||
30258       AI->getType()->getPrimitiveSizeInBits() == 8 ||
30259       AI->getParent() != I->getParent())
30260     return AtomicExpansionKind::CmpXChg;
30261 
30262   unsigned OtherIdx = I->getOperand(0) == AI ? 1 : 0;
30263 
30264   // This is a redundant AND, it should get cleaned up elsewhere.
30265   if (AI == I->getOperand(OtherIdx))
30266     return AtomicExpansionKind::CmpXChg;
30267 
30268   // The following instruction must be a AND single bit.
30269   if (BitChange.second == ConstantBit || BitChange.second == NotConstantBit) {
30270     auto *C1 = cast<ConstantInt>(AI->getValOperand());
30271     auto *C2 = dyn_cast<ConstantInt>(I->getOperand(OtherIdx));
30272     if (!C2 || !isPowerOf2_64(C2->getZExtValue())) {
30273       return AtomicExpansionKind::CmpXChg;
30274     }
30275     if (AI->getOperation() == AtomicRMWInst::And) {
30276       return ~C1->getValue() == C2->getValue()
30277                  ? AtomicExpansionKind::BitTestIntrinsic
30278                  : AtomicExpansionKind::CmpXChg;
30279     }
30280     return C1 == C2 ? AtomicExpansionKind::BitTestIntrinsic
30281                     : AtomicExpansionKind::CmpXChg;
30282   }
30283 
30284   assert(BitChange.second == ShiftBit || BitChange.second == NotShiftBit);
30285 
30286   auto BitTested = FindSingleBitChange(I->getOperand(OtherIdx));
30287   if (BitTested.second != ShiftBit && BitTested.second != NotShiftBit)
30288     return AtomicExpansionKind::CmpXChg;
30289 
30290   assert(BitChange.first != nullptr && BitTested.first != nullptr);
30291 
30292   // If shift amounts are not the same we can't use BitTestIntrinsic.
30293   if (BitChange.first != BitTested.first)
30294     return AtomicExpansionKind::CmpXChg;
30295 
30296   // If atomic AND need to be masking all be one bit and testing the one bit
30297   // unset in the mask.
30298   if (AI->getOperation() == AtomicRMWInst::And)
30299     return (BitChange.second == NotShiftBit && BitTested.second == ShiftBit)
30300                ? AtomicExpansionKind::BitTestIntrinsic
30301                : AtomicExpansionKind::CmpXChg;
30302 
30303   // If atomic XOR/OR need to be setting and testing the same bit.
30304   return (BitChange.second == ShiftBit && BitTested.second == ShiftBit)
30305              ? AtomicExpansionKind::BitTestIntrinsic
30306              : AtomicExpansionKind::CmpXChg;
30307 }
30308 
30309 void X86TargetLowering::emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const {
30310   IRBuilder<> Builder(AI);
30311   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30312   Intrinsic::ID IID_C = Intrinsic::not_intrinsic;
30313   Intrinsic::ID IID_I = Intrinsic::not_intrinsic;
30314   switch (AI->getOperation()) {
30315   default:
30316     llvm_unreachable("Unknown atomic operation");
30317   case AtomicRMWInst::Or:
30318     IID_C = Intrinsic::x86_atomic_bts;
30319     IID_I = Intrinsic::x86_atomic_bts_rm;
30320     break;
30321   case AtomicRMWInst::Xor:
30322     IID_C = Intrinsic::x86_atomic_btc;
30323     IID_I = Intrinsic::x86_atomic_btc_rm;
30324     break;
30325   case AtomicRMWInst::And:
30326     IID_C = Intrinsic::x86_atomic_btr;
30327     IID_I = Intrinsic::x86_atomic_btr_rm;
30328     break;
30329   }
30330   Instruction *I = AI->user_back();
30331   LLVMContext &Ctx = AI->getContext();
30332   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30333                                           PointerType::getUnqual(Ctx));
30334   Function *BitTest = nullptr;
30335   Value *Result = nullptr;
30336   auto BitTested = FindSingleBitChange(AI->getValOperand());
30337   assert(BitTested.first != nullptr);
30338 
30339   if (BitTested.second == ConstantBit || BitTested.second == NotConstantBit) {
30340     auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0));
30341 
30342     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType());
30343 
30344     unsigned Imm = llvm::countr_zero(C->getZExtValue());
30345     Result = Builder.CreateCall(BitTest, {Addr, Builder.getInt8(Imm)});
30346   } else {
30347     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType());
30348 
30349     assert(BitTested.second == ShiftBit || BitTested.second == NotShiftBit);
30350 
30351     Value *SI = BitTested.first;
30352     assert(SI != nullptr);
30353 
30354     // BT{S|R|C} on memory operand don't modulo bit position so we need to
30355     // mask it.
30356     unsigned ShiftBits = SI->getType()->getPrimitiveSizeInBits();
30357     Value *BitPos =
30358         Builder.CreateAnd(SI, Builder.getIntN(ShiftBits, ShiftBits - 1));
30359     // Todo(1): In many cases it may be provable that SI is less than
30360     // ShiftBits in which case this mask is unnecessary
30361     // Todo(2): In the fairly idiomatic case of P[X / sizeof_bits(X)] OP 1
30362     // << (X % sizeof_bits(X)) we can drop the shift mask and AGEN in
30363     // favor of just a raw BT{S|R|C}.
30364 
30365     Result = Builder.CreateCall(BitTest, {Addr, BitPos});
30366     Result = Builder.CreateZExtOrTrunc(Result, AI->getType());
30367 
30368     // If the result is only used for zero/non-zero status then we don't need to
30369     // shift value back. Otherwise do so.
30370     for (auto It = I->user_begin(); It != I->user_end(); ++It) {
30371       if (auto *ICmp = dyn_cast<ICmpInst>(*It)) {
30372         if (ICmp->isEquality()) {
30373           auto *C0 = dyn_cast<ConstantInt>(ICmp->getOperand(0));
30374           auto *C1 = dyn_cast<ConstantInt>(ICmp->getOperand(1));
30375           if (C0 || C1) {
30376             assert(C0 == nullptr || C1 == nullptr);
30377             if ((C0 ? C0 : C1)->isZero())
30378               continue;
30379           }
30380         }
30381       }
30382       Result = Builder.CreateShl(Result, BitPos);
30383       break;
30384     }
30385   }
30386 
30387   I->replaceAllUsesWith(Result);
30388   I->eraseFromParent();
30389   AI->eraseFromParent();
30390 }
30391 
30392 static bool shouldExpandCmpArithRMWInIR(AtomicRMWInst *AI) {
30393   using namespace llvm::PatternMatch;
30394   if (!AI->hasOneUse())
30395     return false;
30396 
30397   Value *Op = AI->getOperand(1);
30398   ICmpInst::Predicate Pred;
30399   Instruction *I = AI->user_back();
30400   AtomicRMWInst::BinOp Opc = AI->getOperation();
30401   if (Opc == AtomicRMWInst::Add) {
30402     if (match(I, m_c_ICmp(Pred, m_Sub(m_ZeroInt(), m_Specific(Op)), m_Value())))
30403       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30404     if (match(I, m_OneUse(m_c_Add(m_Specific(Op), m_Value())))) {
30405       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30406         return Pred == CmpInst::ICMP_SLT;
30407       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30408         return Pred == CmpInst::ICMP_SGT;
30409     }
30410     return false;
30411   }
30412   if (Opc == AtomicRMWInst::Sub) {
30413     if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30414       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30415     if (match(I, m_OneUse(m_Sub(m_Value(), m_Specific(Op))))) {
30416       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30417         return Pred == CmpInst::ICMP_SLT;
30418       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30419         return Pred == CmpInst::ICMP_SGT;
30420     }
30421     return false;
30422   }
30423   if ((Opc == AtomicRMWInst::Or &&
30424        match(I, m_OneUse(m_c_Or(m_Specific(Op), m_Value())))) ||
30425       (Opc == AtomicRMWInst::And &&
30426        match(I, m_OneUse(m_c_And(m_Specific(Op), m_Value()))))) {
30427     if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30428       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE ||
30429              Pred == CmpInst::ICMP_SLT;
30430     if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30431       return Pred == CmpInst::ICMP_SGT;
30432     return false;
30433   }
30434   if (Opc == AtomicRMWInst::Xor) {
30435     if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30436       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30437     if (match(I, m_OneUse(m_c_Xor(m_Specific(Op), m_Value())))) {
30438       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30439         return Pred == CmpInst::ICMP_SLT;
30440       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30441         return Pred == CmpInst::ICMP_SGT;
30442     }
30443     return false;
30444   }
30445 
30446   return false;
30447 }
30448 
30449 void X86TargetLowering::emitCmpArithAtomicRMWIntrinsic(
30450     AtomicRMWInst *AI) const {
30451   IRBuilder<> Builder(AI);
30452   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30453   Instruction *TempI = nullptr;
30454   LLVMContext &Ctx = AI->getContext();
30455   ICmpInst *ICI = dyn_cast<ICmpInst>(AI->user_back());
30456   if (!ICI) {
30457     TempI = AI->user_back();
30458     assert(TempI->hasOneUse() && "Must have one use");
30459     ICI = cast<ICmpInst>(TempI->user_back());
30460   }
30461   X86::CondCode CC = X86::COND_INVALID;
30462   ICmpInst::Predicate Pred = ICI->getPredicate();
30463   switch (Pred) {
30464   default:
30465     llvm_unreachable("Not supported Pred");
30466   case CmpInst::ICMP_EQ:
30467     CC = X86::COND_E;
30468     break;
30469   case CmpInst::ICMP_NE:
30470     CC = X86::COND_NE;
30471     break;
30472   case CmpInst::ICMP_SLT:
30473     CC = X86::COND_S;
30474     break;
30475   case CmpInst::ICMP_SGT:
30476     CC = X86::COND_NS;
30477     break;
30478   }
30479   Intrinsic::ID IID = Intrinsic::not_intrinsic;
30480   switch (AI->getOperation()) {
30481   default:
30482     llvm_unreachable("Unknown atomic operation");
30483   case AtomicRMWInst::Add:
30484     IID = Intrinsic::x86_atomic_add_cc;
30485     break;
30486   case AtomicRMWInst::Sub:
30487     IID = Intrinsic::x86_atomic_sub_cc;
30488     break;
30489   case AtomicRMWInst::Or:
30490     IID = Intrinsic::x86_atomic_or_cc;
30491     break;
30492   case AtomicRMWInst::And:
30493     IID = Intrinsic::x86_atomic_and_cc;
30494     break;
30495   case AtomicRMWInst::Xor:
30496     IID = Intrinsic::x86_atomic_xor_cc;
30497     break;
30498   }
30499   Function *CmpArith =
30500       Intrinsic::getDeclaration(AI->getModule(), IID, AI->getType());
30501   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30502                                           PointerType::getUnqual(Ctx));
30503   Value *Call = Builder.CreateCall(
30504       CmpArith, {Addr, AI->getValOperand(), Builder.getInt32((unsigned)CC)});
30505   Value *Result = Builder.CreateTrunc(Call, Type::getInt1Ty(Ctx));
30506   ICI->replaceAllUsesWith(Result);
30507   ICI->eraseFromParent();
30508   if (TempI)
30509     TempI->eraseFromParent();
30510   AI->eraseFromParent();
30511 }
30512 
30513 TargetLowering::AtomicExpansionKind
30514 X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
30515   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30516   Type *MemType = AI->getType();
30517 
30518   // If the operand is too big, we must see if cmpxchg8/16b is available
30519   // and default to library calls otherwise.
30520   if (MemType->getPrimitiveSizeInBits() > NativeWidth) {
30521     return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30522                                    : AtomicExpansionKind::None;
30523   }
30524 
30525   AtomicRMWInst::BinOp Op = AI->getOperation();
30526   switch (Op) {
30527   case AtomicRMWInst::Xchg:
30528     return AtomicExpansionKind::None;
30529   case AtomicRMWInst::Add:
30530   case AtomicRMWInst::Sub:
30531     if (shouldExpandCmpArithRMWInIR(AI))
30532       return AtomicExpansionKind::CmpArithIntrinsic;
30533     // It's better to use xadd, xsub or xchg for these in other cases.
30534     return AtomicExpansionKind::None;
30535   case AtomicRMWInst::Or:
30536   case AtomicRMWInst::And:
30537   case AtomicRMWInst::Xor:
30538     if (shouldExpandCmpArithRMWInIR(AI))
30539       return AtomicExpansionKind::CmpArithIntrinsic;
30540     return shouldExpandLogicAtomicRMWInIR(AI);
30541   case AtomicRMWInst::Nand:
30542   case AtomicRMWInst::Max:
30543   case AtomicRMWInst::Min:
30544   case AtomicRMWInst::UMax:
30545   case AtomicRMWInst::UMin:
30546   case AtomicRMWInst::FAdd:
30547   case AtomicRMWInst::FSub:
30548   case AtomicRMWInst::FMax:
30549   case AtomicRMWInst::FMin:
30550   case AtomicRMWInst::UIncWrap:
30551   case AtomicRMWInst::UDecWrap:
30552   default:
30553     // These always require a non-trivial set of data operations on x86. We must
30554     // use a cmpxchg loop.
30555     return AtomicExpansionKind::CmpXChg;
30556   }
30557 }
30558 
30559 LoadInst *
30560 X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
30561   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30562   Type *MemType = AI->getType();
30563   // Accesses larger than the native width are turned into cmpxchg/libcalls, so
30564   // there is no benefit in turning such RMWs into loads, and it is actually
30565   // harmful as it introduces a mfence.
30566   if (MemType->getPrimitiveSizeInBits() > NativeWidth)
30567     return nullptr;
30568 
30569   // If this is a canonical idempotent atomicrmw w/no uses, we have a better
30570   // lowering available in lowerAtomicArith.
30571   // TODO: push more cases through this path.
30572   if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand()))
30573     if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() &&
30574         AI->use_empty())
30575       return nullptr;
30576 
30577   IRBuilder<> Builder(AI);
30578   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30579   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
30580   auto SSID = AI->getSyncScopeID();
30581   // We must restrict the ordering to avoid generating loads with Release or
30582   // ReleaseAcquire orderings.
30583   auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
30584 
30585   // Before the load we need a fence. Here is an example lifted from
30586   // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
30587   // is required:
30588   // Thread 0:
30589   //   x.store(1, relaxed);
30590   //   r1 = y.fetch_add(0, release);
30591   // Thread 1:
30592   //   y.fetch_add(42, acquire);
30593   //   r2 = x.load(relaxed);
30594   // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
30595   // lowered to just a load without a fence. A mfence flushes the store buffer,
30596   // making the optimization clearly correct.
30597   // FIXME: it is required if isReleaseOrStronger(Order) but it is not clear
30598   // otherwise, we might be able to be more aggressive on relaxed idempotent
30599   // rmw. In practice, they do not look useful, so we don't try to be
30600   // especially clever.
30601   if (SSID == SyncScope::SingleThread)
30602     // FIXME: we could just insert an ISD::MEMBARRIER here, except we are at
30603     // the IR level, so we must wrap it in an intrinsic.
30604     return nullptr;
30605 
30606   if (!Subtarget.hasMFence())
30607     // FIXME: it might make sense to use a locked operation here but on a
30608     // different cache-line to prevent cache-line bouncing. In practice it
30609     // is probably a small win, and x86 processors without mfence are rare
30610     // enough that we do not bother.
30611     return nullptr;
30612 
30613   Function *MFence =
30614       llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
30615   Builder.CreateCall(MFence, {});
30616 
30617   // Finally we can emit the atomic load.
30618   LoadInst *Loaded = Builder.CreateAlignedLoad(
30619       AI->getType(), AI->getPointerOperand(), AI->getAlign());
30620   Loaded->setAtomic(Order, SSID);
30621   AI->replaceAllUsesWith(Loaded);
30622   AI->eraseFromParent();
30623   return Loaded;
30624 }
30625 
30626 /// Emit a locked operation on a stack location which does not change any
30627 /// memory location, but does involve a lock prefix.  Location is chosen to be
30628 /// a) very likely accessed only by a single thread to minimize cache traffic,
30629 /// and b) definitely dereferenceable.  Returns the new Chain result.
30630 static SDValue emitLockedStackOp(SelectionDAG &DAG,
30631                                  const X86Subtarget &Subtarget, SDValue Chain,
30632                                  const SDLoc &DL) {
30633   // Implementation notes:
30634   // 1) LOCK prefix creates a full read/write reordering barrier for memory
30635   // operations issued by the current processor.  As such, the location
30636   // referenced is not relevant for the ordering properties of the instruction.
30637   // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
30638   // 8.2.3.9  Loads and Stores Are Not Reordered with Locked Instructions
30639   // 2) Using an immediate operand appears to be the best encoding choice
30640   // here since it doesn't require an extra register.
30641   // 3) OR appears to be very slightly faster than ADD. (Though, the difference
30642   // is small enough it might just be measurement noise.)
30643   // 4) When choosing offsets, there are several contributing factors:
30644   //   a) If there's no redzone, we default to TOS.  (We could allocate a cache
30645   //      line aligned stack object to improve this case.)
30646   //   b) To minimize our chances of introducing a false dependence, we prefer
30647   //      to offset the stack usage from TOS slightly.
30648   //   c) To minimize concerns about cross thread stack usage - in particular,
30649   //      the idiomatic MyThreadPool.run([&StackVars]() {...}) pattern which
30650   //      captures state in the TOS frame and accesses it from many threads -
30651   //      we want to use an offset such that the offset is in a distinct cache
30652   //      line from the TOS frame.
30653   //
30654   // For a general discussion of the tradeoffs and benchmark results, see:
30655   // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
30656 
30657   auto &MF = DAG.getMachineFunction();
30658   auto &TFL = *Subtarget.getFrameLowering();
30659   const unsigned SPOffset = TFL.has128ByteRedZone(MF) ? -64 : 0;
30660 
30661   if (Subtarget.is64Bit()) {
30662     SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
30663     SDValue Ops[] = {
30664       DAG.getRegister(X86::RSP, MVT::i64),                  // Base
30665       DAG.getTargetConstant(1, DL, MVT::i8),                // Scale
30666       DAG.getRegister(0, MVT::i64),                         // Index
30667       DAG.getTargetConstant(SPOffset, DL, MVT::i32),        // Disp
30668       DAG.getRegister(0, MVT::i16),                         // Segment.
30669       Zero,
30670       Chain};
30671     SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
30672                                      MVT::Other, Ops);
30673     return SDValue(Res, 1);
30674   }
30675 
30676   SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
30677   SDValue Ops[] = {
30678     DAG.getRegister(X86::ESP, MVT::i32),            // Base
30679     DAG.getTargetConstant(1, DL, MVT::i8),          // Scale
30680     DAG.getRegister(0, MVT::i32),                   // Index
30681     DAG.getTargetConstant(SPOffset, DL, MVT::i32),  // Disp
30682     DAG.getRegister(0, MVT::i16),                   // Segment.
30683     Zero,
30684     Chain
30685   };
30686   SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
30687                                    MVT::Other, Ops);
30688   return SDValue(Res, 1);
30689 }
30690 
30691 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
30692                                  SelectionDAG &DAG) {
30693   SDLoc dl(Op);
30694   AtomicOrdering FenceOrdering =
30695       static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
30696   SyncScope::ID FenceSSID =
30697       static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
30698 
30699   // The only fence that needs an instruction is a sequentially-consistent
30700   // cross-thread fence.
30701   if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
30702       FenceSSID == SyncScope::System) {
30703     if (Subtarget.hasMFence())
30704       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
30705 
30706     SDValue Chain = Op.getOperand(0);
30707     return emitLockedStackOp(DAG, Subtarget, Chain, dl);
30708   }
30709 
30710   // MEMBARRIER is a compiler barrier; it codegens to a no-op.
30711   return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
30712 }
30713 
30714 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
30715                              SelectionDAG &DAG) {
30716   MVT T = Op.getSimpleValueType();
30717   SDLoc DL(Op);
30718   unsigned Reg = 0;
30719   unsigned size = 0;
30720   switch(T.SimpleTy) {
30721   default: llvm_unreachable("Invalid value type!");
30722   case MVT::i8:  Reg = X86::AL;  size = 1; break;
30723   case MVT::i16: Reg = X86::AX;  size = 2; break;
30724   case MVT::i32: Reg = X86::EAX; size = 4; break;
30725   case MVT::i64:
30726     assert(Subtarget.is64Bit() && "Node not type legal!");
30727     Reg = X86::RAX; size = 8;
30728     break;
30729   }
30730   SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
30731                                   Op.getOperand(2), SDValue());
30732   SDValue Ops[] = { cpIn.getValue(0),
30733                     Op.getOperand(1),
30734                     Op.getOperand(3),
30735                     DAG.getTargetConstant(size, DL, MVT::i8),
30736                     cpIn.getValue(1) };
30737   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
30738   MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
30739   SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
30740                                            Ops, T, MMO);
30741 
30742   SDValue cpOut =
30743     DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
30744   SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
30745                                       MVT::i32, cpOut.getValue(2));
30746   SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
30747 
30748   return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
30749                      cpOut, Success, EFLAGS.getValue(1));
30750 }
30751 
30752 // Create MOVMSKB, taking into account whether we need to split for AVX1.
30753 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
30754                            const X86Subtarget &Subtarget) {
30755   MVT InVT = V.getSimpleValueType();
30756 
30757   if (InVT == MVT::v64i8) {
30758     SDValue Lo, Hi;
30759     std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
30760     Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
30761     Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
30762     Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
30763     Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
30764     Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
30765                      DAG.getConstant(32, DL, MVT::i8));
30766     return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
30767   }
30768   if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) {
30769     SDValue Lo, Hi;
30770     std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
30771     Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
30772     Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
30773     Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
30774                      DAG.getConstant(16, DL, MVT::i8));
30775     return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
30776   }
30777 
30778   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
30779 }
30780 
30781 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
30782                             SelectionDAG &DAG) {
30783   SDValue Src = Op.getOperand(0);
30784   MVT SrcVT = Src.getSimpleValueType();
30785   MVT DstVT = Op.getSimpleValueType();
30786 
30787   // Legalize (v64i1 (bitcast i64 (X))) by splitting the i64, bitcasting each
30788   // half to v32i1 and concatenating the result.
30789   if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) {
30790     assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
30791     assert(Subtarget.hasBWI() && "Expected BWI target");
30792     SDLoc dl(Op);
30793     SDValue Lo, Hi;
30794     std::tie(Lo, Hi) = DAG.SplitScalar(Src, dl, MVT::i32, MVT::i32);
30795     Lo = DAG.getBitcast(MVT::v32i1, Lo);
30796     Hi = DAG.getBitcast(MVT::v32i1, Hi);
30797     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
30798   }
30799 
30800   // Use MOVMSK for vector to scalar conversion to prevent scalarization.
30801   if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) {
30802     assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512");
30803     MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8;
30804     SDLoc DL(Op);
30805     SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
30806     V = getPMOVMSKB(DL, V, DAG, Subtarget);
30807     return DAG.getZExtOrTrunc(V, DL, DstVT);
30808   }
30809 
30810   assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||
30811           SrcVT == MVT::i64) && "Unexpected VT!");
30812 
30813   assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
30814   if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
30815       !(DstVT == MVT::x86mmx && SrcVT.isVector()))
30816     // This conversion needs to be expanded.
30817     return SDValue();
30818 
30819   SDLoc dl(Op);
30820   if (SrcVT.isVector()) {
30821     // Widen the vector in input in the case of MVT::v2i32.
30822     // Example: from MVT::v2i32 to MVT::v4i32.
30823     MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
30824                                  SrcVT.getVectorNumElements() * 2);
30825     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
30826                       DAG.getUNDEF(SrcVT));
30827   } else {
30828     assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
30829            "Unexpected source type in LowerBITCAST");
30830     Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
30831   }
30832 
30833   MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64;
30834   Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src);
30835 
30836   if (DstVT == MVT::x86mmx)
30837     return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src);
30838 
30839   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src,
30840                      DAG.getIntPtrConstant(0, dl));
30841 }
30842 
30843 /// Compute the horizontal sum of bytes in V for the elements of VT.
30844 ///
30845 /// Requires V to be a byte vector and VT to be an integer vector type with
30846 /// wider elements than V's type. The width of the elements of VT determines
30847 /// how many bytes of V are summed horizontally to produce each element of the
30848 /// result.
30849 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
30850                                       const X86Subtarget &Subtarget,
30851                                       SelectionDAG &DAG) {
30852   SDLoc DL(V);
30853   MVT ByteVecVT = V.getSimpleValueType();
30854   MVT EltVT = VT.getVectorElementType();
30855   assert(ByteVecVT.getVectorElementType() == MVT::i8 &&
30856          "Expected value to have byte element type.");
30857   assert(EltVT != MVT::i8 &&
30858          "Horizontal byte sum only makes sense for wider elements!");
30859   unsigned VecSize = VT.getSizeInBits();
30860   assert(ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!");
30861 
30862   // PSADBW instruction horizontally add all bytes and leave the result in i64
30863   // chunks, thus directly computes the pop count for v2i64 and v4i64.
30864   if (EltVT == MVT::i64) {
30865     SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
30866     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
30867     V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
30868     return DAG.getBitcast(VT, V);
30869   }
30870 
30871   if (EltVT == MVT::i32) {
30872     // We unpack the low half and high half into i32s interleaved with zeros so
30873     // that we can use PSADBW to horizontally sum them. The most useful part of
30874     // this is that it lines up the results of two PSADBW instructions to be
30875     // two v2i64 vectors which concatenated are the 4 population counts. We can
30876     // then use PACKUSWB to shrink and concatenate them into a v4i32 again.
30877     SDValue Zeros = DAG.getConstant(0, DL, VT);
30878     SDValue V32 = DAG.getBitcast(VT, V);
30879     SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
30880     SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
30881 
30882     // Do the horizontal sums into two v2i64s.
30883     Zeros = DAG.getConstant(0, DL, ByteVecVT);
30884     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
30885     Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
30886                       DAG.getBitcast(ByteVecVT, Low), Zeros);
30887     High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
30888                        DAG.getBitcast(ByteVecVT, High), Zeros);
30889 
30890     // Merge them together.
30891     MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);
30892     V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
30893                     DAG.getBitcast(ShortVecVT, Low),
30894                     DAG.getBitcast(ShortVecVT, High));
30895 
30896     return DAG.getBitcast(VT, V);
30897   }
30898 
30899   // The only element type left is i16.
30900   assert(EltVT == MVT::i16 && "Unknown how to handle type");
30901 
30902   // To obtain pop count for each i16 element starting from the pop count for
30903   // i8 elements, shift the i16s left by 8, sum as i8s, and then shift as i16s
30904   // right by 8. It is important to shift as i16s as i8 vector shift isn't
30905   // directly supported.
30906   SDValue ShifterV = DAG.getConstant(8, DL, VT);
30907   SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
30908   V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
30909                   DAG.getBitcast(ByteVecVT, V));
30910   return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
30911 }
30912 
30913 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
30914                                         const X86Subtarget &Subtarget,
30915                                         SelectionDAG &DAG) {
30916   MVT VT = Op.getSimpleValueType();
30917   MVT EltVT = VT.getVectorElementType();
30918   int NumElts = VT.getVectorNumElements();
30919   (void)EltVT;
30920   assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported.");
30921 
30922   // Implement a lookup table in register by using an algorithm based on:
30923   // http://wm.ite.pl/articles/sse-popcount.html
30924   //
30925   // The general idea is that every lower byte nibble in the input vector is an
30926   // index into a in-register pre-computed pop count table. We then split up the
30927   // input vector in two new ones: (1) a vector with only the shifted-right
30928   // higher nibbles for each byte and (2) a vector with the lower nibbles (and
30929   // masked out higher ones) for each byte. PSHUFB is used separately with both
30930   // to index the in-register table. Next, both are added and the result is a
30931   // i8 vector where each element contains the pop count for input byte.
30932   const int LUT[16] = {/* 0 */ 0, /* 1 */ 1, /* 2 */ 1, /* 3 */ 2,
30933                        /* 4 */ 1, /* 5 */ 2, /* 6 */ 2, /* 7 */ 3,
30934                        /* 8 */ 1, /* 9 */ 2, /* a */ 2, /* b */ 3,
30935                        /* c */ 2, /* d */ 3, /* e */ 3, /* f */ 4};
30936 
30937   SmallVector<SDValue, 64> LUTVec;
30938   for (int i = 0; i < NumElts; ++i)
30939     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
30940   SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
30941   SDValue M0F = DAG.getConstant(0x0F, DL, VT);
30942 
30943   // High nibbles
30944   SDValue FourV = DAG.getConstant(4, DL, VT);
30945   SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
30946 
30947   // Low nibbles
30948   SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
30949 
30950   // The input vector is used as the shuffle mask that index elements into the
30951   // LUT. After counting low and high nibbles, add the vector to obtain the
30952   // final pop count per i8 element.
30953   SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
30954   SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
30955   return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
30956 }
30957 
30958 // Please ensure that any codegen change from LowerVectorCTPOP is reflected in
30959 // updated cost models in X86TTIImpl::getIntrinsicInstrCost.
30960 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
30961                                 SelectionDAG &DAG) {
30962   MVT VT = Op.getSimpleValueType();
30963   assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) &&
30964          "Unknown CTPOP type to handle");
30965   SDLoc DL(Op.getNode());
30966   SDValue Op0 = Op.getOperand(0);
30967 
30968   // TRUNC(CTPOP(ZEXT(X))) to make use of vXi32/vXi64 VPOPCNT instructions.
30969   if (Subtarget.hasVPOPCNTDQ()) {
30970     unsigned NumElems = VT.getVectorNumElements();
30971     assert((VT.getVectorElementType() == MVT::i8 ||
30972             VT.getVectorElementType() == MVT::i16) && "Unexpected type");
30973     if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
30974       MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
30975       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
30976       Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
30977       return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
30978     }
30979   }
30980 
30981   // Decompose 256-bit ops into smaller 128-bit ops.
30982   if (VT.is256BitVector() && !Subtarget.hasInt256())
30983     return splitVectorIntUnary(Op, DAG);
30984 
30985   // Decompose 512-bit ops into smaller 256-bit ops.
30986   if (VT.is512BitVector() && !Subtarget.hasBWI())
30987     return splitVectorIntUnary(Op, DAG);
30988 
30989   // For element types greater than i8, do vXi8 pop counts and a bytesum.
30990   if (VT.getScalarType() != MVT::i8) {
30991     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
30992     SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
30993     SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
30994     return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);
30995   }
30996 
30997   // We can't use the fast LUT approach, so fall back on LegalizeDAG.
30998   if (!Subtarget.hasSSSE3())
30999     return SDValue();
31000 
31001   return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
31002 }
31003 
31004 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
31005                           SelectionDAG &DAG) {
31006   assert(Op.getSimpleValueType().isVector() &&
31007          "We only do custom lowering for vector population count.");
31008   return LowerVectorCTPOP(Op, Subtarget, DAG);
31009 }
31010 
31011 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
31012   MVT VT = Op.getSimpleValueType();
31013   SDValue In = Op.getOperand(0);
31014   SDLoc DL(Op);
31015 
31016   // For scalars, its still beneficial to transfer to/from the SIMD unit to
31017   // perform the BITREVERSE.
31018   if (!VT.isVector()) {
31019     MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
31020     SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
31021     Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
31022     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
31023                        DAG.getIntPtrConstant(0, DL));
31024   }
31025 
31026   int NumElts = VT.getVectorNumElements();
31027   int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
31028 
31029   // Decompose 256-bit ops into smaller 128-bit ops.
31030   if (VT.is256BitVector())
31031     return splitVectorIntUnary(Op, DAG);
31032 
31033   assert(VT.is128BitVector() &&
31034          "Only 128-bit vector bitreverse lowering supported.");
31035 
31036   // VPPERM reverses the bits of a byte with the permute Op (2 << 5), and we
31037   // perform the BSWAP in the shuffle.
31038   // Its best to shuffle using the second operand as this will implicitly allow
31039   // memory folding for multiple vectors.
31040   SmallVector<SDValue, 16> MaskElts;
31041   for (int i = 0; i != NumElts; ++i) {
31042     for (int j = ScalarSizeInBytes - 1; j >= 0; --j) {
31043       int SourceByte = 16 + (i * ScalarSizeInBytes) + j;
31044       int PermuteByte = SourceByte | (2 << 5);
31045       MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
31046     }
31047   }
31048 
31049   SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
31050   SDValue Res = DAG.getBitcast(MVT::v16i8, In);
31051   Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
31052                     Res, Mask);
31053   return DAG.getBitcast(VT, Res);
31054 }
31055 
31056 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
31057                                SelectionDAG &DAG) {
31058   MVT VT = Op.getSimpleValueType();
31059 
31060   if (Subtarget.hasXOP() && !VT.is512BitVector())
31061     return LowerBITREVERSE_XOP(Op, DAG);
31062 
31063   assert(Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE");
31064 
31065   SDValue In = Op.getOperand(0);
31066   SDLoc DL(Op);
31067 
31068   assert(VT.getScalarType() == MVT::i8 &&
31069          "Only byte vector BITREVERSE supported");
31070 
31071   // Split v64i8 without BWI so that we can still use the PSHUFB lowering.
31072   if (VT == MVT::v64i8 && !Subtarget.hasBWI())
31073     return splitVectorIntUnary(Op, DAG);
31074 
31075   // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2.
31076   if (VT == MVT::v32i8 && !Subtarget.hasInt256())
31077     return splitVectorIntUnary(Op, DAG);
31078 
31079   unsigned NumElts = VT.getVectorNumElements();
31080 
31081   // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
31082   if (Subtarget.hasGFNI()) {
31083     MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
31084     SDValue Matrix = DAG.getConstant(0x8040201008040201ULL, DL, MatrixVT);
31085     Matrix = DAG.getBitcast(VT, Matrix);
31086     return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
31087                        DAG.getTargetConstant(0, DL, MVT::i8));
31088   }
31089 
31090   // Perform BITREVERSE using PSHUFB lookups. Each byte is split into
31091   // two nibbles and a PSHUFB lookup to find the bitreverse of each
31092   // 0-15 value (moved to the other nibble).
31093   SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
31094   SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
31095   SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
31096 
31097   const int LoLUT[16] = {
31098       /* 0 */ 0x00, /* 1 */ 0x80, /* 2 */ 0x40, /* 3 */ 0xC0,
31099       /* 4 */ 0x20, /* 5 */ 0xA0, /* 6 */ 0x60, /* 7 */ 0xE0,
31100       /* 8 */ 0x10, /* 9 */ 0x90, /* a */ 0x50, /* b */ 0xD0,
31101       /* c */ 0x30, /* d */ 0xB0, /* e */ 0x70, /* f */ 0xF0};
31102   const int HiLUT[16] = {
31103       /* 0 */ 0x00, /* 1 */ 0x08, /* 2 */ 0x04, /* 3 */ 0x0C,
31104       /* 4 */ 0x02, /* 5 */ 0x0A, /* 6 */ 0x06, /* 7 */ 0x0E,
31105       /* 8 */ 0x01, /* 9 */ 0x09, /* a */ 0x05, /* b */ 0x0D,
31106       /* c */ 0x03, /* d */ 0x0B, /* e */ 0x07, /* f */ 0x0F};
31107 
31108   SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
31109   for (unsigned i = 0; i < NumElts; ++i) {
31110     LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
31111     HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
31112   }
31113 
31114   SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
31115   SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
31116   Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
31117   Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
31118   return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
31119 }
31120 
31121 static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget,
31122                            SelectionDAG &DAG) {
31123   SDLoc DL(Op);
31124   SDValue X = Op.getOperand(0);
31125   MVT VT = Op.getSimpleValueType();
31126 
31127   // Special case. If the input fits in 8-bits we can use a single 8-bit TEST.
31128   if (VT == MVT::i8 ||
31129       DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) {
31130     X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31131     SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
31132                                 DAG.getConstant(0, DL, MVT::i8));
31133     // Copy the inverse of the parity flag into a register with setcc.
31134     SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31135     // Extend to the original type.
31136     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31137   }
31138 
31139   // If we have POPCNT, use the default expansion.
31140   if (Subtarget.hasPOPCNT())
31141     return SDValue();
31142 
31143   if (VT == MVT::i64) {
31144     // Xor the high and low 16-bits together using a 32-bit operation.
31145     SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
31146                              DAG.getNode(ISD::SRL, DL, MVT::i64, X,
31147                                          DAG.getConstant(32, DL, MVT::i8)));
31148     SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
31149     X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
31150   }
31151 
31152   if (VT != MVT::i16) {
31153     // Xor the high and low 16-bits together using a 32-bit operation.
31154     SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,
31155                                DAG.getConstant(16, DL, MVT::i8));
31156     X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16);
31157   } else {
31158     // If the input is 16-bits, we need to extend to use an i32 shift below.
31159     X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X);
31160   }
31161 
31162   // Finally xor the low 2 bytes together and use a 8-bit flag setting xor.
31163   // This should allow an h-reg to be used to save a shift.
31164   SDValue Hi = DAG.getNode(
31165       ISD::TRUNCATE, DL, MVT::i8,
31166       DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8)));
31167   SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31168   SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);
31169   SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
31170 
31171   // Copy the inverse of the parity flag into a register with setcc.
31172   SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31173   // Extend to the original type.
31174   return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31175 }
31176 
31177 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
31178                                         const X86Subtarget &Subtarget) {
31179   unsigned NewOpc = 0;
31180   switch (N->getOpcode()) {
31181   case ISD::ATOMIC_LOAD_ADD:
31182     NewOpc = X86ISD::LADD;
31183     break;
31184   case ISD::ATOMIC_LOAD_SUB:
31185     NewOpc = X86ISD::LSUB;
31186     break;
31187   case ISD::ATOMIC_LOAD_OR:
31188     NewOpc = X86ISD::LOR;
31189     break;
31190   case ISD::ATOMIC_LOAD_XOR:
31191     NewOpc = X86ISD::LXOR;
31192     break;
31193   case ISD::ATOMIC_LOAD_AND:
31194     NewOpc = X86ISD::LAND;
31195     break;
31196   default:
31197     llvm_unreachable("Unknown ATOMIC_LOAD_ opcode");
31198   }
31199 
31200   MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
31201 
31202   return DAG.getMemIntrinsicNode(
31203       NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
31204       {N->getOperand(0), N->getOperand(1), N->getOperand(2)},
31205       /*MemVT=*/N->getSimpleValueType(0), MMO);
31206 }
31207 
31208 /// Lower atomic_load_ops into LOCK-prefixed operations.
31209 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
31210                                 const X86Subtarget &Subtarget) {
31211   AtomicSDNode *AN = cast<AtomicSDNode>(N.getNode());
31212   SDValue Chain = N->getOperand(0);
31213   SDValue LHS = N->getOperand(1);
31214   SDValue RHS = N->getOperand(2);
31215   unsigned Opc = N->getOpcode();
31216   MVT VT = N->getSimpleValueType(0);
31217   SDLoc DL(N);
31218 
31219   // We can lower atomic_load_add into LXADD. However, any other atomicrmw op
31220   // can only be lowered when the result is unused.  They should have already
31221   // been transformed into a cmpxchg loop in AtomicExpand.
31222   if (N->hasAnyUseOfValue(0)) {
31223     // Handle (atomic_load_sub p, v) as (atomic_load_add p, -v), to be able to
31224     // select LXADD if LOCK_SUB can't be selected.
31225     // Handle (atomic_load_xor p, SignBit) as (atomic_load_add p, SignBit) so we
31226     // can use LXADD as opposed to cmpxchg.
31227     if (Opc == ISD::ATOMIC_LOAD_SUB ||
31228         (Opc == ISD::ATOMIC_LOAD_XOR && isMinSignedConstant(RHS))) {
31229       RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
31230       return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, RHS,
31231                            AN->getMemOperand());
31232     }
31233     assert(Opc == ISD::ATOMIC_LOAD_ADD &&
31234            "Used AtomicRMW ops other than Add should have been expanded!");
31235     return N;
31236   }
31237 
31238   // Specialized lowering for the canonical form of an idemptotent atomicrmw.
31239   // The core idea here is that since the memory location isn't actually
31240   // changing, all we need is a lowering for the *ordering* impacts of the
31241   // atomicrmw.  As such, we can chose a different operation and memory
31242   // location to minimize impact on other code.
31243   // The above holds unless the node is marked volatile in which
31244   // case it needs to be preserved according to the langref.
31245   if (Opc == ISD::ATOMIC_LOAD_OR && isNullConstant(RHS) && !AN->isVolatile()) {
31246     // On X86, the only ordering which actually requires an instruction is
31247     // seq_cst which isn't SingleThread, everything just needs to be preserved
31248     // during codegen and then dropped. Note that we expect (but don't assume),
31249     // that orderings other than seq_cst and acq_rel have been canonicalized to
31250     // a store or load.
31251     if (AN->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent &&
31252         AN->getSyncScopeID() == SyncScope::System) {
31253       // Prefer a locked operation against a stack location to minimize cache
31254       // traffic.  This assumes that stack locations are very likely to be
31255       // accessed only by the owning thread.
31256       SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
31257       assert(!N->hasAnyUseOfValue(0));
31258       // NOTE: The getUNDEF is needed to give something for the unused result 0.
31259       return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31260                          DAG.getUNDEF(VT), NewChain);
31261     }
31262     // MEMBARRIER is a compiler barrier; it codegens to a no-op.
31263     SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
31264     assert(!N->hasAnyUseOfValue(0));
31265     // NOTE: The getUNDEF is needed to give something for the unused result 0.
31266     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31267                        DAG.getUNDEF(VT), NewChain);
31268   }
31269 
31270   SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
31271   // RAUW the chain, but don't worry about the result, as it's unused.
31272   assert(!N->hasAnyUseOfValue(0));
31273   // NOTE: The getUNDEF is needed to give something for the unused result 0.
31274   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31275                      DAG.getUNDEF(VT), LockOp.getValue(1));
31276 }
31277 
31278 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
31279                                  const X86Subtarget &Subtarget) {
31280   auto *Node = cast<AtomicSDNode>(Op.getNode());
31281   SDLoc dl(Node);
31282   EVT VT = Node->getMemoryVT();
31283 
31284   bool IsSeqCst =
31285       Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent;
31286   bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT);
31287 
31288   // If this store is not sequentially consistent and the type is legal
31289   // we can just keep it.
31290   if (!IsSeqCst && IsTypeLegal)
31291     return Op;
31292 
31293   if (VT == MVT::i64 && !IsTypeLegal) {
31294     // For illegal i64 atomic_stores, we can try to use MOVQ or MOVLPS if SSE
31295     // is enabled.
31296     bool NoImplicitFloatOps =
31297         DAG.getMachineFunction().getFunction().hasFnAttribute(
31298             Attribute::NoImplicitFloat);
31299     if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
31300       SDValue Chain;
31301       if (Subtarget.hasSSE1()) {
31302         SDValue SclToVec =
31303             DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
31304         MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
31305         SclToVec = DAG.getBitcast(StVT, SclToVec);
31306         SDVTList Tys = DAG.getVTList(MVT::Other);
31307         SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()};
31308         Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
31309                                         MVT::i64, Node->getMemOperand());
31310       } else if (Subtarget.hasX87()) {
31311         // First load this into an 80-bit X87 register using a stack temporary.
31312         // This will put the whole integer into the significand.
31313         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
31314         int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
31315         MachinePointerInfo MPI =
31316             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
31317         Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr,
31318                              MPI, MaybeAlign(), MachineMemOperand::MOStore);
31319         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
31320         SDValue LdOps[] = {Chain, StackPtr};
31321         SDValue Value = DAG.getMemIntrinsicNode(
31322             X86ISD::FILD, dl, Tys, LdOps, MVT::i64, MPI,
31323             /*Align*/ std::nullopt, MachineMemOperand::MOLoad);
31324         Chain = Value.getValue(1);
31325 
31326         // Now use an FIST to do the atomic store.
31327         SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()};
31328         Chain =
31329             DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),
31330                                     StoreOps, MVT::i64, Node->getMemOperand());
31331       }
31332 
31333       if (Chain) {
31334         // If this is a sequentially consistent store, also emit an appropriate
31335         // barrier.
31336         if (IsSeqCst)
31337           Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
31338 
31339         return Chain;
31340       }
31341     }
31342   }
31343 
31344   // Convert seq_cst store -> xchg
31345   // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
31346   // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
31347   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
31348                                Node->getOperand(0), Node->getOperand(2),
31349                                Node->getOperand(1), Node->getMemOperand());
31350   return Swap.getValue(1);
31351 }
31352 
31353 static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
31354   SDNode *N = Op.getNode();
31355   MVT VT = N->getSimpleValueType(0);
31356   unsigned Opc = Op.getOpcode();
31357 
31358   // Let legalize expand this if it isn't a legal type yet.
31359   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
31360     return SDValue();
31361 
31362   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
31363   SDLoc DL(N);
31364 
31365   // Set the carry flag.
31366   SDValue Carry = Op.getOperand(2);
31367   EVT CarryVT = Carry.getValueType();
31368   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
31369                       Carry, DAG.getAllOnesConstant(DL, CarryVT));
31370 
31371   bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
31372   SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
31373                             Op.getOperand(0), Op.getOperand(1),
31374                             Carry.getValue(1));
31375 
31376   bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY;
31377   SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
31378                            Sum.getValue(1), DL, DAG);
31379   if (N->getValueType(1) == MVT::i1)
31380     SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
31381 
31382   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
31383 }
31384 
31385 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
31386                             SelectionDAG &DAG) {
31387   assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit());
31388 
31389   // For MacOSX, we want to call an alternative entry point: __sincos_stret,
31390   // which returns the values as { float, float } (in XMM0) or
31391   // { double, double } (which is returned in XMM0, XMM1).
31392   SDLoc dl(Op);
31393   SDValue Arg = Op.getOperand(0);
31394   EVT ArgVT = Arg.getValueType();
31395   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
31396 
31397   TargetLowering::ArgListTy Args;
31398   TargetLowering::ArgListEntry Entry;
31399 
31400   Entry.Node = Arg;
31401   Entry.Ty = ArgTy;
31402   Entry.IsSExt = false;
31403   Entry.IsZExt = false;
31404   Args.push_back(Entry);
31405 
31406   bool isF64 = ArgVT == MVT::f64;
31407   // Only optimize x86_64 for now. i386 is a bit messy. For f32,
31408   // the small struct {f32, f32} is returned in (eax, edx). For f64,
31409   // the results are returned via SRet in memory.
31410   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
31411   RTLIB::Libcall LC = isF64 ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
31412   const char *LibcallName = TLI.getLibcallName(LC);
31413   SDValue Callee =
31414       DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
31415 
31416   Type *RetTy = isF64 ? (Type *)StructType::get(ArgTy, ArgTy)
31417                       : (Type *)FixedVectorType::get(ArgTy, 4);
31418 
31419   TargetLowering::CallLoweringInfo CLI(DAG);
31420   CLI.setDebugLoc(dl)
31421       .setChain(DAG.getEntryNode())
31422       .setLibCallee(CallingConv::C, RetTy, Callee, std::move(Args));
31423 
31424   std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
31425 
31426   if (isF64)
31427     // Returned in xmm0 and xmm1.
31428     return CallResult.first;
31429 
31430   // Returned in bits 0:31 and 32:64 xmm0.
31431   SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31432                                CallResult.first, DAG.getIntPtrConstant(0, dl));
31433   SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31434                                CallResult.first, DAG.getIntPtrConstant(1, dl));
31435   SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
31436   return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
31437 }
31438 
31439 /// Widen a vector input to a vector of NVT.  The
31440 /// input vector must have the same element type as NVT.
31441 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
31442                             bool FillWithZeroes = false) {
31443   // Check if InOp already has the right width.
31444   MVT InVT = InOp.getSimpleValueType();
31445   if (InVT == NVT)
31446     return InOp;
31447 
31448   if (InOp.isUndef())
31449     return DAG.getUNDEF(NVT);
31450 
31451   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
31452          "input and widen element type must match");
31453 
31454   unsigned InNumElts = InVT.getVectorNumElements();
31455   unsigned WidenNumElts = NVT.getVectorNumElements();
31456   assert(WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 &&
31457          "Unexpected request for vector widening");
31458 
31459   SDLoc dl(InOp);
31460   if (InOp.getOpcode() == ISD::CONCAT_VECTORS &&
31461       InOp.getNumOperands() == 2) {
31462     SDValue N1 = InOp.getOperand(1);
31463     if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) ||
31464         N1.isUndef()) {
31465       InOp = InOp.getOperand(0);
31466       InVT = InOp.getSimpleValueType();
31467       InNumElts = InVT.getVectorNumElements();
31468     }
31469   }
31470   if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) ||
31471       ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) {
31472     SmallVector<SDValue, 16> Ops;
31473     for (unsigned i = 0; i < InNumElts; ++i)
31474       Ops.push_back(InOp.getOperand(i));
31475 
31476     EVT EltVT = InOp.getOperand(0).getValueType();
31477 
31478     SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
31479       DAG.getUNDEF(EltVT);
31480     for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i)
31481       Ops.push_back(FillVal);
31482     return DAG.getBuildVector(NVT, dl, Ops);
31483   }
31484   SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
31485     DAG.getUNDEF(NVT);
31486   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal,
31487                      InOp, DAG.getIntPtrConstant(0, dl));
31488 }
31489 
31490 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
31491                              SelectionDAG &DAG) {
31492   assert(Subtarget.hasAVX512() &&
31493          "MGATHER/MSCATTER are supported on AVX-512 arch only");
31494 
31495   MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
31496   SDValue Src = N->getValue();
31497   MVT VT = Src.getSimpleValueType();
31498   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");
31499   SDLoc dl(Op);
31500 
31501   SDValue Scale = N->getScale();
31502   SDValue Index = N->getIndex();
31503   SDValue Mask = N->getMask();
31504   SDValue Chain = N->getChain();
31505   SDValue BasePtr = N->getBasePtr();
31506 
31507   if (VT == MVT::v2f32 || VT == MVT::v2i32) {
31508     assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
31509     // If the index is v2i64 and we have VLX we can use xmm for data and index.
31510     if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
31511       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
31512       EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
31513       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
31514       SDVTList VTs = DAG.getVTList(MVT::Other);
31515       SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
31516       return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
31517                                      N->getMemoryVT(), N->getMemOperand());
31518     }
31519     return SDValue();
31520   }
31521 
31522   MVT IndexVT = Index.getSimpleValueType();
31523 
31524   // If the index is v2i32, we're being called by type legalization and we
31525   // should just let the default handling take care of it.
31526   if (IndexVT == MVT::v2i32)
31527     return SDValue();
31528 
31529   // If we don't have VLX and neither the passthru or index is 512-bits, we
31530   // need to widen until one is.
31531   if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
31532       !Index.getSimpleValueType().is512BitVector()) {
31533     // Determine how much we need to widen by to get a 512-bit type.
31534     unsigned Factor = std::min(512/VT.getSizeInBits(),
31535                                512/IndexVT.getSizeInBits());
31536     unsigned NumElts = VT.getVectorNumElements() * Factor;
31537 
31538     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
31539     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
31540     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
31541 
31542     Src = ExtendToType(Src, VT, DAG);
31543     Index = ExtendToType(Index, IndexVT, DAG);
31544     Mask = ExtendToType(Mask, MaskVT, DAG, true);
31545   }
31546 
31547   SDVTList VTs = DAG.getVTList(MVT::Other);
31548   SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
31549   return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
31550                                  N->getMemoryVT(), N->getMemOperand());
31551 }
31552 
31553 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
31554                           SelectionDAG &DAG) {
31555 
31556   MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
31557   MVT VT = Op.getSimpleValueType();
31558   MVT ScalarVT = VT.getScalarType();
31559   SDValue Mask = N->getMask();
31560   MVT MaskVT = Mask.getSimpleValueType();
31561   SDValue PassThru = N->getPassThru();
31562   SDLoc dl(Op);
31563 
31564   // Handle AVX masked loads which don't support passthru other than 0.
31565   if (MaskVT.getVectorElementType() != MVT::i1) {
31566     // We also allow undef in the isel pattern.
31567     if (PassThru.isUndef() || ISD::isBuildVectorAllZeros(PassThru.getNode()))
31568       return Op;
31569 
31570     SDValue NewLoad = DAG.getMaskedLoad(
31571         VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
31572         getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(),
31573         N->getMemOperand(), N->getAddressingMode(), N->getExtensionType(),
31574         N->isExpandingLoad());
31575     // Emit a blend.
31576     SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
31577     return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
31578   }
31579 
31580   assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
31581          "Expanding masked load is supported on AVX-512 target only!");
31582 
31583   assert((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) &&
31584          "Expanding masked load is supported for 32 and 64-bit types only!");
31585 
31586   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31587          "Cannot lower masked load op.");
31588 
31589   assert((ScalarVT.getSizeInBits() >= 32 ||
31590           (Subtarget.hasBWI() &&
31591               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
31592          "Unsupported masked load op.");
31593 
31594   // This operation is legal for targets with VLX, but without
31595   // VLX the vector should be widened to 512 bit
31596   unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
31597   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
31598   PassThru = ExtendToType(PassThru, WideDataVT, DAG);
31599 
31600   // Mask element has to be i1.
31601   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
31602          "Unexpected mask type");
31603 
31604   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
31605 
31606   Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
31607   SDValue NewLoad = DAG.getMaskedLoad(
31608       WideDataVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
31609       PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
31610       N->getExtensionType(), N->isExpandingLoad());
31611 
31612   SDValue Extract =
31613       DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0),
31614                   DAG.getIntPtrConstant(0, dl));
31615   SDValue RetOps[] = {Extract, NewLoad.getValue(1)};
31616   return DAG.getMergeValues(RetOps, dl);
31617 }
31618 
31619 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
31620                            SelectionDAG &DAG) {
31621   MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode());
31622   SDValue DataToStore = N->getValue();
31623   MVT VT = DataToStore.getSimpleValueType();
31624   MVT ScalarVT = VT.getScalarType();
31625   SDValue Mask = N->getMask();
31626   SDLoc dl(Op);
31627 
31628   assert((!N->isCompressingStore() || Subtarget.hasAVX512()) &&
31629          "Expanding masked load is supported on AVX-512 target only!");
31630 
31631   assert((!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) &&
31632          "Expanding masked load is supported for 32 and 64-bit types only!");
31633 
31634   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31635          "Cannot lower masked store op.");
31636 
31637   assert((ScalarVT.getSizeInBits() >= 32 ||
31638           (Subtarget.hasBWI() &&
31639               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
31640           "Unsupported masked store op.");
31641 
31642   // This operation is legal for targets with VLX, but without
31643   // VLX the vector should be widened to 512 bit
31644   unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
31645   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
31646 
31647   // Mask element has to be i1.
31648   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
31649          "Unexpected mask type");
31650 
31651   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
31652 
31653   DataToStore = ExtendToType(DataToStore, WideDataVT, DAG);
31654   Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
31655   return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(),
31656                             N->getOffset(), Mask, N->getMemoryVT(),
31657                             N->getMemOperand(), N->getAddressingMode(),
31658                             N->isTruncatingStore(), N->isCompressingStore());
31659 }
31660 
31661 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
31662                             SelectionDAG &DAG) {
31663   assert(Subtarget.hasAVX2() &&
31664          "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only");
31665 
31666   MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
31667   SDLoc dl(Op);
31668   MVT VT = Op.getSimpleValueType();
31669   SDValue Index = N->getIndex();
31670   SDValue Mask = N->getMask();
31671   SDValue PassThru = N->getPassThru();
31672   MVT IndexVT = Index.getSimpleValueType();
31673 
31674   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");
31675 
31676   // If the index is v2i32, we're being called by type legalization.
31677   if (IndexVT == MVT::v2i32)
31678     return SDValue();
31679 
31680   // If we don't have VLX and neither the passthru or index is 512-bits, we
31681   // need to widen until one is.
31682   MVT OrigVT = VT;
31683   if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31684       !IndexVT.is512BitVector()) {
31685     // Determine how much we need to widen by to get a 512-bit type.
31686     unsigned Factor = std::min(512/VT.getSizeInBits(),
31687                                512/IndexVT.getSizeInBits());
31688 
31689     unsigned NumElts = VT.getVectorNumElements() * Factor;
31690 
31691     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
31692     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
31693     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
31694 
31695     PassThru = ExtendToType(PassThru, VT, DAG);
31696     Index = ExtendToType(Index, IndexVT, DAG);
31697     Mask = ExtendToType(Mask, MaskVT, DAG, true);
31698   }
31699 
31700   // Break dependency on the data register.
31701   if (PassThru.isUndef())
31702     PassThru = getZeroVector(VT, Subtarget, DAG, dl);
31703 
31704   SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
31705                     N->getScale() };
31706   SDValue NewGather = DAG.getMemIntrinsicNode(
31707       X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
31708       N->getMemOperand());
31709   SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
31710                                 NewGather, DAG.getIntPtrConstant(0, dl));
31711   return DAG.getMergeValues({Extract, NewGather.getValue(1)}, dl);
31712 }
31713 
31714 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
31715   SDLoc dl(Op);
31716   SDValue Src = Op.getOperand(0);
31717   MVT DstVT = Op.getSimpleValueType();
31718 
31719   AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());
31720   unsigned SrcAS = N->getSrcAddressSpace();
31721 
31722   assert(SrcAS != N->getDestAddressSpace() &&
31723          "addrspacecast must be between different address spaces");
31724 
31725   if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64) {
31726     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
31727   } else if (DstVT == MVT::i64) {
31728     Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
31729   } else if (DstVT == MVT::i32) {
31730     Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
31731   } else {
31732     report_fatal_error("Bad address space in addrspacecast");
31733   }
31734   return Op;
31735 }
31736 
31737 SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
31738                                               SelectionDAG &DAG) const {
31739   // TODO: Eventually, the lowering of these nodes should be informed by or
31740   // deferred to the GC strategy for the function in which they appear. For
31741   // now, however, they must be lowered to something. Since they are logically
31742   // no-ops in the case of a null GC strategy (or a GC strategy which does not
31743   // require special handling for these nodes), lower them as literal NOOPs for
31744   // the time being.
31745   SmallVector<SDValue, 2> Ops;
31746   Ops.push_back(Op.getOperand(0));
31747   if (Op->getGluedNode())
31748     Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
31749 
31750   SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
31751   return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
31752 }
31753 
31754 // Custom split CVTPS2PH with wide types.
31755 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
31756   SDLoc dl(Op);
31757   EVT VT = Op.getValueType();
31758   SDValue Lo, Hi;
31759   std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
31760   EVT LoVT, HiVT;
31761   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
31762   SDValue RC = Op.getOperand(1);
31763   Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC);
31764   Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC);
31765   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
31766 }
31767 
31768 static SDValue LowerPREFETCH(SDValue Op, const X86Subtarget &Subtarget,
31769                              SelectionDAG &DAG) {
31770   unsigned IsData = Op.getConstantOperandVal(4);
31771 
31772   // We don't support non-data prefetch without PREFETCHI.
31773   // Just preserve the chain.
31774   if (!IsData && !Subtarget.hasPREFETCHI())
31775     return Op.getOperand(0);
31776 
31777   return Op;
31778 }
31779 
31780 static StringRef getInstrStrFromOpNo(const SmallVectorImpl<StringRef> &AsmStrs,
31781                                      unsigned OpNo) {
31782   const APInt Operand(32, OpNo);
31783   std::string OpNoStr = llvm::toString(Operand, 10, false);
31784   std::string Str(" $");
31785 
31786   std::string OpNoStr1(Str + OpNoStr);             // e.g. " $1" (OpNo=1)
31787   std::string OpNoStr2(Str + "{" + OpNoStr + ":"); // With modifier, e.g. ${1:P}
31788 
31789   auto I = StringRef::npos;
31790   for (auto &AsmStr : AsmStrs) {
31791     // Match the OpNo string. We should match exactly to exclude match
31792     // sub-string, e.g. "$12" contain "$1"
31793     if (AsmStr.ends_with(OpNoStr1))
31794       I = AsmStr.size() - OpNoStr1.size();
31795 
31796     // Get the index of operand in AsmStr.
31797     if (I == StringRef::npos)
31798       I = AsmStr.find(OpNoStr1 + ",");
31799     if (I == StringRef::npos)
31800       I = AsmStr.find(OpNoStr2);
31801 
31802     if (I == StringRef::npos)
31803       continue;
31804 
31805     assert(I > 0 && "Unexpected inline asm string!");
31806     // Remove the operand string and label (if exsit).
31807     // For example:
31808     // ".L__MSASMLABEL_.${:uid}__l:call dword ptr ${0:P}"
31809     // ==>
31810     // ".L__MSASMLABEL_.${:uid}__l:call dword ptr "
31811     // ==>
31812     // "call dword ptr "
31813     auto TmpStr = AsmStr.substr(0, I);
31814     I = TmpStr.rfind(':');
31815     if (I != StringRef::npos)
31816       TmpStr = TmpStr.substr(I + 1);
31817     return TmpStr.take_while(llvm::isAlpha);
31818   }
31819 
31820   return StringRef();
31821 }
31822 
31823 bool X86TargetLowering::isInlineAsmTargetBranch(
31824     const SmallVectorImpl<StringRef> &AsmStrs, unsigned OpNo) const {
31825   // In a __asm block, __asm inst foo where inst is CALL or JMP should be
31826   // changed from indirect TargetLowering::C_Memory to direct
31827   // TargetLowering::C_Address.
31828   // We don't need to special case LOOP* and Jcc, which cannot target a memory
31829   // location.
31830   StringRef Inst = getInstrStrFromOpNo(AsmStrs, OpNo);
31831   return Inst.equals_insensitive("call") || Inst.equals_insensitive("jmp");
31832 }
31833 
31834 /// Provide custom lowering hooks for some operations.
31835 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
31836   switch (Op.getOpcode()) {
31837   default: llvm_unreachable("Should not custom lower this!");
31838   case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op, Subtarget, DAG);
31839   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
31840     return LowerCMP_SWAP(Op, Subtarget, DAG);
31841   case ISD::CTPOP:              return LowerCTPOP(Op, Subtarget, DAG);
31842   case ISD::ATOMIC_LOAD_ADD:
31843   case ISD::ATOMIC_LOAD_SUB:
31844   case ISD::ATOMIC_LOAD_OR:
31845   case ISD::ATOMIC_LOAD_XOR:
31846   case ISD::ATOMIC_LOAD_AND:    return lowerAtomicArith(Op, DAG, Subtarget);
31847   case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op, DAG, Subtarget);
31848   case ISD::BITREVERSE:         return LowerBITREVERSE(Op, Subtarget, DAG);
31849   case ISD::PARITY:             return LowerPARITY(Op, Subtarget, DAG);
31850   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
31851   case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
31852   case ISD::VECTOR_SHUFFLE:     return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG);
31853   case ISD::VSELECT:            return LowerVSELECT(Op, DAG);
31854   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
31855   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
31856   case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
31857   case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
31858   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG);
31859   case ISD::ConstantPool:       return LowerConstantPool(Op, DAG);
31860   case ISD::GlobalAddress:      return LowerGlobalAddress(Op, DAG);
31861   case ISD::GlobalTLSAddress:   return LowerGlobalTLSAddress(Op, DAG);
31862   case ISD::ExternalSymbol:     return LowerExternalSymbol(Op, DAG);
31863   case ISD::BlockAddress:       return LowerBlockAddress(Op, DAG);
31864   case ISD::SHL_PARTS:
31865   case ISD::SRA_PARTS:
31866   case ISD::SRL_PARTS:          return LowerShiftParts(Op, DAG);
31867   case ISD::FSHL:
31868   case ISD::FSHR:               return LowerFunnelShift(Op, Subtarget, DAG);
31869   case ISD::STRICT_SINT_TO_FP:
31870   case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
31871   case ISD::STRICT_UINT_TO_FP:
31872   case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG);
31873   case ISD::TRUNCATE:           return LowerTRUNCATE(Op, DAG);
31874   case ISD::ZERO_EXTEND:        return LowerZERO_EXTEND(Op, Subtarget, DAG);
31875   case ISD::SIGN_EXTEND:        return LowerSIGN_EXTEND(Op, Subtarget, DAG);
31876   case ISD::ANY_EXTEND:         return LowerANY_EXTEND(Op, Subtarget, DAG);
31877   case ISD::ZERO_EXTEND_VECTOR_INREG:
31878   case ISD::SIGN_EXTEND_VECTOR_INREG:
31879     return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG);
31880   case ISD::FP_TO_SINT:
31881   case ISD::STRICT_FP_TO_SINT:
31882   case ISD::FP_TO_UINT:
31883   case ISD::STRICT_FP_TO_UINT:  return LowerFP_TO_INT(Op, DAG);
31884   case ISD::FP_TO_SINT_SAT:
31885   case ISD::FP_TO_UINT_SAT:     return LowerFP_TO_INT_SAT(Op, DAG);
31886   case ISD::FP_EXTEND:
31887   case ISD::STRICT_FP_EXTEND:   return LowerFP_EXTEND(Op, DAG);
31888   case ISD::FP_ROUND:
31889   case ISD::STRICT_FP_ROUND:    return LowerFP_ROUND(Op, DAG);
31890   case ISD::FP16_TO_FP:
31891   case ISD::STRICT_FP16_TO_FP:  return LowerFP16_TO_FP(Op, DAG);
31892   case ISD::FP_TO_FP16:
31893   case ISD::STRICT_FP_TO_FP16:  return LowerFP_TO_FP16(Op, DAG);
31894   case ISD::FP_TO_BF16:         return LowerFP_TO_BF16(Op, DAG);
31895   case ISD::LOAD:               return LowerLoad(Op, Subtarget, DAG);
31896   case ISD::STORE:              return LowerStore(Op, Subtarget, DAG);
31897   case ISD::FADD:
31898   case ISD::FSUB:               return lowerFaddFsub(Op, DAG);
31899   case ISD::FROUND:             return LowerFROUND(Op, DAG);
31900   case ISD::FABS:
31901   case ISD::FNEG:               return LowerFABSorFNEG(Op, DAG);
31902   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
31903   case ISD::FGETSIGN:           return LowerFGETSIGN(Op, DAG);
31904   case ISD::LRINT:
31905   case ISD::LLRINT:             return LowerLRINT_LLRINT(Op, DAG);
31906   case ISD::SETCC:
31907   case ISD::STRICT_FSETCC:
31908   case ISD::STRICT_FSETCCS:     return LowerSETCC(Op, DAG);
31909   case ISD::SETCCCARRY:         return LowerSETCCCARRY(Op, DAG);
31910   case ISD::SELECT:             return LowerSELECT(Op, DAG);
31911   case ISD::BRCOND:             return LowerBRCOND(Op, DAG);
31912   case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
31913   case ISD::VASTART:            return LowerVASTART(Op, DAG);
31914   case ISD::VAARG:              return LowerVAARG(Op, DAG);
31915   case ISD::VACOPY:             return LowerVACOPY(Op, Subtarget, DAG);
31916   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
31917   case ISD::INTRINSIC_VOID:
31918   case ISD::INTRINSIC_W_CHAIN:  return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
31919   case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
31920   case ISD::ADDROFRETURNADDR:   return LowerADDROFRETURNADDR(Op, DAG);
31921   case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
31922   case ISD::FRAME_TO_ARGS_OFFSET:
31923                                 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
31924   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
31925   case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
31926   case ISD::EH_SJLJ_SETJMP:     return lowerEH_SJLJ_SETJMP(Op, DAG);
31927   case ISD::EH_SJLJ_LONGJMP:    return lowerEH_SJLJ_LONGJMP(Op, DAG);
31928   case ISD::EH_SJLJ_SETUP_DISPATCH:
31929     return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
31930   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
31931   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
31932   case ISD::GET_ROUNDING:       return LowerGET_ROUNDING(Op, DAG);
31933   case ISD::SET_ROUNDING:       return LowerSET_ROUNDING(Op, DAG);
31934   case ISD::GET_FPENV_MEM:      return LowerGET_FPENV_MEM(Op, DAG);
31935   case ISD::SET_FPENV_MEM:      return LowerSET_FPENV_MEM(Op, DAG);
31936   case ISD::RESET_FPENV:        return LowerRESET_FPENV(Op, DAG);
31937   case ISD::CTLZ:
31938   case ISD::CTLZ_ZERO_UNDEF:    return LowerCTLZ(Op, Subtarget, DAG);
31939   case ISD::CTTZ:
31940   case ISD::CTTZ_ZERO_UNDEF:    return LowerCTTZ(Op, Subtarget, DAG);
31941   case ISD::MUL:                return LowerMUL(Op, Subtarget, DAG);
31942   case ISD::MULHS:
31943   case ISD::MULHU:              return LowerMULH(Op, Subtarget, DAG);
31944   case ISD::ROTL:
31945   case ISD::ROTR:               return LowerRotate(Op, Subtarget, DAG);
31946   case ISD::SRA:
31947   case ISD::SRL:
31948   case ISD::SHL:                return LowerShift(Op, Subtarget, DAG);
31949   case ISD::SADDO:
31950   case ISD::UADDO:
31951   case ISD::SSUBO:
31952   case ISD::USUBO:              return LowerXALUO(Op, DAG);
31953   case ISD::SMULO:
31954   case ISD::UMULO:              return LowerMULO(Op, Subtarget, DAG);
31955   case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
31956   case ISD::BITCAST:            return LowerBITCAST(Op, Subtarget, DAG);
31957   case ISD::SADDO_CARRY:
31958   case ISD::SSUBO_CARRY:
31959   case ISD::UADDO_CARRY:
31960   case ISD::USUBO_CARRY:        return LowerADDSUBO_CARRY(Op, DAG);
31961   case ISD::ADD:
31962   case ISD::SUB:                return lowerAddSub(Op, DAG, Subtarget);
31963   case ISD::UADDSAT:
31964   case ISD::SADDSAT:
31965   case ISD::USUBSAT:
31966   case ISD::SSUBSAT:            return LowerADDSAT_SUBSAT(Op, DAG, Subtarget);
31967   case ISD::SMAX:
31968   case ISD::SMIN:
31969   case ISD::UMAX:
31970   case ISD::UMIN:               return LowerMINMAX(Op, Subtarget, DAG);
31971   case ISD::FMINIMUM:
31972   case ISD::FMAXIMUM:
31973     return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG);
31974   case ISD::ABS:                return LowerABS(Op, Subtarget, DAG);
31975   case ISD::ABDS:
31976   case ISD::ABDU:               return LowerABD(Op, Subtarget, DAG);
31977   case ISD::AVGCEILU:           return LowerAVG(Op, Subtarget, DAG);
31978   case ISD::FSINCOS:            return LowerFSINCOS(Op, Subtarget, DAG);
31979   case ISD::MLOAD:              return LowerMLOAD(Op, Subtarget, DAG);
31980   case ISD::MSTORE:             return LowerMSTORE(Op, Subtarget, DAG);
31981   case ISD::MGATHER:            return LowerMGATHER(Op, Subtarget, DAG);
31982   case ISD::MSCATTER:           return LowerMSCATTER(Op, Subtarget, DAG);
31983   case ISD::GC_TRANSITION_START:
31984   case ISD::GC_TRANSITION_END:  return LowerGC_TRANSITION(Op, DAG);
31985   case ISD::ADDRSPACECAST:      return LowerADDRSPACECAST(Op, DAG);
31986   case X86ISD::CVTPS2PH:        return LowerCVTPS2PH(Op, DAG);
31987   case ISD::PREFETCH:           return LowerPREFETCH(Op, Subtarget, DAG);
31988   }
31989 }
31990 
31991 /// Replace a node with an illegal result type with a new node built out of
31992 /// custom code.
31993 void X86TargetLowering::ReplaceNodeResults(SDNode *N,
31994                                            SmallVectorImpl<SDValue>&Results,
31995                                            SelectionDAG &DAG) const {
31996   SDLoc dl(N);
31997   switch (N->getOpcode()) {
31998   default:
31999 #ifndef NDEBUG
32000     dbgs() << "ReplaceNodeResults: ";
32001     N->dump(&DAG);
32002 #endif
32003     llvm_unreachable("Do not know how to custom type legalize this operation!");
32004   case X86ISD::CVTPH2PS: {
32005     EVT VT = N->getValueType(0);
32006     SDValue Lo, Hi;
32007     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
32008     EVT LoVT, HiVT;
32009     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32010     Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo);
32011     Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi);
32012     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32013     Results.push_back(Res);
32014     return;
32015   }
32016   case X86ISD::STRICT_CVTPH2PS: {
32017     EVT VT = N->getValueType(0);
32018     SDValue Lo, Hi;
32019     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 1);
32020     EVT LoVT, HiVT;
32021     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32022     Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other},
32023                      {N->getOperand(0), Lo});
32024     Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other},
32025                      {N->getOperand(0), Hi});
32026     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
32027                                 Lo.getValue(1), Hi.getValue(1));
32028     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32029     Results.push_back(Res);
32030     Results.push_back(Chain);
32031     return;
32032   }
32033   case X86ISD::CVTPS2PH:
32034     Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG));
32035     return;
32036   case ISD::CTPOP: {
32037     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
32038     // Use a v2i64 if possible.
32039     bool NoImplicitFloatOps =
32040         DAG.getMachineFunction().getFunction().hasFnAttribute(
32041             Attribute::NoImplicitFloat);
32042     if (isTypeLegal(MVT::v2i64) && !NoImplicitFloatOps) {
32043       SDValue Wide =
32044           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));
32045       Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);
32046       // Bit count should fit in 32-bits, extract it as that and then zero
32047       // extend to i64. Otherwise we end up extracting bits 63:32 separately.
32048       Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);
32049       Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,
32050                          DAG.getIntPtrConstant(0, dl));
32051       Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);
32052       Results.push_back(Wide);
32053     }
32054     return;
32055   }
32056   case ISD::MUL: {
32057     EVT VT = N->getValueType(0);
32058     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32059            VT.getVectorElementType() == MVT::i8 && "Unexpected VT!");
32060     // Pre-promote these to vXi16 to avoid op legalization thinking all 16
32061     // elements are needed.
32062     MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
32063     SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
32064     SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
32065     SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
32066     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32067     unsigned NumConcats = 16 / VT.getVectorNumElements();
32068     SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
32069     ConcatOps[0] = Res;
32070     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
32071     Results.push_back(Res);
32072     return;
32073   }
32074   case ISD::SMULO:
32075   case ISD::UMULO: {
32076     EVT VT = N->getValueType(0);
32077     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32078            VT == MVT::v2i32 && "Unexpected VT!");
32079     bool IsSigned = N->getOpcode() == ISD::SMULO;
32080     unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
32081     SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
32082     SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
32083     SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);
32084     // Extract the high 32 bits from each result using PSHUFD.
32085     // TODO: Could use SRL+TRUNCATE but that doesn't become a PSHUFD.
32086     SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);
32087     Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1});
32088     Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi,
32089                      DAG.getIntPtrConstant(0, dl));
32090 
32091     // Truncate the low bits of the result. This will become PSHUFD.
32092     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32093 
32094     SDValue HiCmp;
32095     if (IsSigned) {
32096       // SMULO overflows if the high bits don't match the sign of the low.
32097       HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT));
32098     } else {
32099       // UMULO overflows if the high bits are non-zero.
32100       HiCmp = DAG.getConstant(0, dl, VT);
32101     }
32102     SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
32103 
32104     // Widen the result with by padding with undef.
32105     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
32106                       DAG.getUNDEF(VT));
32107     Results.push_back(Res);
32108     Results.push_back(Ovf);
32109     return;
32110   }
32111   case X86ISD::VPMADDWD: {
32112     // Legalize types for X86ISD::VPMADDWD by widening.
32113     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32114 
32115     EVT VT = N->getValueType(0);
32116     EVT InVT = N->getOperand(0).getValueType();
32117     assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&
32118            "Expected a VT that divides into 128 bits.");
32119     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32120            "Unexpected type action!");
32121     unsigned NumConcat = 128 / InVT.getSizeInBits();
32122 
32123     EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
32124                                     InVT.getVectorElementType(),
32125                                     NumConcat * InVT.getVectorNumElements());
32126     EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
32127                                   VT.getVectorElementType(),
32128                                   NumConcat * VT.getVectorNumElements());
32129 
32130     SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
32131     Ops[0] = N->getOperand(0);
32132     SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32133     Ops[0] = N->getOperand(1);
32134     SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32135 
32136     SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
32137     Results.push_back(Res);
32138     return;
32139   }
32140   // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
32141   case X86ISD::FMINC:
32142   case X86ISD::FMIN:
32143   case X86ISD::FMAXC:
32144   case X86ISD::FMAX: {
32145     EVT VT = N->getValueType(0);
32146     assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.");
32147     SDValue UNDEF = DAG.getUNDEF(VT);
32148     SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32149                               N->getOperand(0), UNDEF);
32150     SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32151                               N->getOperand(1), UNDEF);
32152     Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
32153     return;
32154   }
32155   case ISD::SDIV:
32156   case ISD::UDIV:
32157   case ISD::SREM:
32158   case ISD::UREM: {
32159     EVT VT = N->getValueType(0);
32160     if (VT.isVector()) {
32161       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32162              "Unexpected type action!");
32163       // If this RHS is a constant splat vector we can widen this and let
32164       // division/remainder by constant optimize it.
32165       // TODO: Can we do something for non-splat?
32166       APInt SplatVal;
32167       if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) {
32168         unsigned NumConcats = 128 / VT.getSizeInBits();
32169         SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
32170         Ops0[0] = N->getOperand(0);
32171         EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
32172         SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
32173         SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
32174         SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
32175         Results.push_back(Res);
32176       }
32177       return;
32178     }
32179 
32180     SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
32181     Results.push_back(V);
32182     return;
32183   }
32184   case ISD::TRUNCATE: {
32185     MVT VT = N->getSimpleValueType(0);
32186     if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
32187       return;
32188 
32189     // The generic legalizer will try to widen the input type to the same
32190     // number of elements as the widened result type. But this isn't always
32191     // the best thing so do some custom legalization to avoid some cases.
32192     MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
32193     SDValue In = N->getOperand(0);
32194     EVT InVT = In.getValueType();
32195     EVT InEltVT = InVT.getVectorElementType();
32196     EVT EltVT = VT.getVectorElementType();
32197     unsigned MinElts = VT.getVectorNumElements();
32198     unsigned WidenNumElts = WidenVT.getVectorNumElements();
32199     unsigned InBits = InVT.getSizeInBits();
32200 
32201     // See if there are sufficient leading bits to perform a PACKUS/PACKSS.
32202     unsigned PackOpcode;
32203     if (SDValue Src =
32204             matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG, Subtarget)) {
32205       if (SDValue Res = truncateVectorWithPACK(PackOpcode, VT, Src,
32206                                                dl, DAG, Subtarget)) {
32207         Res = widenSubVector(WidenVT, Res, false, Subtarget, DAG, dl);
32208         Results.push_back(Res);
32209         return;
32210       }
32211     }
32212 
32213     if (128 % InBits == 0) {
32214       // 128 bit and smaller inputs should avoid truncate all together and
32215       // just use a build_vector that will become a shuffle.
32216       // TODO: Widen and use a shuffle directly?
32217       SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
32218       // Use the original element count so we don't do more scalar opts than
32219       // necessary.
32220       for (unsigned i=0; i < MinElts; ++i) {
32221         SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In,
32222                                   DAG.getIntPtrConstant(i, dl));
32223         Ops[i] = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Val);
32224       }
32225       Results.push_back(DAG.getBuildVector(WidenVT, dl, Ops));
32226       return;
32227     }
32228 
32229     // With AVX512 there are some cases that can use a target specific
32230     // truncate node to go from 256/512 to less than 128 with zeros in the
32231     // upper elements of the 128 bit result.
32232     if (Subtarget.hasAVX512() && isTypeLegal(InVT)) {
32233       // We can use VTRUNC directly if for 256 bits with VLX or for any 512.
32234       if ((InBits == 256 && Subtarget.hasVLX()) || InBits == 512) {
32235         Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32236         return;
32237       }
32238       // There's one case we can widen to 512 bits and use VTRUNC.
32239       if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {
32240         In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,
32241                          DAG.getUNDEF(MVT::v4i64));
32242         Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32243         return;
32244       }
32245     }
32246     if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 &&
32247         getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector &&
32248         isTypeLegal(MVT::v4i64)) {
32249       // Input needs to be split and output needs to widened. Let's use two
32250       // VTRUNCs, and shuffle their results together into the wider type.
32251       SDValue Lo, Hi;
32252       std::tie(Lo, Hi) = DAG.SplitVector(In, dl);
32253 
32254       Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
32255       Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
32256       SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
32257                                          { 0,  1,  2,  3, 16, 17, 18, 19,
32258                                           -1, -1, -1, -1, -1, -1, -1, -1 });
32259       Results.push_back(Res);
32260       return;
32261     }
32262 
32263     // Attempt to widen the truncation input vector to let LowerTRUNCATE handle
32264     // this via type legalization.
32265     if ((InEltVT == MVT::i16 || InEltVT == MVT::i32 || InEltVT == MVT::i64) &&
32266         (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32) &&
32267         (!Subtarget.hasSSSE3() ||
32268          (!isTypeLegal(InVT) &&
32269           !(MinElts <= 4 && InEltVT == MVT::i64 && EltVT == MVT::i8)))) {
32270       SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl,
32271                                        InEltVT.getSizeInBits() * WidenNumElts);
32272       Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, WidenVT, WidenIn));
32273       return;
32274     }
32275 
32276     return;
32277   }
32278   case ISD::ANY_EXTEND:
32279     // Right now, only MVT::v8i8 has Custom action for an illegal type.
32280     // It's intended to custom handle the input type.
32281     assert(N->getValueType(0) == MVT::v8i8 &&
32282            "Do not know how to legalize this Node");
32283     return;
32284   case ISD::SIGN_EXTEND:
32285   case ISD::ZERO_EXTEND: {
32286     EVT VT = N->getValueType(0);
32287     SDValue In = N->getOperand(0);
32288     EVT InVT = In.getValueType();
32289     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
32290         (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
32291       assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
32292              "Unexpected type action!");
32293       assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode");
32294       // Custom split this so we can extend i8/i16->i32 invec. This is better
32295       // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
32296       // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
32297       // we allow the sra from the extend to i32 to be shared by the split.
32298       In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
32299 
32300       // Fill a vector with sign bits for each element.
32301       SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
32302       SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
32303 
32304       // Create an unpackl and unpackh to interleave the sign bits then bitcast
32305       // to v2i64.
32306       SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32307                                         {0, 4, 1, 5});
32308       Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
32309       SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32310                                         {2, 6, 3, 7});
32311       Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
32312 
32313       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32314       Results.push_back(Res);
32315       return;
32316     }
32317 
32318     if (VT == MVT::v16i32 || VT == MVT::v8i64) {
32319       if (!InVT.is128BitVector()) {
32320         // Not a 128 bit vector, but maybe type legalization will promote
32321         // it to 128 bits.
32322         if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger)
32323           return;
32324         InVT = getTypeToTransformTo(*DAG.getContext(), InVT);
32325         if (!InVT.is128BitVector())
32326           return;
32327 
32328         // Promote the input to 128 bits. Type legalization will turn this into
32329         // zext_inreg/sext_inreg.
32330         In = DAG.getNode(N->getOpcode(), dl, InVT, In);
32331       }
32332 
32333       // Perform custom splitting instead of the two stage extend we would get
32334       // by default.
32335       EVT LoVT, HiVT;
32336       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
32337       assert(isTypeLegal(LoVT) && "Split VT not legal?");
32338 
32339       SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG);
32340 
32341       // We need to shift the input over by half the number of elements.
32342       unsigned NumElts = InVT.getVectorNumElements();
32343       unsigned HalfNumElts = NumElts / 2;
32344       SmallVector<int, 16> ShufMask(NumElts, SM_SentinelUndef);
32345       for (unsigned i = 0; i != HalfNumElts; ++i)
32346         ShufMask[i] = i + HalfNumElts;
32347 
32348       SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
32349       Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG);
32350 
32351       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32352       Results.push_back(Res);
32353     }
32354     return;
32355   }
32356   case ISD::FP_TO_SINT:
32357   case ISD::STRICT_FP_TO_SINT:
32358   case ISD::FP_TO_UINT:
32359   case ISD::STRICT_FP_TO_UINT: {
32360     bool IsStrict = N->isStrictFPOpcode();
32361     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
32362                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
32363     EVT VT = N->getValueType(0);
32364     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32365     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
32366     EVT SrcVT = Src.getValueType();
32367 
32368     SDValue Res;
32369     if (isSoftF16(SrcVT, Subtarget)) {
32370       EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
32371       if (IsStrict) {
32372         Res =
32373             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
32374                         {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
32375                                             {NVT, MVT::Other}, {Chain, Src})});
32376         Chain = Res.getValue(1);
32377       } else {
32378         Res = DAG.getNode(N->getOpcode(), dl, VT,
32379                           DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
32380       }
32381       Results.push_back(Res);
32382       if (IsStrict)
32383         Results.push_back(Chain);
32384 
32385       return;
32386     }
32387 
32388     if (VT.isVector() && Subtarget.hasFP16() &&
32389         SrcVT.getVectorElementType() == MVT::f16) {
32390       EVT EleVT = VT.getVectorElementType();
32391       EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
32392 
32393       if (SrcVT != MVT::v8f16) {
32394         SDValue Tmp =
32395             IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
32396         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
32397         Ops[0] = Src;
32398         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
32399       }
32400 
32401       if (IsStrict) {
32402         unsigned Opc =
32403             IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32404         Res =
32405             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
32406         Chain = Res.getValue(1);
32407       } else {
32408         unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32409         Res = DAG.getNode(Opc, dl, ResVT, Src);
32410       }
32411 
32412       // TODO: Need to add exception check code for strict FP.
32413       if (EleVT.getSizeInBits() < 16) {
32414         MVT TmpVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8);
32415         Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res);
32416 
32417         // Now widen to 128 bits.
32418         unsigned NumConcats = 128 / TmpVT.getSizeInBits();
32419         MVT ConcatVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8 * NumConcats);
32420         SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT));
32421         ConcatOps[0] = Res;
32422         Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
32423       }
32424 
32425       Results.push_back(Res);
32426       if (IsStrict)
32427         Results.push_back(Chain);
32428 
32429       return;
32430     }
32431 
32432     if (VT.isVector() && VT.getScalarSizeInBits() < 32) {
32433       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32434              "Unexpected type action!");
32435 
32436       // Try to create a 128 bit vector, but don't exceed a 32 bit element.
32437       unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);
32438       MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth),
32439                                        VT.getVectorNumElements());
32440       SDValue Res;
32441       SDValue Chain;
32442       if (IsStrict) {
32443         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},
32444                           {N->getOperand(0), Src});
32445         Chain = Res.getValue(1);
32446       } else
32447         Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
32448 
32449       // Preserve what we know about the size of the original result. If the
32450       // result is v2i32, we have to manually widen the assert.
32451       if (PromoteVT == MVT::v2i32)
32452         Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
32453                           DAG.getUNDEF(MVT::v2i32));
32454 
32455       Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl,
32456                         Res.getValueType(), Res,
32457                         DAG.getValueType(VT.getVectorElementType()));
32458 
32459       if (PromoteVT == MVT::v2i32)
32460         Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
32461                           DAG.getIntPtrConstant(0, dl));
32462 
32463       // Truncate back to the original width.
32464       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32465 
32466       // Now widen to 128 bits.
32467       unsigned NumConcats = 128 / VT.getSizeInBits();
32468       MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),
32469                                       VT.getVectorNumElements() * NumConcats);
32470       SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
32471       ConcatOps[0] = Res;
32472       Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
32473       Results.push_back(Res);
32474       if (IsStrict)
32475         Results.push_back(Chain);
32476       return;
32477     }
32478 
32479 
32480     if (VT == MVT::v2i32) {
32481       assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) &&
32482              "Strict unsigned conversion requires AVX512");
32483       assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32484       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32485              "Unexpected type action!");
32486       if (Src.getValueType() == MVT::v2f64) {
32487         if (!IsSigned && !Subtarget.hasAVX512()) {
32488           SDValue Res =
32489               expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget);
32490           Results.push_back(Res);
32491           return;
32492         }
32493 
32494         unsigned Opc;
32495         if (IsStrict)
32496           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32497         else
32498           Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32499 
32500         // If we have VLX we can emit a target specific FP_TO_UINT node,.
32501         if (!IsSigned && !Subtarget.hasVLX()) {
32502           // Otherwise we can defer to the generic legalizer which will widen
32503           // the input as well. This will be further widened during op
32504           // legalization to v8i32<-v8f64.
32505           // For strict nodes we'll need to widen ourselves.
32506           // FIXME: Fix the type legalizer to safely widen strict nodes?
32507           if (!IsStrict)
32508             return;
32509           Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,
32510                             DAG.getConstantFP(0.0, dl, MVT::v2f64));
32511           Opc = N->getOpcode();
32512         }
32513         SDValue Res;
32514         SDValue Chain;
32515         if (IsStrict) {
32516           Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
32517                             {N->getOperand(0), Src});
32518           Chain = Res.getValue(1);
32519         } else {
32520           Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
32521         }
32522         Results.push_back(Res);
32523         if (IsStrict)
32524           Results.push_back(Chain);
32525         return;
32526       }
32527 
32528       // Custom widen strict v2f32->v2i32 by padding with zeros.
32529       // FIXME: Should generic type legalizer do this?
32530       if (Src.getValueType() == MVT::v2f32 && IsStrict) {
32531         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
32532                           DAG.getConstantFP(0.0, dl, MVT::v2f32));
32533         SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
32534                                   {N->getOperand(0), Src});
32535         Results.push_back(Res);
32536         Results.push_back(Res.getValue(1));
32537         return;
32538       }
32539 
32540       // The FP_TO_INTHelper below only handles f32/f64/f80 scalar inputs,
32541       // so early out here.
32542       return;
32543     }
32544 
32545     assert(!VT.isVector() && "Vectors should have been handled above!");
32546 
32547     if ((Subtarget.hasDQI() && VT == MVT::i64 &&
32548          (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
32549         (Subtarget.hasFP16() && SrcVT == MVT::f16)) {
32550       assert(!Subtarget.is64Bit() && "i64 should be legal");
32551       unsigned NumElts = Subtarget.hasVLX() ? 2 : 8;
32552       // If we use a 128-bit result we might need to use a target specific node.
32553       unsigned SrcElts =
32554           std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits());
32555       MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);
32556       MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);
32557       unsigned Opc = N->getOpcode();
32558       if (NumElts != SrcElts) {
32559         if (IsStrict)
32560           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32561         else
32562           Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32563       }
32564 
32565       SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
32566       SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
32567                                 DAG.getConstantFP(0.0, dl, VecInVT), Src,
32568                                 ZeroIdx);
32569       SDValue Chain;
32570       if (IsStrict) {
32571         SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
32572         Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res);
32573         Chain = Res.getValue(1);
32574       } else
32575         Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res);
32576       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);
32577       Results.push_back(Res);
32578       if (IsStrict)
32579         Results.push_back(Chain);
32580       return;
32581     }
32582 
32583     if (VT == MVT::i128 && Subtarget.isTargetWin64()) {
32584       SDValue Chain;
32585       SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain);
32586       Results.push_back(V);
32587       if (IsStrict)
32588         Results.push_back(Chain);
32589       return;
32590     }
32591 
32592     if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
32593       Results.push_back(V);
32594       if (IsStrict)
32595         Results.push_back(Chain);
32596     }
32597     return;
32598   }
32599   case ISD::LRINT:
32600   case ISD::LLRINT: {
32601     if (SDValue V = LRINT_LLRINTHelper(N, DAG))
32602       Results.push_back(V);
32603     return;
32604   }
32605 
32606   case ISD::SINT_TO_FP:
32607   case ISD::STRICT_SINT_TO_FP:
32608   case ISD::UINT_TO_FP:
32609   case ISD::STRICT_UINT_TO_FP: {
32610     bool IsStrict = N->isStrictFPOpcode();
32611     bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
32612                     N->getOpcode() == ISD::STRICT_SINT_TO_FP;
32613     EVT VT = N->getValueType(0);
32614     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32615     if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() &&
32616         Subtarget.hasVLX()) {
32617       if (Src.getValueType().getVectorElementType() == MVT::i16)
32618         return;
32619 
32620       if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32)
32621         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
32622                           IsStrict ? DAG.getConstant(0, dl, MVT::v2i32)
32623                                    : DAG.getUNDEF(MVT::v2i32));
32624       if (IsStrict) {
32625         unsigned Opc =
32626             IsSigned ? X86ISD::STRICT_CVTSI2P : X86ISD::STRICT_CVTUI2P;
32627         SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
32628                                   {N->getOperand(0), Src});
32629         Results.push_back(Res);
32630         Results.push_back(Res.getValue(1));
32631       } else {
32632         unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
32633         Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src));
32634       }
32635       return;
32636     }
32637     if (VT != MVT::v2f32)
32638       return;
32639     EVT SrcVT = Src.getValueType();
32640     if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
32641       if (IsStrict) {
32642         unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
32643                                 : X86ISD::STRICT_CVTUI2P;
32644         SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
32645                                   {N->getOperand(0), Src});
32646         Results.push_back(Res);
32647         Results.push_back(Res.getValue(1));
32648       } else {
32649         unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
32650         Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));
32651       }
32652       return;
32653     }
32654     if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
32655         Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
32656       SDValue Zero = DAG.getConstant(0, dl, SrcVT);
32657       SDValue One  = DAG.getConstant(1, dl, SrcVT);
32658       SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
32659                                  DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
32660                                  DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
32661       SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
32662       SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
32663       SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
32664       for (int i = 0; i != 2; ++i) {
32665         SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
32666                                   SignSrc, DAG.getIntPtrConstant(i, dl));
32667         if (IsStrict)
32668           SignCvts[i] =
32669               DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},
32670                           {N->getOperand(0), Elt});
32671         else
32672           SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt);
32673       };
32674       SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
32675       SDValue Slow, Chain;
32676       if (IsStrict) {
32677         Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
32678                             SignCvts[0].getValue(1), SignCvts[1].getValue(1));
32679         Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},
32680                            {Chain, SignCvt, SignCvt});
32681         Chain = Slow.getValue(1);
32682       } else {
32683         Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);
32684       }
32685       IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);
32686       IsNeg =
32687           DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});
32688       SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
32689       Results.push_back(Cvt);
32690       if (IsStrict)
32691         Results.push_back(Chain);
32692       return;
32693     }
32694 
32695     if (SrcVT != MVT::v2i32)
32696       return;
32697 
32698     if (IsSigned || Subtarget.hasAVX512()) {
32699       if (!IsStrict)
32700         return;
32701 
32702       // Custom widen strict v2i32->v2f32 to avoid scalarization.
32703       // FIXME: Should generic type legalizer do this?
32704       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
32705                         DAG.getConstant(0, dl, MVT::v2i32));
32706       SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
32707                                 {N->getOperand(0), Src});
32708       Results.push_back(Res);
32709       Results.push_back(Res.getValue(1));
32710       return;
32711     }
32712 
32713     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32714     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
32715     SDValue VBias = DAG.getConstantFP(
32716         llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::v2f64);
32717     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
32718                              DAG.getBitcast(MVT::v2i64, VBias));
32719     Or = DAG.getBitcast(MVT::v2f64, Or);
32720     if (IsStrict) {
32721       SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
32722                                 {N->getOperand(0), Or, VBias});
32723       SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
32724                                 {MVT::v4f32, MVT::Other},
32725                                 {Sub.getValue(1), Sub});
32726       Results.push_back(Res);
32727       Results.push_back(Res.getValue(1));
32728     } else {
32729       // TODO: Are there any fast-math-flags to propagate here?
32730       SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
32731       Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
32732     }
32733     return;
32734   }
32735   case ISD::STRICT_FP_ROUND:
32736   case ISD::FP_ROUND: {
32737     bool IsStrict = N->isStrictFPOpcode();
32738     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
32739     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32740     SDValue Rnd = N->getOperand(IsStrict ? 2 : 1);
32741     EVT SrcVT = Src.getValueType();
32742     EVT VT = N->getValueType(0);
32743     SDValue V;
32744     if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) {
32745       SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)
32746                              : DAG.getUNDEF(MVT::v2f32);
32747       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext);
32748     }
32749     if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) {
32750       assert(Subtarget.hasF16C() && "Cannot widen f16 without F16C");
32751       if (SrcVT.getVectorElementType() != MVT::f32)
32752         return;
32753 
32754       if (IsStrict)
32755         V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
32756                         {Chain, Src, Rnd});
32757       else
32758         V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd);
32759 
32760       Results.push_back(DAG.getBitcast(MVT::v8f16, V));
32761       if (IsStrict)
32762         Results.push_back(V.getValue(1));
32763       return;
32764     }
32765     if (!isTypeLegal(Src.getValueType()))
32766       return;
32767     EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32;
32768     if (IsStrict)
32769       V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other},
32770                       {Chain, Src});
32771     else
32772       V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src);
32773     Results.push_back(V);
32774     if (IsStrict)
32775       Results.push_back(V.getValue(1));
32776     return;
32777   }
32778   case ISD::FP_EXTEND:
32779   case ISD::STRICT_FP_EXTEND: {
32780     // Right now, only MVT::v2f32 has OperationAction for FP_EXTEND.
32781     // No other ValueType for FP_EXTEND should reach this point.
32782     assert(N->getValueType(0) == MVT::v2f32 &&
32783            "Do not know how to legalize this Node");
32784     if (!Subtarget.hasFP16() || !Subtarget.hasVLX())
32785       return;
32786     bool IsStrict = N->isStrictFPOpcode();
32787     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32788     SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)
32789                            : DAG.getUNDEF(MVT::v2f16);
32790     SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);
32791     if (IsStrict)
32792       V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other},
32793                       {N->getOperand(0), V});
32794     else
32795       V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V);
32796     Results.push_back(V);
32797     if (IsStrict)
32798       Results.push_back(V.getValue(1));
32799     return;
32800   }
32801   case ISD::INTRINSIC_W_CHAIN: {
32802     unsigned IntNo = N->getConstantOperandVal(1);
32803     switch (IntNo) {
32804     default : llvm_unreachable("Do not know how to custom type "
32805                                "legalize this intrinsic operation!");
32806     case Intrinsic::x86_rdtsc:
32807       return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
32808                                      Results);
32809     case Intrinsic::x86_rdtscp:
32810       return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
32811                                      Results);
32812     case Intrinsic::x86_rdpmc:
32813       expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
32814                                   Results);
32815       return;
32816     case Intrinsic::x86_rdpru:
32817       expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
32818         Results);
32819       return;
32820     case Intrinsic::x86_xgetbv:
32821       expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
32822                                   Results);
32823       return;
32824     }
32825   }
32826   case ISD::READCYCLECOUNTER: {
32827     return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
32828   }
32829   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
32830     EVT T = N->getValueType(0);
32831     assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
32832     bool Regs64bit = T == MVT::i128;
32833     assert((!Regs64bit || Subtarget.canUseCMPXCHG16B()) &&
32834            "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B");
32835     MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
32836     SDValue cpInL, cpInH;
32837     std::tie(cpInL, cpInH) =
32838         DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT);
32839     cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
32840                              Regs64bit ? X86::RAX : X86::EAX, cpInL, SDValue());
32841     cpInH =
32842         DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX,
32843                          cpInH, cpInL.getValue(1));
32844     SDValue swapInL, swapInH;
32845     std::tie(swapInL, swapInH) =
32846         DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT);
32847     swapInH =
32848         DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
32849                          swapInH, cpInH.getValue(1));
32850 
32851     // In 64-bit mode we might need the base pointer in RBX, but we can't know
32852     // until later. So we keep the RBX input in a vreg and use a custom
32853     // inserter.
32854     // Since RBX will be a reserved register the register allocator will not
32855     // make sure its value will be properly saved and restored around this
32856     // live-range.
32857     SDValue Result;
32858     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
32859     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
32860     if (Regs64bit) {
32861       SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL,
32862                        swapInH.getValue(1)};
32863       Result =
32864           DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO);
32865     } else {
32866       swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL,
32867                                  swapInH.getValue(1));
32868       SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
32869                        swapInL.getValue(1)};
32870       Result =
32871           DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO);
32872     }
32873 
32874     SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
32875                                         Regs64bit ? X86::RAX : X86::EAX,
32876                                         HalfT, Result.getValue(1));
32877     SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
32878                                         Regs64bit ? X86::RDX : X86::EDX,
32879                                         HalfT, cpOutL.getValue(2));
32880     SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
32881 
32882     SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
32883                                         MVT::i32, cpOutH.getValue(2));
32884     SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
32885     Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
32886 
32887     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
32888     Results.push_back(Success);
32889     Results.push_back(EFLAGS.getValue(1));
32890     return;
32891   }
32892   case ISD::ATOMIC_LOAD: {
32893     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
32894     bool NoImplicitFloatOps =
32895         DAG.getMachineFunction().getFunction().hasFnAttribute(
32896             Attribute::NoImplicitFloat);
32897     if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
32898       auto *Node = cast<AtomicSDNode>(N);
32899       if (Subtarget.hasSSE1()) {
32900         // Use a VZEXT_LOAD which will be selected as MOVQ or XORPS+MOVLPS.
32901         // Then extract the lower 64-bits.
32902         MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
32903         SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
32904         SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
32905         SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
32906                                              MVT::i64, Node->getMemOperand());
32907         if (Subtarget.hasSSE2()) {
32908           SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
32909                                     DAG.getIntPtrConstant(0, dl));
32910           Results.push_back(Res);
32911           Results.push_back(Ld.getValue(1));
32912           return;
32913         }
32914         // We use an alternative sequence for SSE1 that extracts as v2f32 and
32915         // then casts to i64. This avoids a 128-bit stack temporary being
32916         // created by type legalization if we were to cast v4f32->v2i64.
32917         SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
32918                                   DAG.getIntPtrConstant(0, dl));
32919         Res = DAG.getBitcast(MVT::i64, Res);
32920         Results.push_back(Res);
32921         Results.push_back(Ld.getValue(1));
32922         return;
32923       }
32924       if (Subtarget.hasX87()) {
32925         // First load this into an 80-bit X87 register. This will put the whole
32926         // integer into the significand.
32927         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
32928         SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
32929         SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
32930                                                  dl, Tys, Ops, MVT::i64,
32931                                                  Node->getMemOperand());
32932         SDValue Chain = Result.getValue(1);
32933 
32934         // Now store the X87 register to a stack temporary and convert to i64.
32935         // This store is not atomic and doesn't need to be.
32936         // FIXME: We don't need a stack temporary if the result of the load
32937         // is already being stored. We could just directly store there.
32938         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
32939         int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
32940         MachinePointerInfo MPI =
32941             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
32942         SDValue StoreOps[] = { Chain, Result, StackPtr };
32943         Chain = DAG.getMemIntrinsicNode(
32944             X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
32945             MPI, std::nullopt /*Align*/, MachineMemOperand::MOStore);
32946 
32947         // Finally load the value back from the stack temporary and return it.
32948         // This load is not atomic and doesn't need to be.
32949         // This load will be further type legalized.
32950         Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);
32951         Results.push_back(Result);
32952         Results.push_back(Result.getValue(1));
32953         return;
32954       }
32955     }
32956     // TODO: Use MOVLPS when SSE1 is available?
32957     // Delegate to generic TypeLegalization. Situations we can really handle
32958     // should have already been dealt with by AtomicExpandPass.cpp.
32959     break;
32960   }
32961   case ISD::ATOMIC_SWAP:
32962   case ISD::ATOMIC_LOAD_ADD:
32963   case ISD::ATOMIC_LOAD_SUB:
32964   case ISD::ATOMIC_LOAD_AND:
32965   case ISD::ATOMIC_LOAD_OR:
32966   case ISD::ATOMIC_LOAD_XOR:
32967   case ISD::ATOMIC_LOAD_NAND:
32968   case ISD::ATOMIC_LOAD_MIN:
32969   case ISD::ATOMIC_LOAD_MAX:
32970   case ISD::ATOMIC_LOAD_UMIN:
32971   case ISD::ATOMIC_LOAD_UMAX:
32972     // Delegate to generic TypeLegalization. Situations we can really handle
32973     // should have already been dealt with by AtomicExpandPass.cpp.
32974     break;
32975 
32976   case ISD::BITCAST: {
32977     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32978     EVT DstVT = N->getValueType(0);
32979     EVT SrcVT = N->getOperand(0).getValueType();
32980 
32981     // If this is a bitcast from a v64i1 k-register to a i64 on a 32-bit target
32982     // we can split using the k-register rather than memory.
32983     if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) {
32984       assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
32985       SDValue Lo, Hi;
32986       std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
32987       Lo = DAG.getBitcast(MVT::i32, Lo);
32988       Hi = DAG.getBitcast(MVT::i32, Hi);
32989       SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
32990       Results.push_back(Res);
32991       return;
32992     }
32993 
32994     if (DstVT.isVector() && SrcVT == MVT::x86mmx) {
32995       // FIXME: Use v4f32 for SSE1?
32996       assert(Subtarget.hasSSE2() && "Requires SSE2");
32997       assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
32998              "Unexpected type action!");
32999       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
33000       SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
33001                                 N->getOperand(0));
33002       Res = DAG.getBitcast(WideVT, Res);
33003       Results.push_back(Res);
33004       return;
33005     }
33006 
33007     return;
33008   }
33009   case ISD::MGATHER: {
33010     EVT VT = N->getValueType(0);
33011     if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&
33012         (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
33013       auto *Gather = cast<MaskedGatherSDNode>(N);
33014       SDValue Index = Gather->getIndex();
33015       if (Index.getValueType() != MVT::v2i64)
33016         return;
33017       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33018              "Unexpected type action!");
33019       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33020       SDValue Mask = Gather->getMask();
33021       assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
33022       SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
33023                                      Gather->getPassThru(),
33024                                      DAG.getUNDEF(VT));
33025       if (!Subtarget.hasVLX()) {
33026         // We need to widen the mask, but the instruction will only use 2
33027         // of its elements. So we can use undef.
33028         Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
33029                            DAG.getUNDEF(MVT::v2i1));
33030         Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
33031       }
33032       SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
33033                         Gather->getBasePtr(), Index, Gather->getScale() };
33034       SDValue Res = DAG.getMemIntrinsicNode(
33035           X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
33036           Gather->getMemoryVT(), Gather->getMemOperand());
33037       Results.push_back(Res);
33038       Results.push_back(Res.getValue(1));
33039       return;
33040     }
33041     return;
33042   }
33043   case ISD::LOAD: {
33044     // Use an f64/i64 load and a scalar_to_vector for v2f32/v2i32 loads. This
33045     // avoids scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
33046     // cast since type legalization will try to use an i64 load.
33047     MVT VT = N->getSimpleValueType(0);
33048     assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT");
33049     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33050            "Unexpected type action!");
33051     if (!ISD::isNON_EXTLoad(N))
33052       return;
33053     auto *Ld = cast<LoadSDNode>(N);
33054     if (Subtarget.hasSSE2()) {
33055       MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;
33056       SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
33057                                 Ld->getPointerInfo(), Ld->getOriginalAlign(),
33058                                 Ld->getMemOperand()->getFlags());
33059       SDValue Chain = Res.getValue(1);
33060       MVT VecVT = MVT::getVectorVT(LdVT, 2);
33061       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
33062       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33063       Res = DAG.getBitcast(WideVT, Res);
33064       Results.push_back(Res);
33065       Results.push_back(Chain);
33066       return;
33067     }
33068     assert(Subtarget.hasSSE1() && "Expected SSE");
33069     SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
33070     SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
33071     SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
33072                                           MVT::i64, Ld->getMemOperand());
33073     Results.push_back(Res);
33074     Results.push_back(Res.getValue(1));
33075     return;
33076   }
33077   case ISD::ADDRSPACECAST: {
33078     SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
33079     Results.push_back(V);
33080     return;
33081   }
33082   case ISD::BITREVERSE: {
33083     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
33084     assert(Subtarget.hasXOP() && "Expected XOP");
33085     // We can use VPPERM by copying to a vector register and back. We'll need
33086     // to move the scalar in two i32 pieces.
33087     Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
33088     return;
33089   }
33090   case ISD::EXTRACT_VECTOR_ELT: {
33091     // f16 = extract vXf16 %vec, i64 %idx
33092     assert(N->getSimpleValueType(0) == MVT::f16 &&
33093            "Unexpected Value type of EXTRACT_VECTOR_ELT!");
33094     assert(Subtarget.hasFP16() && "Expected FP16");
33095     SDValue VecOp = N->getOperand(0);
33096     EVT ExtVT = VecOp.getValueType().changeVectorElementTypeToInteger();
33097     SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0));
33098     Split = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Split,
33099                         N->getOperand(1));
33100     Split = DAG.getBitcast(MVT::f16, Split);
33101     Results.push_back(Split);
33102     return;
33103   }
33104   }
33105 }
33106 
33107 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
33108   switch ((X86ISD::NodeType)Opcode) {
33109   case X86ISD::FIRST_NUMBER:       break;
33110 #define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
33111   NODE_NAME_CASE(BSF)
33112   NODE_NAME_CASE(BSR)
33113   NODE_NAME_CASE(FSHL)
33114   NODE_NAME_CASE(FSHR)
33115   NODE_NAME_CASE(FAND)
33116   NODE_NAME_CASE(FANDN)
33117   NODE_NAME_CASE(FOR)
33118   NODE_NAME_CASE(FXOR)
33119   NODE_NAME_CASE(FILD)
33120   NODE_NAME_CASE(FIST)
33121   NODE_NAME_CASE(FP_TO_INT_IN_MEM)
33122   NODE_NAME_CASE(FLD)
33123   NODE_NAME_CASE(FST)
33124   NODE_NAME_CASE(CALL)
33125   NODE_NAME_CASE(CALL_RVMARKER)
33126   NODE_NAME_CASE(BT)
33127   NODE_NAME_CASE(CMP)
33128   NODE_NAME_CASE(FCMP)
33129   NODE_NAME_CASE(STRICT_FCMP)
33130   NODE_NAME_CASE(STRICT_FCMPS)
33131   NODE_NAME_CASE(COMI)
33132   NODE_NAME_CASE(UCOMI)
33133   NODE_NAME_CASE(CMPM)
33134   NODE_NAME_CASE(CMPMM)
33135   NODE_NAME_CASE(STRICT_CMPM)
33136   NODE_NAME_CASE(CMPMM_SAE)
33137   NODE_NAME_CASE(SETCC)
33138   NODE_NAME_CASE(SETCC_CARRY)
33139   NODE_NAME_CASE(FSETCC)
33140   NODE_NAME_CASE(FSETCCM)
33141   NODE_NAME_CASE(FSETCCM_SAE)
33142   NODE_NAME_CASE(CMOV)
33143   NODE_NAME_CASE(BRCOND)
33144   NODE_NAME_CASE(RET_GLUE)
33145   NODE_NAME_CASE(IRET)
33146   NODE_NAME_CASE(REP_STOS)
33147   NODE_NAME_CASE(REP_MOVS)
33148   NODE_NAME_CASE(GlobalBaseReg)
33149   NODE_NAME_CASE(Wrapper)
33150   NODE_NAME_CASE(WrapperRIP)
33151   NODE_NAME_CASE(MOVQ2DQ)
33152   NODE_NAME_CASE(MOVDQ2Q)
33153   NODE_NAME_CASE(MMX_MOVD2W)
33154   NODE_NAME_CASE(MMX_MOVW2D)
33155   NODE_NAME_CASE(PEXTRB)
33156   NODE_NAME_CASE(PEXTRW)
33157   NODE_NAME_CASE(INSERTPS)
33158   NODE_NAME_CASE(PINSRB)
33159   NODE_NAME_CASE(PINSRW)
33160   NODE_NAME_CASE(PSHUFB)
33161   NODE_NAME_CASE(ANDNP)
33162   NODE_NAME_CASE(BLENDI)
33163   NODE_NAME_CASE(BLENDV)
33164   NODE_NAME_CASE(HADD)
33165   NODE_NAME_CASE(HSUB)
33166   NODE_NAME_CASE(FHADD)
33167   NODE_NAME_CASE(FHSUB)
33168   NODE_NAME_CASE(CONFLICT)
33169   NODE_NAME_CASE(FMAX)
33170   NODE_NAME_CASE(FMAXS)
33171   NODE_NAME_CASE(FMAX_SAE)
33172   NODE_NAME_CASE(FMAXS_SAE)
33173   NODE_NAME_CASE(FMIN)
33174   NODE_NAME_CASE(FMINS)
33175   NODE_NAME_CASE(FMIN_SAE)
33176   NODE_NAME_CASE(FMINS_SAE)
33177   NODE_NAME_CASE(FMAXC)
33178   NODE_NAME_CASE(FMINC)
33179   NODE_NAME_CASE(FRSQRT)
33180   NODE_NAME_CASE(FRCP)
33181   NODE_NAME_CASE(EXTRQI)
33182   NODE_NAME_CASE(INSERTQI)
33183   NODE_NAME_CASE(TLSADDR)
33184   NODE_NAME_CASE(TLSBASEADDR)
33185   NODE_NAME_CASE(TLSCALL)
33186   NODE_NAME_CASE(EH_SJLJ_SETJMP)
33187   NODE_NAME_CASE(EH_SJLJ_LONGJMP)
33188   NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH)
33189   NODE_NAME_CASE(EH_RETURN)
33190   NODE_NAME_CASE(TC_RETURN)
33191   NODE_NAME_CASE(FNSTCW16m)
33192   NODE_NAME_CASE(FLDCW16m)
33193   NODE_NAME_CASE(FNSTENVm)
33194   NODE_NAME_CASE(FLDENVm)
33195   NODE_NAME_CASE(LCMPXCHG_DAG)
33196   NODE_NAME_CASE(LCMPXCHG8_DAG)
33197   NODE_NAME_CASE(LCMPXCHG16_DAG)
33198   NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
33199   NODE_NAME_CASE(LADD)
33200   NODE_NAME_CASE(LSUB)
33201   NODE_NAME_CASE(LOR)
33202   NODE_NAME_CASE(LXOR)
33203   NODE_NAME_CASE(LAND)
33204   NODE_NAME_CASE(LBTS)
33205   NODE_NAME_CASE(LBTC)
33206   NODE_NAME_CASE(LBTR)
33207   NODE_NAME_CASE(LBTS_RM)
33208   NODE_NAME_CASE(LBTC_RM)
33209   NODE_NAME_CASE(LBTR_RM)
33210   NODE_NAME_CASE(AADD)
33211   NODE_NAME_CASE(AOR)
33212   NODE_NAME_CASE(AXOR)
33213   NODE_NAME_CASE(AAND)
33214   NODE_NAME_CASE(VZEXT_MOVL)
33215   NODE_NAME_CASE(VZEXT_LOAD)
33216   NODE_NAME_CASE(VEXTRACT_STORE)
33217   NODE_NAME_CASE(VTRUNC)
33218   NODE_NAME_CASE(VTRUNCS)
33219   NODE_NAME_CASE(VTRUNCUS)
33220   NODE_NAME_CASE(VMTRUNC)
33221   NODE_NAME_CASE(VMTRUNCS)
33222   NODE_NAME_CASE(VMTRUNCUS)
33223   NODE_NAME_CASE(VTRUNCSTORES)
33224   NODE_NAME_CASE(VTRUNCSTOREUS)
33225   NODE_NAME_CASE(VMTRUNCSTORES)
33226   NODE_NAME_CASE(VMTRUNCSTOREUS)
33227   NODE_NAME_CASE(VFPEXT)
33228   NODE_NAME_CASE(STRICT_VFPEXT)
33229   NODE_NAME_CASE(VFPEXT_SAE)
33230   NODE_NAME_CASE(VFPEXTS)
33231   NODE_NAME_CASE(VFPEXTS_SAE)
33232   NODE_NAME_CASE(VFPROUND)
33233   NODE_NAME_CASE(STRICT_VFPROUND)
33234   NODE_NAME_CASE(VMFPROUND)
33235   NODE_NAME_CASE(VFPROUND_RND)
33236   NODE_NAME_CASE(VFPROUNDS)
33237   NODE_NAME_CASE(VFPROUNDS_RND)
33238   NODE_NAME_CASE(VSHLDQ)
33239   NODE_NAME_CASE(VSRLDQ)
33240   NODE_NAME_CASE(VSHL)
33241   NODE_NAME_CASE(VSRL)
33242   NODE_NAME_CASE(VSRA)
33243   NODE_NAME_CASE(VSHLI)
33244   NODE_NAME_CASE(VSRLI)
33245   NODE_NAME_CASE(VSRAI)
33246   NODE_NAME_CASE(VSHLV)
33247   NODE_NAME_CASE(VSRLV)
33248   NODE_NAME_CASE(VSRAV)
33249   NODE_NAME_CASE(VROTLI)
33250   NODE_NAME_CASE(VROTRI)
33251   NODE_NAME_CASE(VPPERM)
33252   NODE_NAME_CASE(CMPP)
33253   NODE_NAME_CASE(STRICT_CMPP)
33254   NODE_NAME_CASE(PCMPEQ)
33255   NODE_NAME_CASE(PCMPGT)
33256   NODE_NAME_CASE(PHMINPOS)
33257   NODE_NAME_CASE(ADD)
33258   NODE_NAME_CASE(SUB)
33259   NODE_NAME_CASE(ADC)
33260   NODE_NAME_CASE(SBB)
33261   NODE_NAME_CASE(SMUL)
33262   NODE_NAME_CASE(UMUL)
33263   NODE_NAME_CASE(OR)
33264   NODE_NAME_CASE(XOR)
33265   NODE_NAME_CASE(AND)
33266   NODE_NAME_CASE(BEXTR)
33267   NODE_NAME_CASE(BEXTRI)
33268   NODE_NAME_CASE(BZHI)
33269   NODE_NAME_CASE(PDEP)
33270   NODE_NAME_CASE(PEXT)
33271   NODE_NAME_CASE(MUL_IMM)
33272   NODE_NAME_CASE(MOVMSK)
33273   NODE_NAME_CASE(PTEST)
33274   NODE_NAME_CASE(TESTP)
33275   NODE_NAME_CASE(KORTEST)
33276   NODE_NAME_CASE(KTEST)
33277   NODE_NAME_CASE(KADD)
33278   NODE_NAME_CASE(KSHIFTL)
33279   NODE_NAME_CASE(KSHIFTR)
33280   NODE_NAME_CASE(PACKSS)
33281   NODE_NAME_CASE(PACKUS)
33282   NODE_NAME_CASE(PALIGNR)
33283   NODE_NAME_CASE(VALIGN)
33284   NODE_NAME_CASE(VSHLD)
33285   NODE_NAME_CASE(VSHRD)
33286   NODE_NAME_CASE(VSHLDV)
33287   NODE_NAME_CASE(VSHRDV)
33288   NODE_NAME_CASE(PSHUFD)
33289   NODE_NAME_CASE(PSHUFHW)
33290   NODE_NAME_CASE(PSHUFLW)
33291   NODE_NAME_CASE(SHUFP)
33292   NODE_NAME_CASE(SHUF128)
33293   NODE_NAME_CASE(MOVLHPS)
33294   NODE_NAME_CASE(MOVHLPS)
33295   NODE_NAME_CASE(MOVDDUP)
33296   NODE_NAME_CASE(MOVSHDUP)
33297   NODE_NAME_CASE(MOVSLDUP)
33298   NODE_NAME_CASE(MOVSD)
33299   NODE_NAME_CASE(MOVSS)
33300   NODE_NAME_CASE(MOVSH)
33301   NODE_NAME_CASE(UNPCKL)
33302   NODE_NAME_CASE(UNPCKH)
33303   NODE_NAME_CASE(VBROADCAST)
33304   NODE_NAME_CASE(VBROADCAST_LOAD)
33305   NODE_NAME_CASE(VBROADCASTM)
33306   NODE_NAME_CASE(SUBV_BROADCAST_LOAD)
33307   NODE_NAME_CASE(VPERMILPV)
33308   NODE_NAME_CASE(VPERMILPI)
33309   NODE_NAME_CASE(VPERM2X128)
33310   NODE_NAME_CASE(VPERMV)
33311   NODE_NAME_CASE(VPERMV3)
33312   NODE_NAME_CASE(VPERMI)
33313   NODE_NAME_CASE(VPTERNLOG)
33314   NODE_NAME_CASE(VFIXUPIMM)
33315   NODE_NAME_CASE(VFIXUPIMM_SAE)
33316   NODE_NAME_CASE(VFIXUPIMMS)
33317   NODE_NAME_CASE(VFIXUPIMMS_SAE)
33318   NODE_NAME_CASE(VRANGE)
33319   NODE_NAME_CASE(VRANGE_SAE)
33320   NODE_NAME_CASE(VRANGES)
33321   NODE_NAME_CASE(VRANGES_SAE)
33322   NODE_NAME_CASE(PMULUDQ)
33323   NODE_NAME_CASE(PMULDQ)
33324   NODE_NAME_CASE(PSADBW)
33325   NODE_NAME_CASE(DBPSADBW)
33326   NODE_NAME_CASE(VASTART_SAVE_XMM_REGS)
33327   NODE_NAME_CASE(VAARG_64)
33328   NODE_NAME_CASE(VAARG_X32)
33329   NODE_NAME_CASE(DYN_ALLOCA)
33330   NODE_NAME_CASE(MFENCE)
33331   NODE_NAME_CASE(SEG_ALLOCA)
33332   NODE_NAME_CASE(PROBED_ALLOCA)
33333   NODE_NAME_CASE(RDRAND)
33334   NODE_NAME_CASE(RDSEED)
33335   NODE_NAME_CASE(RDPKRU)
33336   NODE_NAME_CASE(WRPKRU)
33337   NODE_NAME_CASE(VPMADDUBSW)
33338   NODE_NAME_CASE(VPMADDWD)
33339   NODE_NAME_CASE(VPSHA)
33340   NODE_NAME_CASE(VPSHL)
33341   NODE_NAME_CASE(VPCOM)
33342   NODE_NAME_CASE(VPCOMU)
33343   NODE_NAME_CASE(VPERMIL2)
33344   NODE_NAME_CASE(FMSUB)
33345   NODE_NAME_CASE(STRICT_FMSUB)
33346   NODE_NAME_CASE(FNMADD)
33347   NODE_NAME_CASE(STRICT_FNMADD)
33348   NODE_NAME_CASE(FNMSUB)
33349   NODE_NAME_CASE(STRICT_FNMSUB)
33350   NODE_NAME_CASE(FMADDSUB)
33351   NODE_NAME_CASE(FMSUBADD)
33352   NODE_NAME_CASE(FMADD_RND)
33353   NODE_NAME_CASE(FNMADD_RND)
33354   NODE_NAME_CASE(FMSUB_RND)
33355   NODE_NAME_CASE(FNMSUB_RND)
33356   NODE_NAME_CASE(FMADDSUB_RND)
33357   NODE_NAME_CASE(FMSUBADD_RND)
33358   NODE_NAME_CASE(VFMADDC)
33359   NODE_NAME_CASE(VFMADDC_RND)
33360   NODE_NAME_CASE(VFCMADDC)
33361   NODE_NAME_CASE(VFCMADDC_RND)
33362   NODE_NAME_CASE(VFMULC)
33363   NODE_NAME_CASE(VFMULC_RND)
33364   NODE_NAME_CASE(VFCMULC)
33365   NODE_NAME_CASE(VFCMULC_RND)
33366   NODE_NAME_CASE(VFMULCSH)
33367   NODE_NAME_CASE(VFMULCSH_RND)
33368   NODE_NAME_CASE(VFCMULCSH)
33369   NODE_NAME_CASE(VFCMULCSH_RND)
33370   NODE_NAME_CASE(VFMADDCSH)
33371   NODE_NAME_CASE(VFMADDCSH_RND)
33372   NODE_NAME_CASE(VFCMADDCSH)
33373   NODE_NAME_CASE(VFCMADDCSH_RND)
33374   NODE_NAME_CASE(VPMADD52H)
33375   NODE_NAME_CASE(VPMADD52L)
33376   NODE_NAME_CASE(VRNDSCALE)
33377   NODE_NAME_CASE(STRICT_VRNDSCALE)
33378   NODE_NAME_CASE(VRNDSCALE_SAE)
33379   NODE_NAME_CASE(VRNDSCALES)
33380   NODE_NAME_CASE(VRNDSCALES_SAE)
33381   NODE_NAME_CASE(VREDUCE)
33382   NODE_NAME_CASE(VREDUCE_SAE)
33383   NODE_NAME_CASE(VREDUCES)
33384   NODE_NAME_CASE(VREDUCES_SAE)
33385   NODE_NAME_CASE(VGETMANT)
33386   NODE_NAME_CASE(VGETMANT_SAE)
33387   NODE_NAME_CASE(VGETMANTS)
33388   NODE_NAME_CASE(VGETMANTS_SAE)
33389   NODE_NAME_CASE(PCMPESTR)
33390   NODE_NAME_CASE(PCMPISTR)
33391   NODE_NAME_CASE(XTEST)
33392   NODE_NAME_CASE(COMPRESS)
33393   NODE_NAME_CASE(EXPAND)
33394   NODE_NAME_CASE(SELECTS)
33395   NODE_NAME_CASE(ADDSUB)
33396   NODE_NAME_CASE(RCP14)
33397   NODE_NAME_CASE(RCP14S)
33398   NODE_NAME_CASE(RCP28)
33399   NODE_NAME_CASE(RCP28_SAE)
33400   NODE_NAME_CASE(RCP28S)
33401   NODE_NAME_CASE(RCP28S_SAE)
33402   NODE_NAME_CASE(EXP2)
33403   NODE_NAME_CASE(EXP2_SAE)
33404   NODE_NAME_CASE(RSQRT14)
33405   NODE_NAME_CASE(RSQRT14S)
33406   NODE_NAME_CASE(RSQRT28)
33407   NODE_NAME_CASE(RSQRT28_SAE)
33408   NODE_NAME_CASE(RSQRT28S)
33409   NODE_NAME_CASE(RSQRT28S_SAE)
33410   NODE_NAME_CASE(FADD_RND)
33411   NODE_NAME_CASE(FADDS)
33412   NODE_NAME_CASE(FADDS_RND)
33413   NODE_NAME_CASE(FSUB_RND)
33414   NODE_NAME_CASE(FSUBS)
33415   NODE_NAME_CASE(FSUBS_RND)
33416   NODE_NAME_CASE(FMUL_RND)
33417   NODE_NAME_CASE(FMULS)
33418   NODE_NAME_CASE(FMULS_RND)
33419   NODE_NAME_CASE(FDIV_RND)
33420   NODE_NAME_CASE(FDIVS)
33421   NODE_NAME_CASE(FDIVS_RND)
33422   NODE_NAME_CASE(FSQRT_RND)
33423   NODE_NAME_CASE(FSQRTS)
33424   NODE_NAME_CASE(FSQRTS_RND)
33425   NODE_NAME_CASE(FGETEXP)
33426   NODE_NAME_CASE(FGETEXP_SAE)
33427   NODE_NAME_CASE(FGETEXPS)
33428   NODE_NAME_CASE(FGETEXPS_SAE)
33429   NODE_NAME_CASE(SCALEF)
33430   NODE_NAME_CASE(SCALEF_RND)
33431   NODE_NAME_CASE(SCALEFS)
33432   NODE_NAME_CASE(SCALEFS_RND)
33433   NODE_NAME_CASE(MULHRS)
33434   NODE_NAME_CASE(SINT_TO_FP_RND)
33435   NODE_NAME_CASE(UINT_TO_FP_RND)
33436   NODE_NAME_CASE(CVTTP2SI)
33437   NODE_NAME_CASE(CVTTP2UI)
33438   NODE_NAME_CASE(STRICT_CVTTP2SI)
33439   NODE_NAME_CASE(STRICT_CVTTP2UI)
33440   NODE_NAME_CASE(MCVTTP2SI)
33441   NODE_NAME_CASE(MCVTTP2UI)
33442   NODE_NAME_CASE(CVTTP2SI_SAE)
33443   NODE_NAME_CASE(CVTTP2UI_SAE)
33444   NODE_NAME_CASE(CVTTS2SI)
33445   NODE_NAME_CASE(CVTTS2UI)
33446   NODE_NAME_CASE(CVTTS2SI_SAE)
33447   NODE_NAME_CASE(CVTTS2UI_SAE)
33448   NODE_NAME_CASE(CVTSI2P)
33449   NODE_NAME_CASE(CVTUI2P)
33450   NODE_NAME_CASE(STRICT_CVTSI2P)
33451   NODE_NAME_CASE(STRICT_CVTUI2P)
33452   NODE_NAME_CASE(MCVTSI2P)
33453   NODE_NAME_CASE(MCVTUI2P)
33454   NODE_NAME_CASE(VFPCLASS)
33455   NODE_NAME_CASE(VFPCLASSS)
33456   NODE_NAME_CASE(MULTISHIFT)
33457   NODE_NAME_CASE(SCALAR_SINT_TO_FP)
33458   NODE_NAME_CASE(SCALAR_SINT_TO_FP_RND)
33459   NODE_NAME_CASE(SCALAR_UINT_TO_FP)
33460   NODE_NAME_CASE(SCALAR_UINT_TO_FP_RND)
33461   NODE_NAME_CASE(CVTPS2PH)
33462   NODE_NAME_CASE(STRICT_CVTPS2PH)
33463   NODE_NAME_CASE(CVTPS2PH_SAE)
33464   NODE_NAME_CASE(MCVTPS2PH)
33465   NODE_NAME_CASE(MCVTPS2PH_SAE)
33466   NODE_NAME_CASE(CVTPH2PS)
33467   NODE_NAME_CASE(STRICT_CVTPH2PS)
33468   NODE_NAME_CASE(CVTPH2PS_SAE)
33469   NODE_NAME_CASE(CVTP2SI)
33470   NODE_NAME_CASE(CVTP2UI)
33471   NODE_NAME_CASE(MCVTP2SI)
33472   NODE_NAME_CASE(MCVTP2UI)
33473   NODE_NAME_CASE(CVTP2SI_RND)
33474   NODE_NAME_CASE(CVTP2UI_RND)
33475   NODE_NAME_CASE(CVTS2SI)
33476   NODE_NAME_CASE(CVTS2UI)
33477   NODE_NAME_CASE(CVTS2SI_RND)
33478   NODE_NAME_CASE(CVTS2UI_RND)
33479   NODE_NAME_CASE(CVTNE2PS2BF16)
33480   NODE_NAME_CASE(CVTNEPS2BF16)
33481   NODE_NAME_CASE(MCVTNEPS2BF16)
33482   NODE_NAME_CASE(DPBF16PS)
33483   NODE_NAME_CASE(LWPINS)
33484   NODE_NAME_CASE(MGATHER)
33485   NODE_NAME_CASE(MSCATTER)
33486   NODE_NAME_CASE(VPDPBUSD)
33487   NODE_NAME_CASE(VPDPBUSDS)
33488   NODE_NAME_CASE(VPDPWSSD)
33489   NODE_NAME_CASE(VPDPWSSDS)
33490   NODE_NAME_CASE(VPSHUFBITQMB)
33491   NODE_NAME_CASE(GF2P8MULB)
33492   NODE_NAME_CASE(GF2P8AFFINEQB)
33493   NODE_NAME_CASE(GF2P8AFFINEINVQB)
33494   NODE_NAME_CASE(NT_CALL)
33495   NODE_NAME_CASE(NT_BRIND)
33496   NODE_NAME_CASE(UMWAIT)
33497   NODE_NAME_CASE(TPAUSE)
33498   NODE_NAME_CASE(ENQCMD)
33499   NODE_NAME_CASE(ENQCMDS)
33500   NODE_NAME_CASE(VP2INTERSECT)
33501   NODE_NAME_CASE(VPDPBSUD)
33502   NODE_NAME_CASE(VPDPBSUDS)
33503   NODE_NAME_CASE(VPDPBUUD)
33504   NODE_NAME_CASE(VPDPBUUDS)
33505   NODE_NAME_CASE(VPDPBSSD)
33506   NODE_NAME_CASE(VPDPBSSDS)
33507   NODE_NAME_CASE(AESENC128KL)
33508   NODE_NAME_CASE(AESDEC128KL)
33509   NODE_NAME_CASE(AESENC256KL)
33510   NODE_NAME_CASE(AESDEC256KL)
33511   NODE_NAME_CASE(AESENCWIDE128KL)
33512   NODE_NAME_CASE(AESDECWIDE128KL)
33513   NODE_NAME_CASE(AESENCWIDE256KL)
33514   NODE_NAME_CASE(AESDECWIDE256KL)
33515   NODE_NAME_CASE(CMPCCXADD)
33516   NODE_NAME_CASE(TESTUI)
33517   NODE_NAME_CASE(FP80_ADD)
33518   NODE_NAME_CASE(STRICT_FP80_ADD)
33519   }
33520   return nullptr;
33521 #undef NODE_NAME_CASE
33522 }
33523 
33524 /// Return true if the addressing mode represented by AM is legal for this
33525 /// target, for a load/store of the specified type.
33526 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
33527                                               const AddrMode &AM, Type *Ty,
33528                                               unsigned AS,
33529                                               Instruction *I) const {
33530   // X86 supports extremely general addressing modes.
33531   CodeModel::Model M = getTargetMachine().getCodeModel();
33532 
33533   // X86 allows a sign-extended 32-bit immediate field as a displacement.
33534   if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
33535     return false;
33536 
33537   if (AM.BaseGV) {
33538     unsigned GVFlags = Subtarget.classifyGlobalReference(AM.BaseGV);
33539 
33540     // If a reference to this global requires an extra load, we can't fold it.
33541     if (isGlobalStubReference(GVFlags))
33542       return false;
33543 
33544     // If BaseGV requires a register for the PIC base, we cannot also have a
33545     // BaseReg specified.
33546     if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
33547       return false;
33548 
33549     // If lower 4G is not available, then we must use rip-relative addressing.
33550     if ((M != CodeModel::Small || isPositionIndependent()) &&
33551         Subtarget.is64Bit() && (AM.BaseOffs || AM.Scale > 1))
33552       return false;
33553   }
33554 
33555   switch (AM.Scale) {
33556   case 0:
33557   case 1:
33558   case 2:
33559   case 4:
33560   case 8:
33561     // These scales always work.
33562     break;
33563   case 3:
33564   case 5:
33565   case 9:
33566     // These scales are formed with basereg+scalereg.  Only accept if there is
33567     // no basereg yet.
33568     if (AM.HasBaseReg)
33569       return false;
33570     break;
33571   default:  // Other stuff never works.
33572     return false;
33573   }
33574 
33575   return true;
33576 }
33577 
33578 bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
33579   unsigned Bits = Ty->getScalarSizeInBits();
33580 
33581   // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
33582   // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
33583   if (Subtarget.hasXOP() &&
33584       (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
33585     return false;
33586 
33587   // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
33588   // shifts just as cheap as scalar ones.
33589   if (Subtarget.hasAVX2() && (Bits == 32 || Bits == 64))
33590     return false;
33591 
33592   // AVX512BW has shifts such as vpsllvw.
33593   if (Subtarget.hasBWI() && Bits == 16)
33594     return false;
33595 
33596   // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
33597   // fully general vector.
33598   return true;
33599 }
33600 
33601 bool X86TargetLowering::isBinOp(unsigned Opcode) const {
33602   switch (Opcode) {
33603   // These are non-commutative binops.
33604   // TODO: Add more X86ISD opcodes once we have test coverage.
33605   case X86ISD::ANDNP:
33606   case X86ISD::PCMPGT:
33607   case X86ISD::FMAX:
33608   case X86ISD::FMIN:
33609   case X86ISD::FANDN:
33610   case X86ISD::VPSHA:
33611   case X86ISD::VPSHL:
33612   case X86ISD::VSHLV:
33613   case X86ISD::VSRLV:
33614   case X86ISD::VSRAV:
33615     return true;
33616   }
33617 
33618   return TargetLoweringBase::isBinOp(Opcode);
33619 }
33620 
33621 bool X86TargetLowering::isCommutativeBinOp(unsigned Opcode) const {
33622   switch (Opcode) {
33623   // TODO: Add more X86ISD opcodes once we have test coverage.
33624   case X86ISD::PCMPEQ:
33625   case X86ISD::PMULDQ:
33626   case X86ISD::PMULUDQ:
33627   case X86ISD::FMAXC:
33628   case X86ISD::FMINC:
33629   case X86ISD::FAND:
33630   case X86ISD::FOR:
33631   case X86ISD::FXOR:
33632     return true;
33633   }
33634 
33635   return TargetLoweringBase::isCommutativeBinOp(Opcode);
33636 }
33637 
33638 bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
33639   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
33640     return false;
33641   unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
33642   unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
33643   return NumBits1 > NumBits2;
33644 }
33645 
33646 bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
33647   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
33648     return false;
33649 
33650   if (!isTypeLegal(EVT::getEVT(Ty1)))
33651     return false;
33652 
33653   assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
33654 
33655   // Assuming the caller doesn't have a zeroext or signext return parameter,
33656   // truncation all the way down to i1 is valid.
33657   return true;
33658 }
33659 
33660 bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
33661   return isInt<32>(Imm);
33662 }
33663 
33664 bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
33665   // Can also use sub to handle negated immediates.
33666   return isInt<32>(Imm);
33667 }
33668 
33669 bool X86TargetLowering::isLegalStoreImmediate(int64_t Imm) const {
33670   return isInt<32>(Imm);
33671 }
33672 
33673 bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
33674   if (!VT1.isScalarInteger() || !VT2.isScalarInteger())
33675     return false;
33676   unsigned NumBits1 = VT1.getSizeInBits();
33677   unsigned NumBits2 = VT2.getSizeInBits();
33678   return NumBits1 > NumBits2;
33679 }
33680 
33681 bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
33682   // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
33683   return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit();
33684 }
33685 
33686 bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
33687   // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
33688   return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit();
33689 }
33690 
33691 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
33692   EVT VT1 = Val.getValueType();
33693   if (isZExtFree(VT1, VT2))
33694     return true;
33695 
33696   if (Val.getOpcode() != ISD::LOAD)
33697     return false;
33698 
33699   if (!VT1.isSimple() || !VT1.isInteger() ||
33700       !VT2.isSimple() || !VT2.isInteger())
33701     return false;
33702 
33703   switch (VT1.getSimpleVT().SimpleTy) {
33704   default: break;
33705   case MVT::i8:
33706   case MVT::i16:
33707   case MVT::i32:
33708     // X86 has 8, 16, and 32-bit zero-extending loads.
33709     return true;
33710   }
33711 
33712   return false;
33713 }
33714 
33715 bool X86TargetLowering::shouldSinkOperands(Instruction *I,
33716                                            SmallVectorImpl<Use *> &Ops) const {
33717   using namespace llvm::PatternMatch;
33718 
33719   FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType());
33720   if (!VTy)
33721     return false;
33722 
33723   if (I->getOpcode() == Instruction::Mul &&
33724       VTy->getElementType()->isIntegerTy(64)) {
33725     for (auto &Op : I->operands()) {
33726       // Make sure we are not already sinking this operand
33727       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
33728         continue;
33729 
33730       // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
33731       // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
33732       if (Subtarget.hasSSE41() &&
33733           match(Op.get(), m_AShr(m_Shl(m_Value(), m_SpecificInt(32)),
33734                                  m_SpecificInt(32)))) {
33735         Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
33736         Ops.push_back(&Op);
33737       } else if (Subtarget.hasSSE2() &&
33738                  match(Op.get(),
33739                        m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))))) {
33740         Ops.push_back(&Op);
33741       }
33742     }
33743 
33744     return !Ops.empty();
33745   }
33746 
33747   // A uniform shift amount in a vector shift or funnel shift may be much
33748   // cheaper than a generic variable vector shift, so make that pattern visible
33749   // to SDAG by sinking the shuffle instruction next to the shift.
33750   int ShiftAmountOpNum = -1;
33751   if (I->isShift())
33752     ShiftAmountOpNum = 1;
33753   else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
33754     if (II->getIntrinsicID() == Intrinsic::fshl ||
33755         II->getIntrinsicID() == Intrinsic::fshr)
33756       ShiftAmountOpNum = 2;
33757   }
33758 
33759   if (ShiftAmountOpNum == -1)
33760     return false;
33761 
33762   auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum));
33763   if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 &&
33764       isVectorShiftByScalarCheap(I->getType())) {
33765     Ops.push_back(&I->getOperandUse(ShiftAmountOpNum));
33766     return true;
33767   }
33768 
33769   return false;
33770 }
33771 
33772 bool X86TargetLowering::shouldConvertPhiType(Type *From, Type *To) const {
33773   if (!Subtarget.is64Bit())
33774     return false;
33775   return TargetLowering::shouldConvertPhiType(From, To);
33776 }
33777 
33778 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
33779   if (isa<MaskedLoadSDNode>(ExtVal.getOperand(0)))
33780     return false;
33781 
33782   EVT SrcVT = ExtVal.getOperand(0).getValueType();
33783 
33784   // There is no extending load for vXi1.
33785   if (SrcVT.getScalarType() == MVT::i1)
33786     return false;
33787 
33788   return true;
33789 }
33790 
33791 bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
33792                                                    EVT VT) const {
33793   if (!Subtarget.hasAnyFMA())
33794     return false;
33795 
33796   VT = VT.getScalarType();
33797 
33798   if (!VT.isSimple())
33799     return false;
33800 
33801   switch (VT.getSimpleVT().SimpleTy) {
33802   case MVT::f16:
33803     return Subtarget.hasFP16();
33804   case MVT::f32:
33805   case MVT::f64:
33806     return true;
33807   default:
33808     break;
33809   }
33810 
33811   return false;
33812 }
33813 
33814 bool X86TargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
33815   // i16 instructions are longer (0x66 prefix) and potentially slower.
33816   return !(SrcVT == MVT::i32 && DestVT == MVT::i16);
33817 }
33818 
33819 bool X86TargetLowering::shouldFoldSelectWithIdentityConstant(unsigned Opcode,
33820                                                              EVT VT) const {
33821   // TODO: This is too general. There are cases where pre-AVX512 codegen would
33822   //       benefit. The transform may also be profitable for scalar code.
33823   if (!Subtarget.hasAVX512())
33824     return false;
33825   if (!Subtarget.hasVLX() && !VT.is512BitVector())
33826     return false;
33827   if (!VT.isVector() || VT.getScalarType() == MVT::i1)
33828     return false;
33829 
33830   return true;
33831 }
33832 
33833 /// Targets can use this to indicate that they only support *some*
33834 /// VECTOR_SHUFFLE operations, those with specific masks.
33835 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
33836 /// are assumed to be legal.
33837 bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
33838   if (!VT.isSimple())
33839     return false;
33840 
33841   // Not for i1 vectors
33842   if (VT.getSimpleVT().getScalarType() == MVT::i1)
33843     return false;
33844 
33845   // Very little shuffling can be done for 64-bit vectors right now.
33846   if (VT.getSimpleVT().getSizeInBits() == 64)
33847     return false;
33848 
33849   // We only care that the types being shuffled are legal. The lowering can
33850   // handle any possible shuffle mask that results.
33851   return isTypeLegal(VT.getSimpleVT());
33852 }
33853 
33854 bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
33855                                                EVT VT) const {
33856   // Don't convert an 'and' into a shuffle that we don't directly support.
33857   // vpblendw and vpshufb for 256-bit vectors are not available on AVX1.
33858   if (!Subtarget.hasAVX2())
33859     if (VT == MVT::v32i8 || VT == MVT::v16i16)
33860       return false;
33861 
33862   // Just delegate to the generic legality, clear masks aren't special.
33863   return isShuffleMaskLegal(Mask, VT);
33864 }
33865 
33866 bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
33867   // If the subtarget is using thunks, we need to not generate jump tables.
33868   if (Subtarget.useIndirectThunkBranches())
33869     return false;
33870 
33871   // Otherwise, fallback on the generic logic.
33872   return TargetLowering::areJTsAllowed(Fn);
33873 }
33874 
33875 MVT X86TargetLowering::getPreferredSwitchConditionType(LLVMContext &Context,
33876                                                        EVT ConditionVT) const {
33877   // Avoid 8 and 16 bit types because they increase the chance for unnecessary
33878   // zero-extensions.
33879   if (ConditionVT.getSizeInBits() < 32)
33880     return MVT::i32;
33881   return TargetLoweringBase::getPreferredSwitchConditionType(Context,
33882                                                              ConditionVT);
33883 }
33884 
33885 //===----------------------------------------------------------------------===//
33886 //                           X86 Scheduler Hooks
33887 //===----------------------------------------------------------------------===//
33888 
33889 // Returns true if EFLAG is consumed after this iterator in the rest of the
33890 // basic block or any successors of the basic block.
33891 static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr,
33892                               MachineBasicBlock *BB) {
33893   // Scan forward through BB for a use/def of EFLAGS.
33894   for (const MachineInstr &mi : llvm::make_range(std::next(Itr), BB->end())) {
33895     if (mi.readsRegister(X86::EFLAGS))
33896       return true;
33897     // If we found a def, we can stop searching.
33898     if (mi.definesRegister(X86::EFLAGS))
33899       return false;
33900   }
33901 
33902   // If we hit the end of the block, check whether EFLAGS is live into a
33903   // successor.
33904   for (MachineBasicBlock *Succ : BB->successors())
33905     if (Succ->isLiveIn(X86::EFLAGS))
33906       return true;
33907 
33908   return false;
33909 }
33910 
33911 /// Utility function to emit xbegin specifying the start of an RTM region.
33912 static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
33913                                      const TargetInstrInfo *TII) {
33914   const MIMetadata MIMD(MI);
33915 
33916   const BasicBlock *BB = MBB->getBasicBlock();
33917   MachineFunction::iterator I = ++MBB->getIterator();
33918 
33919   // For the v = xbegin(), we generate
33920   //
33921   // thisMBB:
33922   //  xbegin sinkMBB
33923   //
33924   // mainMBB:
33925   //  s0 = -1
33926   //
33927   // fallBB:
33928   //  eax = # XABORT_DEF
33929   //  s1 = eax
33930   //
33931   // sinkMBB:
33932   //  v = phi(s0/mainBB, s1/fallBB)
33933 
33934   MachineBasicBlock *thisMBB = MBB;
33935   MachineFunction *MF = MBB->getParent();
33936   MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
33937   MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
33938   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
33939   MF->insert(I, mainMBB);
33940   MF->insert(I, fallMBB);
33941   MF->insert(I, sinkMBB);
33942 
33943   if (isEFLAGSLiveAfter(MI, MBB)) {
33944     mainMBB->addLiveIn(X86::EFLAGS);
33945     fallMBB->addLiveIn(X86::EFLAGS);
33946     sinkMBB->addLiveIn(X86::EFLAGS);
33947   }
33948 
33949   // Transfer the remainder of BB and its successor edges to sinkMBB.
33950   sinkMBB->splice(sinkMBB->begin(), MBB,
33951                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
33952   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
33953 
33954   MachineRegisterInfo &MRI = MF->getRegInfo();
33955   Register DstReg = MI.getOperand(0).getReg();
33956   const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
33957   Register mainDstReg = MRI.createVirtualRegister(RC);
33958   Register fallDstReg = MRI.createVirtualRegister(RC);
33959 
33960   // thisMBB:
33961   //  xbegin fallMBB
33962   //  # fallthrough to mainMBB
33963   //  # abortion to fallMBB
33964   BuildMI(thisMBB, MIMD, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
33965   thisMBB->addSuccessor(mainMBB);
33966   thisMBB->addSuccessor(fallMBB);
33967 
33968   // mainMBB:
33969   //  mainDstReg := -1
33970   BuildMI(mainMBB, MIMD, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
33971   BuildMI(mainMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
33972   mainMBB->addSuccessor(sinkMBB);
33973 
33974   // fallMBB:
33975   //  ; pseudo instruction to model hardware's definition from XABORT
33976   //  EAX := XABORT_DEF
33977   //  fallDstReg := EAX
33978   BuildMI(fallMBB, MIMD, TII->get(X86::XABORT_DEF));
33979   BuildMI(fallMBB, MIMD, TII->get(TargetOpcode::COPY), fallDstReg)
33980       .addReg(X86::EAX);
33981   fallMBB->addSuccessor(sinkMBB);
33982 
33983   // sinkMBB:
33984   //  DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
33985   BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
33986       .addReg(mainDstReg).addMBB(mainMBB)
33987       .addReg(fallDstReg).addMBB(fallMBB);
33988 
33989   MI.eraseFromParent();
33990   return sinkMBB;
33991 }
33992 
33993 MachineBasicBlock *
33994 X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
33995                                                MachineBasicBlock *MBB) const {
33996   // Emit va_arg instruction on X86-64.
33997 
33998   // Operands to this pseudo-instruction:
33999   // 0  ) Output        : destination address (reg)
34000   // 1-5) Input         : va_list address (addr, i64mem)
34001   // 6  ) ArgSize       : Size (in bytes) of vararg type
34002   // 7  ) ArgMode       : 0=overflow only, 1=use gp_offset, 2=use fp_offset
34003   // 8  ) Align         : Alignment of type
34004   // 9  ) EFLAGS (implicit-def)
34005 
34006   assert(MI.getNumOperands() == 10 && "VAARG should have 10 operands!");
34007   static_assert(X86::AddrNumOperands == 5, "VAARG assumes 5 address operands");
34008 
34009   Register DestReg = MI.getOperand(0).getReg();
34010   MachineOperand &Base = MI.getOperand(1);
34011   MachineOperand &Scale = MI.getOperand(2);
34012   MachineOperand &Index = MI.getOperand(3);
34013   MachineOperand &Disp = MI.getOperand(4);
34014   MachineOperand &Segment = MI.getOperand(5);
34015   unsigned ArgSize = MI.getOperand(6).getImm();
34016   unsigned ArgMode = MI.getOperand(7).getImm();
34017   Align Alignment = Align(MI.getOperand(8).getImm());
34018 
34019   MachineFunction *MF = MBB->getParent();
34020 
34021   // Memory Reference
34022   assert(MI.hasOneMemOperand() && "Expected VAARG to have one memoperand");
34023 
34024   MachineMemOperand *OldMMO = MI.memoperands().front();
34025 
34026   // Clone the MMO into two separate MMOs for loading and storing
34027   MachineMemOperand *LoadOnlyMMO = MF->getMachineMemOperand(
34028       OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOStore);
34029   MachineMemOperand *StoreOnlyMMO = MF->getMachineMemOperand(
34030       OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOLoad);
34031 
34032   // Machine Information
34033   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34034   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
34035   const TargetRegisterClass *AddrRegClass =
34036       getRegClassFor(getPointerTy(MBB->getParent()->getDataLayout()));
34037   const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
34038   const MIMetadata MIMD(MI);
34039 
34040   // struct va_list {
34041   //   i32   gp_offset
34042   //   i32   fp_offset
34043   //   i64   overflow_area (address)
34044   //   i64   reg_save_area (address)
34045   // }
34046   // sizeof(va_list) = 24
34047   // alignment(va_list) = 8
34048 
34049   unsigned TotalNumIntRegs = 6;
34050   unsigned TotalNumXMMRegs = 8;
34051   bool UseGPOffset = (ArgMode == 1);
34052   bool UseFPOffset = (ArgMode == 2);
34053   unsigned MaxOffset = TotalNumIntRegs * 8 +
34054                        (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
34055 
34056   /* Align ArgSize to a multiple of 8 */
34057   unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
34058   bool NeedsAlign = (Alignment > 8);
34059 
34060   MachineBasicBlock *thisMBB = MBB;
34061   MachineBasicBlock *overflowMBB;
34062   MachineBasicBlock *offsetMBB;
34063   MachineBasicBlock *endMBB;
34064 
34065   unsigned OffsetDestReg = 0;    // Argument address computed by offsetMBB
34066   unsigned OverflowDestReg = 0;  // Argument address computed by overflowMBB
34067   unsigned OffsetReg = 0;
34068 
34069   if (!UseGPOffset && !UseFPOffset) {
34070     // If we only pull from the overflow region, we don't create a branch.
34071     // We don't need to alter control flow.
34072     OffsetDestReg = 0; // unused
34073     OverflowDestReg = DestReg;
34074 
34075     offsetMBB = nullptr;
34076     overflowMBB = thisMBB;
34077     endMBB = thisMBB;
34078   } else {
34079     // First emit code to check if gp_offset (or fp_offset) is below the bound.
34080     // If so, pull the argument from reg_save_area. (branch to offsetMBB)
34081     // If not, pull from overflow_area. (branch to overflowMBB)
34082     //
34083     //       thisMBB
34084     //         |     .
34085     //         |        .
34086     //     offsetMBB   overflowMBB
34087     //         |        .
34088     //         |     .
34089     //        endMBB
34090 
34091     // Registers for the PHI in endMBB
34092     OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
34093     OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
34094 
34095     const BasicBlock *LLVM_BB = MBB->getBasicBlock();
34096     overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34097     offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34098     endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34099 
34100     MachineFunction::iterator MBBIter = ++MBB->getIterator();
34101 
34102     // Insert the new basic blocks
34103     MF->insert(MBBIter, offsetMBB);
34104     MF->insert(MBBIter, overflowMBB);
34105     MF->insert(MBBIter, endMBB);
34106 
34107     // Transfer the remainder of MBB and its successor edges to endMBB.
34108     endMBB->splice(endMBB->begin(), thisMBB,
34109                    std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
34110     endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
34111 
34112     // Make offsetMBB and overflowMBB successors of thisMBB
34113     thisMBB->addSuccessor(offsetMBB);
34114     thisMBB->addSuccessor(overflowMBB);
34115 
34116     // endMBB is a successor of both offsetMBB and overflowMBB
34117     offsetMBB->addSuccessor(endMBB);
34118     overflowMBB->addSuccessor(endMBB);
34119 
34120     // Load the offset value into a register
34121     OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34122     BuildMI(thisMBB, MIMD, TII->get(X86::MOV32rm), OffsetReg)
34123         .add(Base)
34124         .add(Scale)
34125         .add(Index)
34126         .addDisp(Disp, UseFPOffset ? 4 : 0)
34127         .add(Segment)
34128         .setMemRefs(LoadOnlyMMO);
34129 
34130     // Check if there is enough room left to pull this argument.
34131     BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
34132       .addReg(OffsetReg)
34133       .addImm(MaxOffset + 8 - ArgSizeA8);
34134 
34135     // Branch to "overflowMBB" if offset >= max
34136     // Fall through to "offsetMBB" otherwise
34137     BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
34138       .addMBB(overflowMBB).addImm(X86::COND_AE);
34139   }
34140 
34141   // In offsetMBB, emit code to use the reg_save_area.
34142   if (offsetMBB) {
34143     assert(OffsetReg != 0);
34144 
34145     // Read the reg_save_area address.
34146     Register RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
34147     BuildMI(
34148         offsetMBB, MIMD,
34149         TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34150         RegSaveReg)
34151         .add(Base)
34152         .add(Scale)
34153         .add(Index)
34154         .addDisp(Disp, Subtarget.isTarget64BitLP64() ? 16 : 12)
34155         .add(Segment)
34156         .setMemRefs(LoadOnlyMMO);
34157 
34158     if (Subtarget.isTarget64BitLP64()) {
34159       // Zero-extend the offset
34160       Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
34161       BuildMI(offsetMBB, MIMD, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
34162           .addImm(0)
34163           .addReg(OffsetReg)
34164           .addImm(X86::sub_32bit);
34165 
34166       // Add the offset to the reg_save_area to get the final address.
34167       BuildMI(offsetMBB, MIMD, TII->get(X86::ADD64rr), OffsetDestReg)
34168           .addReg(OffsetReg64)
34169           .addReg(RegSaveReg);
34170     } else {
34171       // Add the offset to the reg_save_area to get the final address.
34172       BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32rr), OffsetDestReg)
34173           .addReg(OffsetReg)
34174           .addReg(RegSaveReg);
34175     }
34176 
34177     // Compute the offset for the next argument
34178     Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34179     BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
34180       .addReg(OffsetReg)
34181       .addImm(UseFPOffset ? 16 : 8);
34182 
34183     // Store it back into the va_list.
34184     BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
34185         .add(Base)
34186         .add(Scale)
34187         .add(Index)
34188         .addDisp(Disp, UseFPOffset ? 4 : 0)
34189         .add(Segment)
34190         .addReg(NextOffsetReg)
34191         .setMemRefs(StoreOnlyMMO);
34192 
34193     // Jump to endMBB
34194     BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
34195       .addMBB(endMBB);
34196   }
34197 
34198   //
34199   // Emit code to use overflow area
34200   //
34201 
34202   // Load the overflow_area address into a register.
34203   Register OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
34204   BuildMI(overflowMBB, MIMD,
34205           TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34206           OverflowAddrReg)
34207       .add(Base)
34208       .add(Scale)
34209       .add(Index)
34210       .addDisp(Disp, 8)
34211       .add(Segment)
34212       .setMemRefs(LoadOnlyMMO);
34213 
34214   // If we need to align it, do so. Otherwise, just copy the address
34215   // to OverflowDestReg.
34216   if (NeedsAlign) {
34217     // Align the overflow address
34218     Register TmpReg = MRI.createVirtualRegister(AddrRegClass);
34219 
34220     // aligned_addr = (addr + (align-1)) & ~(align-1)
34221     BuildMI(
34222         overflowMBB, MIMD,
34223         TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34224         TmpReg)
34225         .addReg(OverflowAddrReg)
34226         .addImm(Alignment.value() - 1);
34227 
34228     BuildMI(
34229         overflowMBB, MIMD,
34230         TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri),
34231         OverflowDestReg)
34232         .addReg(TmpReg)
34233         .addImm(~(uint64_t)(Alignment.value() - 1));
34234   } else {
34235     BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg)
34236       .addReg(OverflowAddrReg);
34237   }
34238 
34239   // Compute the next overflow address after this argument.
34240   // (the overflow address should be kept 8-byte aligned)
34241   Register NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
34242   BuildMI(
34243       overflowMBB, MIMD,
34244       TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34245       NextAddrReg)
34246       .addReg(OverflowDestReg)
34247       .addImm(ArgSizeA8);
34248 
34249   // Store the new overflow address.
34250   BuildMI(overflowMBB, MIMD,
34251           TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr))
34252       .add(Base)
34253       .add(Scale)
34254       .add(Index)
34255       .addDisp(Disp, 8)
34256       .add(Segment)
34257       .addReg(NextAddrReg)
34258       .setMemRefs(StoreOnlyMMO);
34259 
34260   // If we branched, emit the PHI to the front of endMBB.
34261   if (offsetMBB) {
34262     BuildMI(*endMBB, endMBB->begin(), MIMD,
34263             TII->get(X86::PHI), DestReg)
34264       .addReg(OffsetDestReg).addMBB(offsetMBB)
34265       .addReg(OverflowDestReg).addMBB(overflowMBB);
34266   }
34267 
34268   // Erase the pseudo instruction
34269   MI.eraseFromParent();
34270 
34271   return endMBB;
34272 }
34273 
34274 // The EFLAGS operand of SelectItr might be missing a kill marker
34275 // because there were multiple uses of EFLAGS, and ISel didn't know
34276 // which to mark. Figure out whether SelectItr should have had a
34277 // kill marker, and set it if it should. Returns the correct kill
34278 // marker value.
34279 static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
34280                                      MachineBasicBlock* BB,
34281                                      const TargetRegisterInfo* TRI) {
34282   if (isEFLAGSLiveAfter(SelectItr, BB))
34283     return false;
34284 
34285   // We found a def, or hit the end of the basic block and EFLAGS wasn't live
34286   // out. SelectMI should have a kill flag on EFLAGS.
34287   SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
34288   return true;
34289 }
34290 
34291 // Return true if it is OK for this CMOV pseudo-opcode to be cascaded
34292 // together with other CMOV pseudo-opcodes into a single basic-block with
34293 // conditional jump around it.
34294 static bool isCMOVPseudo(MachineInstr &MI) {
34295   switch (MI.getOpcode()) {
34296   case X86::CMOV_FR16:
34297   case X86::CMOV_FR16X:
34298   case X86::CMOV_FR32:
34299   case X86::CMOV_FR32X:
34300   case X86::CMOV_FR64:
34301   case X86::CMOV_FR64X:
34302   case X86::CMOV_GR8:
34303   case X86::CMOV_GR16:
34304   case X86::CMOV_GR32:
34305   case X86::CMOV_RFP32:
34306   case X86::CMOV_RFP64:
34307   case X86::CMOV_RFP80:
34308   case X86::CMOV_VR64:
34309   case X86::CMOV_VR128:
34310   case X86::CMOV_VR128X:
34311   case X86::CMOV_VR256:
34312   case X86::CMOV_VR256X:
34313   case X86::CMOV_VR512:
34314   case X86::CMOV_VK1:
34315   case X86::CMOV_VK2:
34316   case X86::CMOV_VK4:
34317   case X86::CMOV_VK8:
34318   case X86::CMOV_VK16:
34319   case X86::CMOV_VK32:
34320   case X86::CMOV_VK64:
34321     return true;
34322 
34323   default:
34324     return false;
34325   }
34326 }
34327 
34328 // Helper function, which inserts PHI functions into SinkMBB:
34329 //   %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
34330 // where %FalseValue(i) and %TrueValue(i) are taken from the consequent CMOVs
34331 // in [MIItBegin, MIItEnd) range. It returns the last MachineInstrBuilder for
34332 // the last PHI function inserted.
34333 static MachineInstrBuilder createPHIsForCMOVsInSinkBB(
34334     MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd,
34335     MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB,
34336     MachineBasicBlock *SinkMBB) {
34337   MachineFunction *MF = TrueMBB->getParent();
34338   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
34339   const MIMetadata MIMD(*MIItBegin);
34340 
34341   X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm());
34342   X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
34343 
34344   MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
34345 
34346   // As we are creating the PHIs, we have to be careful if there is more than
34347   // one.  Later CMOVs may reference the results of earlier CMOVs, but later
34348   // PHIs have to reference the individual true/false inputs from earlier PHIs.
34349   // That also means that PHI construction must work forward from earlier to
34350   // later, and that the code must maintain a mapping from earlier PHI's
34351   // destination registers, and the registers that went into the PHI.
34352   DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
34353   MachineInstrBuilder MIB;
34354 
34355   for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
34356     Register DestReg = MIIt->getOperand(0).getReg();
34357     Register Op1Reg = MIIt->getOperand(1).getReg();
34358     Register Op2Reg = MIIt->getOperand(2).getReg();
34359 
34360     // If this CMOV we are generating is the opposite condition from
34361     // the jump we generated, then we have to swap the operands for the
34362     // PHI that is going to be generated.
34363     if (MIIt->getOperand(3).getImm() == OppCC)
34364       std::swap(Op1Reg, Op2Reg);
34365 
34366     if (RegRewriteTable.contains(Op1Reg))
34367       Op1Reg = RegRewriteTable[Op1Reg].first;
34368 
34369     if (RegRewriteTable.contains(Op2Reg))
34370       Op2Reg = RegRewriteTable[Op2Reg].second;
34371 
34372     MIB =
34373         BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg)
34374             .addReg(Op1Reg)
34375             .addMBB(FalseMBB)
34376             .addReg(Op2Reg)
34377             .addMBB(TrueMBB);
34378 
34379     // Add this PHI to the rewrite table.
34380     RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
34381   }
34382 
34383   return MIB;
34384 }
34385 
34386 // Lower cascaded selects in form of (SecondCmov (FirstCMOV F, T, cc1), T, cc2).
34387 MachineBasicBlock *
34388 X86TargetLowering::EmitLoweredCascadedSelect(MachineInstr &FirstCMOV,
34389                                              MachineInstr &SecondCascadedCMOV,
34390                                              MachineBasicBlock *ThisMBB) const {
34391   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34392   const MIMetadata MIMD(FirstCMOV);
34393 
34394   // We lower cascaded CMOVs such as
34395   //
34396   //   (SecondCascadedCMOV (FirstCMOV F, T, cc1), T, cc2)
34397   //
34398   // to two successive branches.
34399   //
34400   // Without this, we would add a PHI between the two jumps, which ends up
34401   // creating a few copies all around. For instance, for
34402   //
34403   //    (sitofp (zext (fcmp une)))
34404   //
34405   // we would generate:
34406   //
34407   //         ucomiss %xmm1, %xmm0
34408   //         movss  <1.0f>, %xmm0
34409   //         movaps  %xmm0, %xmm1
34410   //         jne     .LBB5_2
34411   //         xorps   %xmm1, %xmm1
34412   // .LBB5_2:
34413   //         jp      .LBB5_4
34414   //         movaps  %xmm1, %xmm0
34415   // .LBB5_4:
34416   //         retq
34417   //
34418   // because this custom-inserter would have generated:
34419   //
34420   //   A
34421   //   | \
34422   //   |  B
34423   //   | /
34424   //   C
34425   //   | \
34426   //   |  D
34427   //   | /
34428   //   E
34429   //
34430   // A: X = ...; Y = ...
34431   // B: empty
34432   // C: Z = PHI [X, A], [Y, B]
34433   // D: empty
34434   // E: PHI [X, C], [Z, D]
34435   //
34436   // If we lower both CMOVs in a single step, we can instead generate:
34437   //
34438   //   A
34439   //   | \
34440   //   |  C
34441   //   | /|
34442   //   |/ |
34443   //   |  |
34444   //   |  D
34445   //   | /
34446   //   E
34447   //
34448   // A: X = ...; Y = ...
34449   // D: empty
34450   // E: PHI [X, A], [X, C], [Y, D]
34451   //
34452   // Which, in our sitofp/fcmp example, gives us something like:
34453   //
34454   //         ucomiss %xmm1, %xmm0
34455   //         movss  <1.0f>, %xmm0
34456   //         jne     .LBB5_4
34457   //         jp      .LBB5_4
34458   //         xorps   %xmm0, %xmm0
34459   // .LBB5_4:
34460   //         retq
34461   //
34462 
34463   // We lower cascaded CMOV into two successive branches to the same block.
34464   // EFLAGS is used by both, so mark it as live in the second.
34465   const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
34466   MachineFunction *F = ThisMBB->getParent();
34467   MachineBasicBlock *FirstInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
34468   MachineBasicBlock *SecondInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
34469   MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
34470 
34471   MachineFunction::iterator It = ++ThisMBB->getIterator();
34472   F->insert(It, FirstInsertedMBB);
34473   F->insert(It, SecondInsertedMBB);
34474   F->insert(It, SinkMBB);
34475 
34476   // For a cascaded CMOV, we lower it to two successive branches to
34477   // the same block (SinkMBB).  EFLAGS is used by both, so mark it as live in
34478   // the FirstInsertedMBB.
34479   FirstInsertedMBB->addLiveIn(X86::EFLAGS);
34480 
34481   // If the EFLAGS register isn't dead in the terminator, then claim that it's
34482   // live into the sink and copy blocks.
34483   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
34484   if (!SecondCascadedCMOV.killsRegister(X86::EFLAGS) &&
34485       !checkAndUpdateEFLAGSKill(SecondCascadedCMOV, ThisMBB, TRI)) {
34486     SecondInsertedMBB->addLiveIn(X86::EFLAGS);
34487     SinkMBB->addLiveIn(X86::EFLAGS);
34488   }
34489 
34490   // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
34491   SinkMBB->splice(SinkMBB->begin(), ThisMBB,
34492                   std::next(MachineBasicBlock::iterator(FirstCMOV)),
34493                   ThisMBB->end());
34494   SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
34495 
34496   // Fallthrough block for ThisMBB.
34497   ThisMBB->addSuccessor(FirstInsertedMBB);
34498   // The true block target of the first branch is always SinkMBB.
34499   ThisMBB->addSuccessor(SinkMBB);
34500   // Fallthrough block for FirstInsertedMBB.
34501   FirstInsertedMBB->addSuccessor(SecondInsertedMBB);
34502   // The true block for the branch of FirstInsertedMBB.
34503   FirstInsertedMBB->addSuccessor(SinkMBB);
34504   // This is fallthrough.
34505   SecondInsertedMBB->addSuccessor(SinkMBB);
34506 
34507   // Create the conditional branch instructions.
34508   X86::CondCode FirstCC = X86::CondCode(FirstCMOV.getOperand(3).getImm());
34509   BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
34510 
34511   X86::CondCode SecondCC =
34512       X86::CondCode(SecondCascadedCMOV.getOperand(3).getImm());
34513   BuildMI(FirstInsertedMBB, MIMD, TII->get(X86::JCC_1))
34514       .addMBB(SinkMBB)
34515       .addImm(SecondCC);
34516 
34517   //  SinkMBB:
34518   //   %Result = phi [ %FalseValue, SecondInsertedMBB ], [ %TrueValue, ThisMBB ]
34519   Register DestReg = SecondCascadedCMOV.getOperand(0).getReg();
34520   Register Op1Reg = FirstCMOV.getOperand(1).getReg();
34521   Register Op2Reg = FirstCMOV.getOperand(2).getReg();
34522   MachineInstrBuilder MIB =
34523       BuildMI(*SinkMBB, SinkMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
34524           .addReg(Op1Reg)
34525           .addMBB(SecondInsertedMBB)
34526           .addReg(Op2Reg)
34527           .addMBB(ThisMBB);
34528 
34529   // The second SecondInsertedMBB provides the same incoming value as the
34530   // FirstInsertedMBB (the True operand of the SELECT_CC/CMOV nodes).
34531   MIB.addReg(FirstCMOV.getOperand(2).getReg()).addMBB(FirstInsertedMBB);
34532 
34533   // Now remove the CMOVs.
34534   FirstCMOV.eraseFromParent();
34535   SecondCascadedCMOV.eraseFromParent();
34536 
34537   return SinkMBB;
34538 }
34539 
34540 MachineBasicBlock *
34541 X86TargetLowering::EmitLoweredSelect(MachineInstr &MI,
34542                                      MachineBasicBlock *ThisMBB) const {
34543   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34544   const MIMetadata MIMD(MI);
34545 
34546   // To "insert" a SELECT_CC instruction, we actually have to insert the
34547   // diamond control-flow pattern.  The incoming instruction knows the
34548   // destination vreg to set, the condition code register to branch on, the
34549   // true/false values to select between and a branch opcode to use.
34550 
34551   //  ThisMBB:
34552   //  ...
34553   //   TrueVal = ...
34554   //   cmpTY ccX, r1, r2
34555   //   bCC copy1MBB
34556   //   fallthrough --> FalseMBB
34557 
34558   // This code lowers all pseudo-CMOV instructions. Generally it lowers these
34559   // as described above, by inserting a BB, and then making a PHI at the join
34560   // point to select the true and false operands of the CMOV in the PHI.
34561   //
34562   // The code also handles two different cases of multiple CMOV opcodes
34563   // in a row.
34564   //
34565   // Case 1:
34566   // In this case, there are multiple CMOVs in a row, all which are based on
34567   // the same condition setting (or the exact opposite condition setting).
34568   // In this case we can lower all the CMOVs using a single inserted BB, and
34569   // then make a number of PHIs at the join point to model the CMOVs. The only
34570   // trickiness here, is that in a case like:
34571   //
34572   // t2 = CMOV cond1 t1, f1
34573   // t3 = CMOV cond1 t2, f2
34574   //
34575   // when rewriting this into PHIs, we have to perform some renaming on the
34576   // temps since you cannot have a PHI operand refer to a PHI result earlier
34577   // in the same block.  The "simple" but wrong lowering would be:
34578   //
34579   // t2 = PHI t1(BB1), f1(BB2)
34580   // t3 = PHI t2(BB1), f2(BB2)
34581   //
34582   // but clearly t2 is not defined in BB1, so that is incorrect. The proper
34583   // renaming is to note that on the path through BB1, t2 is really just a
34584   // copy of t1, and do that renaming, properly generating:
34585   //
34586   // t2 = PHI t1(BB1), f1(BB2)
34587   // t3 = PHI t1(BB1), f2(BB2)
34588   //
34589   // Case 2:
34590   // CMOV ((CMOV F, T, cc1), T, cc2) is checked here and handled by a separate
34591   // function - EmitLoweredCascadedSelect.
34592 
34593   X86::CondCode CC = X86::CondCode(MI.getOperand(3).getImm());
34594   X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
34595   MachineInstr *LastCMOV = &MI;
34596   MachineBasicBlock::iterator NextMIIt = MachineBasicBlock::iterator(MI);
34597 
34598   // Check for case 1, where there are multiple CMOVs with the same condition
34599   // first.  Of the two cases of multiple CMOV lowerings, case 1 reduces the
34600   // number of jumps the most.
34601 
34602   if (isCMOVPseudo(MI)) {
34603     // See if we have a string of CMOVS with the same condition. Skip over
34604     // intervening debug insts.
34605     while (NextMIIt != ThisMBB->end() && isCMOVPseudo(*NextMIIt) &&
34606            (NextMIIt->getOperand(3).getImm() == CC ||
34607             NextMIIt->getOperand(3).getImm() == OppCC)) {
34608       LastCMOV = &*NextMIIt;
34609       NextMIIt = next_nodbg(NextMIIt, ThisMBB->end());
34610     }
34611   }
34612 
34613   // This checks for case 2, but only do this if we didn't already find
34614   // case 1, as indicated by LastCMOV == MI.
34615   if (LastCMOV == &MI && NextMIIt != ThisMBB->end() &&
34616       NextMIIt->getOpcode() == MI.getOpcode() &&
34617       NextMIIt->getOperand(2).getReg() == MI.getOperand(2).getReg() &&
34618       NextMIIt->getOperand(1).getReg() == MI.getOperand(0).getReg() &&
34619       NextMIIt->getOperand(1).isKill()) {
34620     return EmitLoweredCascadedSelect(MI, *NextMIIt, ThisMBB);
34621   }
34622 
34623   const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
34624   MachineFunction *F = ThisMBB->getParent();
34625   MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
34626   MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
34627 
34628   MachineFunction::iterator It = ++ThisMBB->getIterator();
34629   F->insert(It, FalseMBB);
34630   F->insert(It, SinkMBB);
34631 
34632   // Set the call frame size on entry to the new basic blocks.
34633   unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
34634   FalseMBB->setCallFrameSize(CallFrameSize);
34635   SinkMBB->setCallFrameSize(CallFrameSize);
34636 
34637   // If the EFLAGS register isn't dead in the terminator, then claim that it's
34638   // live into the sink and copy blocks.
34639   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
34640   if (!LastCMOV->killsRegister(X86::EFLAGS) &&
34641       !checkAndUpdateEFLAGSKill(LastCMOV, ThisMBB, TRI)) {
34642     FalseMBB->addLiveIn(X86::EFLAGS);
34643     SinkMBB->addLiveIn(X86::EFLAGS);
34644   }
34645 
34646   // Transfer any debug instructions inside the CMOV sequence to the sunk block.
34647   auto DbgRange = llvm::make_range(MachineBasicBlock::iterator(MI),
34648                                    MachineBasicBlock::iterator(LastCMOV));
34649   for (MachineInstr &MI : llvm::make_early_inc_range(DbgRange))
34650     if (MI.isDebugInstr())
34651       SinkMBB->push_back(MI.removeFromParent());
34652 
34653   // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
34654   SinkMBB->splice(SinkMBB->end(), ThisMBB,
34655                   std::next(MachineBasicBlock::iterator(LastCMOV)),
34656                   ThisMBB->end());
34657   SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
34658 
34659   // Fallthrough block for ThisMBB.
34660   ThisMBB->addSuccessor(FalseMBB);
34661   // The true block target of the first (or only) branch is always a SinkMBB.
34662   ThisMBB->addSuccessor(SinkMBB);
34663   // Fallthrough block for FalseMBB.
34664   FalseMBB->addSuccessor(SinkMBB);
34665 
34666   // Create the conditional branch instruction.
34667   BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
34668 
34669   //  SinkMBB:
34670   //   %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, ThisMBB ]
34671   //  ...
34672   MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
34673   MachineBasicBlock::iterator MIItEnd =
34674       std::next(MachineBasicBlock::iterator(LastCMOV));
34675   createPHIsForCMOVsInSinkBB(MIItBegin, MIItEnd, ThisMBB, FalseMBB, SinkMBB);
34676 
34677   // Now remove the CMOV(s).
34678   ThisMBB->erase(MIItBegin, MIItEnd);
34679 
34680   return SinkMBB;
34681 }
34682 
34683 static unsigned getSUBriOpcode(bool IsLP64) {
34684   if (IsLP64)
34685     return X86::SUB64ri32;
34686   else
34687     return X86::SUB32ri;
34688 }
34689 
34690 MachineBasicBlock *
34691 X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
34692                                            MachineBasicBlock *MBB) const {
34693   MachineFunction *MF = MBB->getParent();
34694   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34695   const X86FrameLowering &TFI = *Subtarget.getFrameLowering();
34696   const MIMetadata MIMD(MI);
34697   const BasicBlock *LLVM_BB = MBB->getBasicBlock();
34698 
34699   const unsigned ProbeSize = getStackProbeSize(*MF);
34700 
34701   MachineRegisterInfo &MRI = MF->getRegInfo();
34702   MachineBasicBlock *testMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34703   MachineBasicBlock *tailMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34704   MachineBasicBlock *blockMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34705 
34706   MachineFunction::iterator MBBIter = ++MBB->getIterator();
34707   MF->insert(MBBIter, testMBB);
34708   MF->insert(MBBIter, blockMBB);
34709   MF->insert(MBBIter, tailMBB);
34710 
34711   Register sizeVReg = MI.getOperand(1).getReg();
34712 
34713   Register physSPReg = TFI.Uses64BitFramePtr ? X86::RSP : X86::ESP;
34714 
34715   Register TmpStackPtr = MRI.createVirtualRegister(
34716       TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
34717   Register FinalStackPtr = MRI.createVirtualRegister(
34718       TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
34719 
34720   BuildMI(*MBB, {MI}, MIMD, TII->get(TargetOpcode::COPY), TmpStackPtr)
34721       .addReg(physSPReg);
34722   {
34723     const unsigned Opc = TFI.Uses64BitFramePtr ? X86::SUB64rr : X86::SUB32rr;
34724     BuildMI(*MBB, {MI}, MIMD, TII->get(Opc), FinalStackPtr)
34725         .addReg(TmpStackPtr)
34726         .addReg(sizeVReg);
34727   }
34728 
34729   // test rsp size
34730 
34731   BuildMI(testMBB, MIMD,
34732           TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
34733       .addReg(FinalStackPtr)
34734       .addReg(physSPReg);
34735 
34736   BuildMI(testMBB, MIMD, TII->get(X86::JCC_1))
34737       .addMBB(tailMBB)
34738       .addImm(X86::COND_GE);
34739   testMBB->addSuccessor(blockMBB);
34740   testMBB->addSuccessor(tailMBB);
34741 
34742   // Touch the block then extend it. This is done on the opposite side of
34743   // static probe where we allocate then touch, to avoid the need of probing the
34744   // tail of the static alloca. Possible scenarios are:
34745   //
34746   //       + ---- <- ------------ <- ------------- <- ------------ +
34747   //       |                                                       |
34748   // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
34749   //                                                               |                                                               |
34750   //                                                               + <- ----------- <- ------------ <- ----------- <- ------------ +
34751   //
34752   // The property we want to enforce is to never have more than [page alloc] between two probes.
34753 
34754   const unsigned XORMIOpc =
34755       TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
34756   addRegOffset(BuildMI(blockMBB, MIMD, TII->get(XORMIOpc)), physSPReg, false, 0)
34757       .addImm(0);
34758 
34759   BuildMI(blockMBB, MIMD, TII->get(getSUBriOpcode(TFI.Uses64BitFramePtr)),
34760           physSPReg)
34761       .addReg(physSPReg)
34762       .addImm(ProbeSize);
34763 
34764   BuildMI(blockMBB, MIMD, TII->get(X86::JMP_1)).addMBB(testMBB);
34765   blockMBB->addSuccessor(testMBB);
34766 
34767   // Replace original instruction by the expected stack ptr
34768   BuildMI(tailMBB, MIMD, TII->get(TargetOpcode::COPY),
34769           MI.getOperand(0).getReg())
34770       .addReg(FinalStackPtr);
34771 
34772   tailMBB->splice(tailMBB->end(), MBB,
34773                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
34774   tailMBB->transferSuccessorsAndUpdatePHIs(MBB);
34775   MBB->addSuccessor(testMBB);
34776 
34777   // Delete the original pseudo instruction.
34778   MI.eraseFromParent();
34779 
34780   // And we're done.
34781   return tailMBB;
34782 }
34783 
34784 MachineBasicBlock *
34785 X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
34786                                         MachineBasicBlock *BB) const {
34787   MachineFunction *MF = BB->getParent();
34788   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34789   const MIMetadata MIMD(MI);
34790   const BasicBlock *LLVM_BB = BB->getBasicBlock();
34791 
34792   assert(MF->shouldSplitStack());
34793 
34794   const bool Is64Bit = Subtarget.is64Bit();
34795   const bool IsLP64 = Subtarget.isTarget64BitLP64();
34796 
34797   const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
34798   const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
34799 
34800   // BB:
34801   //  ... [Till the alloca]
34802   // If stacklet is not large enough, jump to mallocMBB
34803   //
34804   // bumpMBB:
34805   //  Allocate by subtracting from RSP
34806   //  Jump to continueMBB
34807   //
34808   // mallocMBB:
34809   //  Allocate by call to runtime
34810   //
34811   // continueMBB:
34812   //  ...
34813   //  [rest of original BB]
34814   //
34815 
34816   MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34817   MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34818   MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34819 
34820   MachineRegisterInfo &MRI = MF->getRegInfo();
34821   const TargetRegisterClass *AddrRegClass =
34822       getRegClassFor(getPointerTy(MF->getDataLayout()));
34823 
34824   Register mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
34825            bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
34826            tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
34827            SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
34828            sizeVReg = MI.getOperand(1).getReg(),
34829            physSPReg =
34830                IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP;
34831 
34832   MachineFunction::iterator MBBIter = ++BB->getIterator();
34833 
34834   MF->insert(MBBIter, bumpMBB);
34835   MF->insert(MBBIter, mallocMBB);
34836   MF->insert(MBBIter, continueMBB);
34837 
34838   continueMBB->splice(continueMBB->begin(), BB,
34839                       std::next(MachineBasicBlock::iterator(MI)), BB->end());
34840   continueMBB->transferSuccessorsAndUpdatePHIs(BB);
34841 
34842   // Add code to the main basic block to check if the stack limit has been hit,
34843   // and if so, jump to mallocMBB otherwise to bumpMBB.
34844   BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
34845   BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
34846     .addReg(tmpSPVReg).addReg(sizeVReg);
34847   BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
34848     .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
34849     .addReg(SPLimitVReg);
34850   BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
34851 
34852   // bumpMBB simply decreases the stack pointer, since we know the current
34853   // stacklet has enough space.
34854   BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg)
34855     .addReg(SPLimitVReg);
34856   BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
34857     .addReg(SPLimitVReg);
34858   BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
34859 
34860   // Calls into a routine in libgcc to allocate more space from the heap.
34861   const uint32_t *RegMask =
34862       Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
34863   if (IsLP64) {
34864     BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
34865       .addReg(sizeVReg);
34866     BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
34867       .addExternalSymbol("__morestack_allocate_stack_space")
34868       .addRegMask(RegMask)
34869       .addReg(X86::RDI, RegState::Implicit)
34870       .addReg(X86::RAX, RegState::ImplicitDefine);
34871   } else if (Is64Bit) {
34872     BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
34873       .addReg(sizeVReg);
34874     BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
34875       .addExternalSymbol("__morestack_allocate_stack_space")
34876       .addRegMask(RegMask)
34877       .addReg(X86::EDI, RegState::Implicit)
34878       .addReg(X86::EAX, RegState::ImplicitDefine);
34879   } else {
34880     BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
34881       .addImm(12);
34882     BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
34883     BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
34884       .addExternalSymbol("__morestack_allocate_stack_space")
34885       .addRegMask(RegMask)
34886       .addReg(X86::EAX, RegState::ImplicitDefine);
34887   }
34888 
34889   if (!Is64Bit)
34890     BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
34891       .addImm(16);
34892 
34893   BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg)
34894     .addReg(IsLP64 ? X86::RAX : X86::EAX);
34895   BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
34896 
34897   // Set up the CFG correctly.
34898   BB->addSuccessor(bumpMBB);
34899   BB->addSuccessor(mallocMBB);
34900   mallocMBB->addSuccessor(continueMBB);
34901   bumpMBB->addSuccessor(continueMBB);
34902 
34903   // Take care of the PHI nodes.
34904   BuildMI(*continueMBB, continueMBB->begin(), MIMD, TII->get(X86::PHI),
34905           MI.getOperand(0).getReg())
34906       .addReg(mallocPtrVReg)
34907       .addMBB(mallocMBB)
34908       .addReg(bumpSPPtrVReg)
34909       .addMBB(bumpMBB);
34910 
34911   // Delete the original pseudo instruction.
34912   MI.eraseFromParent();
34913 
34914   // And we're done.
34915   return continueMBB;
34916 }
34917 
34918 MachineBasicBlock *
34919 X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
34920                                        MachineBasicBlock *BB) const {
34921   MachineFunction *MF = BB->getParent();
34922   const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
34923   MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
34924   const MIMetadata MIMD(MI);
34925 
34926   assert(!isAsynchronousEHPersonality(
34927              classifyEHPersonality(MF->getFunction().getPersonalityFn())) &&
34928          "SEH does not use catchret!");
34929 
34930   // Only 32-bit EH needs to worry about manually restoring stack pointers.
34931   if (!Subtarget.is32Bit())
34932     return BB;
34933 
34934   // C++ EH creates a new target block to hold the restore code, and wires up
34935   // the new block to the return destination with a normal JMP_4.
34936   MachineBasicBlock *RestoreMBB =
34937       MF->CreateMachineBasicBlock(BB->getBasicBlock());
34938   assert(BB->succ_size() == 1);
34939   MF->insert(std::next(BB->getIterator()), RestoreMBB);
34940   RestoreMBB->transferSuccessorsAndUpdatePHIs(BB);
34941   BB->addSuccessor(RestoreMBB);
34942   MI.getOperand(0).setMBB(RestoreMBB);
34943 
34944   // Marking this as an EH pad but not a funclet entry block causes PEI to
34945   // restore stack pointers in the block.
34946   RestoreMBB->setIsEHPad(true);
34947 
34948   auto RestoreMBBI = RestoreMBB->begin();
34949   BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
34950   return BB;
34951 }
34952 
34953 MachineBasicBlock *
34954 X86TargetLowering::EmitLoweredTLSAddr(MachineInstr &MI,
34955                                       MachineBasicBlock *BB) const {
34956   // So, here we replace TLSADDR with the sequence:
34957   // adjust_stackdown -> TLSADDR -> adjust_stackup.
34958   // We need this because TLSADDR is lowered into calls
34959   // inside MC, therefore without the two markers shrink-wrapping
34960   // may push the prologue/epilogue pass them.
34961   const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
34962   const MIMetadata MIMD(MI);
34963   MachineFunction &MF = *BB->getParent();
34964 
34965   // Emit CALLSEQ_START right before the instruction.
34966   unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
34967   MachineInstrBuilder CallseqStart =
34968       BuildMI(MF, MIMD, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
34969   BB->insert(MachineBasicBlock::iterator(MI), CallseqStart);
34970 
34971   // Emit CALLSEQ_END right after the instruction.
34972   // We don't call erase from parent because we want to keep the
34973   // original instruction around.
34974   unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
34975   MachineInstrBuilder CallseqEnd =
34976       BuildMI(MF, MIMD, TII.get(AdjStackUp)).addImm(0).addImm(0);
34977   BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd);
34978 
34979   return BB;
34980 }
34981 
34982 MachineBasicBlock *
34983 X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
34984                                       MachineBasicBlock *BB) const {
34985   // This is pretty easy.  We're taking the value that we received from
34986   // our load from the relocation, sticking it in either RDI (x86-64)
34987   // or EAX and doing an indirect call.  The return value will then
34988   // be in the normal return register.
34989   MachineFunction *F = BB->getParent();
34990   const X86InstrInfo *TII = Subtarget.getInstrInfo();
34991   const MIMetadata MIMD(MI);
34992 
34993   assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?");
34994   assert(MI.getOperand(3).isGlobal() && "This should be a global");
34995 
34996   // Get a register mask for the lowered call.
34997   // FIXME: The 32-bit calls have non-standard calling conventions. Use a
34998   // proper register mask.
34999   const uint32_t *RegMask =
35000       Subtarget.is64Bit() ?
35001       Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
35002       Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
35003   if (Subtarget.is64Bit()) {
35004     MachineInstrBuilder MIB =
35005         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
35006             .addReg(X86::RIP)
35007             .addImm(0)
35008             .addReg(0)
35009             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35010                               MI.getOperand(3).getTargetFlags())
35011             .addReg(0);
35012     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL64m));
35013     addDirectMem(MIB, X86::RDI);
35014     MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
35015   } else if (!isPositionIndependent()) {
35016     MachineInstrBuilder MIB =
35017         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35018             .addReg(0)
35019             .addImm(0)
35020             .addReg(0)
35021             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35022                               MI.getOperand(3).getTargetFlags())
35023             .addReg(0);
35024     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35025     addDirectMem(MIB, X86::EAX);
35026     MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35027   } else {
35028     MachineInstrBuilder MIB =
35029         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35030             .addReg(TII->getGlobalBaseReg(F))
35031             .addImm(0)
35032             .addReg(0)
35033             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35034                               MI.getOperand(3).getTargetFlags())
35035             .addReg(0);
35036     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35037     addDirectMem(MIB, X86::EAX);
35038     MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35039   }
35040 
35041   MI.eraseFromParent(); // The pseudo instruction is gone now.
35042   return BB;
35043 }
35044 
35045 static unsigned getOpcodeForIndirectThunk(unsigned RPOpc) {
35046   switch (RPOpc) {
35047   case X86::INDIRECT_THUNK_CALL32:
35048     return X86::CALLpcrel32;
35049   case X86::INDIRECT_THUNK_CALL64:
35050     return X86::CALL64pcrel32;
35051   case X86::INDIRECT_THUNK_TCRETURN32:
35052     return X86::TCRETURNdi;
35053   case X86::INDIRECT_THUNK_TCRETURN64:
35054     return X86::TCRETURNdi64;
35055   }
35056   llvm_unreachable("not indirect thunk opcode");
35057 }
35058 
35059 static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget,
35060                                           unsigned Reg) {
35061   if (Subtarget.useRetpolineExternalThunk()) {
35062     // When using an external thunk for retpolines, we pick names that match the
35063     // names GCC happens to use as well. This helps simplify the implementation
35064     // of the thunks for kernels where they have no easy ability to create
35065     // aliases and are doing non-trivial configuration of the thunk's body. For
35066     // example, the Linux kernel will do boot-time hot patching of the thunk
35067     // bodies and cannot easily export aliases of these to loaded modules.
35068     //
35069     // Note that at any point in the future, we may need to change the semantics
35070     // of how we implement retpolines and at that time will likely change the
35071     // name of the called thunk. Essentially, there is no hard guarantee that
35072     // LLVM will generate calls to specific thunks, we merely make a best-effort
35073     // attempt to help out kernels and other systems where duplicating the
35074     // thunks is costly.
35075     switch (Reg) {
35076     case X86::EAX:
35077       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35078       return "__x86_indirect_thunk_eax";
35079     case X86::ECX:
35080       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35081       return "__x86_indirect_thunk_ecx";
35082     case X86::EDX:
35083       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35084       return "__x86_indirect_thunk_edx";
35085     case X86::EDI:
35086       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35087       return "__x86_indirect_thunk_edi";
35088     case X86::R11:
35089       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35090       return "__x86_indirect_thunk_r11";
35091     }
35092     llvm_unreachable("unexpected reg for external indirect thunk");
35093   }
35094 
35095   if (Subtarget.useRetpolineIndirectCalls() ||
35096       Subtarget.useRetpolineIndirectBranches()) {
35097     // When targeting an internal COMDAT thunk use an LLVM-specific name.
35098     switch (Reg) {
35099     case X86::EAX:
35100       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35101       return "__llvm_retpoline_eax";
35102     case X86::ECX:
35103       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35104       return "__llvm_retpoline_ecx";
35105     case X86::EDX:
35106       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35107       return "__llvm_retpoline_edx";
35108     case X86::EDI:
35109       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35110       return "__llvm_retpoline_edi";
35111     case X86::R11:
35112       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35113       return "__llvm_retpoline_r11";
35114     }
35115     llvm_unreachable("unexpected reg for retpoline");
35116   }
35117 
35118   if (Subtarget.useLVIControlFlowIntegrity()) {
35119     assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35120     return "__llvm_lvi_thunk_r11";
35121   }
35122   llvm_unreachable("getIndirectThunkSymbol() invoked without thunk feature");
35123 }
35124 
35125 MachineBasicBlock *
35126 X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI,
35127                                             MachineBasicBlock *BB) const {
35128   // Copy the virtual register into the R11 physical register and
35129   // call the retpoline thunk.
35130   const MIMetadata MIMD(MI);
35131   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35132   Register CalleeVReg = MI.getOperand(0).getReg();
35133   unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode());
35134 
35135   // Find an available scratch register to hold the callee. On 64-bit, we can
35136   // just use R11, but we scan for uses anyway to ensure we don't generate
35137   // incorrect code. On 32-bit, we use one of EAX, ECX, or EDX that isn't
35138   // already a register use operand to the call to hold the callee. If none
35139   // are available, use EDI instead. EDI is chosen because EBX is the PIC base
35140   // register and ESI is the base pointer to realigned stack frames with VLAs.
35141   SmallVector<unsigned, 3> AvailableRegs;
35142   if (Subtarget.is64Bit())
35143     AvailableRegs.push_back(X86::R11);
35144   else
35145     AvailableRegs.append({X86::EAX, X86::ECX, X86::EDX, X86::EDI});
35146 
35147   // Zero out any registers that are already used.
35148   for (const auto &MO : MI.operands()) {
35149     if (MO.isReg() && MO.isUse())
35150       for (unsigned &Reg : AvailableRegs)
35151         if (Reg == MO.getReg())
35152           Reg = 0;
35153   }
35154 
35155   // Choose the first remaining non-zero available register.
35156   unsigned AvailableReg = 0;
35157   for (unsigned MaybeReg : AvailableRegs) {
35158     if (MaybeReg) {
35159       AvailableReg = MaybeReg;
35160       break;
35161     }
35162   }
35163   if (!AvailableReg)
35164     report_fatal_error("calling convention incompatible with retpoline, no "
35165                        "available registers");
35166 
35167   const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg);
35168 
35169   BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), AvailableReg)
35170       .addReg(CalleeVReg);
35171   MI.getOperand(0).ChangeToES(Symbol);
35172   MI.setDesc(TII->get(Opc));
35173   MachineInstrBuilder(*BB->getParent(), &MI)
35174       .addReg(AvailableReg, RegState::Implicit | RegState::Kill);
35175   return BB;
35176 }
35177 
35178 /// SetJmp implies future control flow change upon calling the corresponding
35179 /// LongJmp.
35180 /// Instead of using the 'return' instruction, the long jump fixes the stack and
35181 /// performs an indirect branch. To do so it uses the registers that were stored
35182 /// in the jump buffer (when calling SetJmp).
35183 /// In case the shadow stack is enabled we need to fix it as well, because some
35184 /// return addresses will be skipped.
35185 /// The function will save the SSP for future fixing in the function
35186 /// emitLongJmpShadowStackFix.
35187 /// \sa emitLongJmpShadowStackFix
35188 /// \param [in] MI The temporary Machine Instruction for the builtin.
35189 /// \param [in] MBB The Machine Basic Block that will be modified.
35190 void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI,
35191                                                  MachineBasicBlock *MBB) const {
35192   const MIMetadata MIMD(MI);
35193   MachineFunction *MF = MBB->getParent();
35194   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35195   MachineRegisterInfo &MRI = MF->getRegInfo();
35196   MachineInstrBuilder MIB;
35197 
35198   // Memory Reference.
35199   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35200                                            MI.memoperands_end());
35201 
35202   // Initialize a register with zero.
35203   MVT PVT = getPointerTy(MF->getDataLayout());
35204   const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35205   Register ZReg = MRI.createVirtualRegister(PtrRC);
35206   unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
35207   BuildMI(*MBB, MI, MIMD, TII->get(XorRROpc))
35208       .addDef(ZReg)
35209       .addReg(ZReg, RegState::Undef)
35210       .addReg(ZReg, RegState::Undef);
35211 
35212   // Read the current SSP Register value to the zeroed register.
35213   Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
35214   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
35215   BuildMI(*MBB, MI, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
35216 
35217   // Write the SSP register value to offset 3 in input memory buffer.
35218   unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35219   MIB = BuildMI(*MBB, MI, MIMD, TII->get(PtrStoreOpc));
35220   const int64_t SSPOffset = 3 * PVT.getStoreSize();
35221   const unsigned MemOpndSlot = 1;
35222   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35223     if (i == X86::AddrDisp)
35224       MIB.addDisp(MI.getOperand(MemOpndSlot + i), SSPOffset);
35225     else
35226       MIB.add(MI.getOperand(MemOpndSlot + i));
35227   }
35228   MIB.addReg(SSPCopyReg);
35229   MIB.setMemRefs(MMOs);
35230 }
35231 
35232 MachineBasicBlock *
35233 X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
35234                                     MachineBasicBlock *MBB) const {
35235   const MIMetadata MIMD(MI);
35236   MachineFunction *MF = MBB->getParent();
35237   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35238   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
35239   MachineRegisterInfo &MRI = MF->getRegInfo();
35240 
35241   const BasicBlock *BB = MBB->getBasicBlock();
35242   MachineFunction::iterator I = ++MBB->getIterator();
35243 
35244   // Memory Reference
35245   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35246                                            MI.memoperands_end());
35247 
35248   unsigned DstReg;
35249   unsigned MemOpndSlot = 0;
35250 
35251   unsigned CurOp = 0;
35252 
35253   DstReg = MI.getOperand(CurOp++).getReg();
35254   const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
35255   assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
35256   (void)TRI;
35257   Register mainDstReg = MRI.createVirtualRegister(RC);
35258   Register restoreDstReg = MRI.createVirtualRegister(RC);
35259 
35260   MemOpndSlot = CurOp;
35261 
35262   MVT PVT = getPointerTy(MF->getDataLayout());
35263   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
35264          "Invalid Pointer Size!");
35265 
35266   // For v = setjmp(buf), we generate
35267   //
35268   // thisMBB:
35269   //  buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB
35270   //  SjLjSetup restoreMBB
35271   //
35272   // mainMBB:
35273   //  v_main = 0
35274   //
35275   // sinkMBB:
35276   //  v = phi(main, restore)
35277   //
35278   // restoreMBB:
35279   //  if base pointer being used, load it from frame
35280   //  v_restore = 1
35281 
35282   MachineBasicBlock *thisMBB = MBB;
35283   MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
35284   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
35285   MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
35286   MF->insert(I, mainMBB);
35287   MF->insert(I, sinkMBB);
35288   MF->push_back(restoreMBB);
35289   restoreMBB->setMachineBlockAddressTaken();
35290 
35291   MachineInstrBuilder MIB;
35292 
35293   // Transfer the remainder of BB and its successor edges to sinkMBB.
35294   sinkMBB->splice(sinkMBB->begin(), MBB,
35295                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
35296   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
35297 
35298   // thisMBB:
35299   unsigned PtrStoreOpc = 0;
35300   unsigned LabelReg = 0;
35301   const int64_t LabelOffset = 1 * PVT.getStoreSize();
35302   bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
35303                      !isPositionIndependent();
35304 
35305   // Prepare IP either in reg or imm.
35306   if (!UseImmLabel) {
35307     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35308     const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35309     LabelReg = MRI.createVirtualRegister(PtrRC);
35310     if (Subtarget.is64Bit()) {
35311       MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
35312               .addReg(X86::RIP)
35313               .addImm(0)
35314               .addReg(0)
35315               .addMBB(restoreMBB)
35316               .addReg(0);
35317     } else {
35318       const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
35319       MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
35320               .addReg(XII->getGlobalBaseReg(MF))
35321               .addImm(0)
35322               .addReg(0)
35323               .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
35324               .addReg(0);
35325     }
35326   } else
35327     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
35328   // Store IP
35329   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrStoreOpc));
35330   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35331     if (i == X86::AddrDisp)
35332       MIB.addDisp(MI.getOperand(MemOpndSlot + i), LabelOffset);
35333     else
35334       MIB.add(MI.getOperand(MemOpndSlot + i));
35335   }
35336   if (!UseImmLabel)
35337     MIB.addReg(LabelReg);
35338   else
35339     MIB.addMBB(restoreMBB);
35340   MIB.setMemRefs(MMOs);
35341 
35342   if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
35343     emitSetJmpShadowStackFix(MI, thisMBB);
35344   }
35345 
35346   // Setup
35347   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
35348           .addMBB(restoreMBB);
35349 
35350   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
35351   MIB.addRegMask(RegInfo->getNoPreservedMask());
35352   thisMBB->addSuccessor(mainMBB);
35353   thisMBB->addSuccessor(restoreMBB);
35354 
35355   // mainMBB:
35356   //  EAX = 0
35357   BuildMI(mainMBB, MIMD, TII->get(X86::MOV32r0), mainDstReg);
35358   mainMBB->addSuccessor(sinkMBB);
35359 
35360   // sinkMBB:
35361   BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
35362       .addReg(mainDstReg)
35363       .addMBB(mainMBB)
35364       .addReg(restoreDstReg)
35365       .addMBB(restoreMBB);
35366 
35367   // restoreMBB:
35368   if (RegInfo->hasBasePointer(*MF)) {
35369     const bool Uses64BitFramePtr =
35370         Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
35371     X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
35372     X86FI->setRestoreBasePointer(MF);
35373     Register FramePtr = RegInfo->getFrameRegister(*MF);
35374     Register BasePtr = RegInfo->getBaseRegister();
35375     unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
35376     addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr),
35377                  FramePtr, true, X86FI->getRestoreBasePointerOffset())
35378       .setMIFlag(MachineInstr::FrameSetup);
35379   }
35380   BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
35381   BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
35382   restoreMBB->addSuccessor(sinkMBB);
35383 
35384   MI.eraseFromParent();
35385   return sinkMBB;
35386 }
35387 
35388 /// Fix the shadow stack using the previously saved SSP pointer.
35389 /// \sa emitSetJmpShadowStackFix
35390 /// \param [in] MI The temporary Machine Instruction for the builtin.
35391 /// \param [in] MBB The Machine Basic Block that will be modified.
35392 /// \return The sink MBB that will perform the future indirect branch.
35393 MachineBasicBlock *
35394 X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI,
35395                                              MachineBasicBlock *MBB) const {
35396   const MIMetadata MIMD(MI);
35397   MachineFunction *MF = MBB->getParent();
35398   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35399   MachineRegisterInfo &MRI = MF->getRegInfo();
35400 
35401   // Memory Reference
35402   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35403                                            MI.memoperands_end());
35404 
35405   MVT PVT = getPointerTy(MF->getDataLayout());
35406   const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35407 
35408   // checkSspMBB:
35409   //         xor vreg1, vreg1
35410   //         rdssp vreg1
35411   //         test vreg1, vreg1
35412   //         je sinkMBB   # Jump if Shadow Stack is not supported
35413   // fallMBB:
35414   //         mov buf+24/12(%rip), vreg2
35415   //         sub vreg1, vreg2
35416   //         jbe sinkMBB  # No need to fix the Shadow Stack
35417   // fixShadowMBB:
35418   //         shr 3/2, vreg2
35419   //         incssp vreg2  # fix the SSP according to the lower 8 bits
35420   //         shr 8, vreg2
35421   //         je sinkMBB
35422   // fixShadowLoopPrepareMBB:
35423   //         shl vreg2
35424   //         mov 128, vreg3
35425   // fixShadowLoopMBB:
35426   //         incssp vreg3
35427   //         dec vreg2
35428   //         jne fixShadowLoopMBB # Iterate until you finish fixing
35429   //                              # the Shadow Stack
35430   // sinkMBB:
35431 
35432   MachineFunction::iterator I = ++MBB->getIterator();
35433   const BasicBlock *BB = MBB->getBasicBlock();
35434 
35435   MachineBasicBlock *checkSspMBB = MF->CreateMachineBasicBlock(BB);
35436   MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
35437   MachineBasicBlock *fixShadowMBB = MF->CreateMachineBasicBlock(BB);
35438   MachineBasicBlock *fixShadowLoopPrepareMBB = MF->CreateMachineBasicBlock(BB);
35439   MachineBasicBlock *fixShadowLoopMBB = MF->CreateMachineBasicBlock(BB);
35440   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
35441   MF->insert(I, checkSspMBB);
35442   MF->insert(I, fallMBB);
35443   MF->insert(I, fixShadowMBB);
35444   MF->insert(I, fixShadowLoopPrepareMBB);
35445   MF->insert(I, fixShadowLoopMBB);
35446   MF->insert(I, sinkMBB);
35447 
35448   // Transfer the remainder of BB and its successor edges to sinkMBB.
35449   sinkMBB->splice(sinkMBB->begin(), MBB, MachineBasicBlock::iterator(MI),
35450                   MBB->end());
35451   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
35452 
35453   MBB->addSuccessor(checkSspMBB);
35454 
35455   // Initialize a register with zero.
35456   Register ZReg = MRI.createVirtualRegister(&X86::GR32RegClass);
35457   BuildMI(checkSspMBB, MIMD, TII->get(X86::MOV32r0), ZReg);
35458 
35459   if (PVT == MVT::i64) {
35460     Register TmpZReg = MRI.createVirtualRegister(PtrRC);
35461     BuildMI(checkSspMBB, MIMD, TII->get(X86::SUBREG_TO_REG), TmpZReg)
35462       .addImm(0)
35463       .addReg(ZReg)
35464       .addImm(X86::sub_32bit);
35465     ZReg = TmpZReg;
35466   }
35467 
35468   // Read the current SSP Register value to the zeroed register.
35469   Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
35470   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
35471   BuildMI(checkSspMBB, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
35472 
35473   // Check whether the result of the SSP register is zero and jump directly
35474   // to the sink.
35475   unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;
35476   BuildMI(checkSspMBB, MIMD, TII->get(TestRROpc))
35477       .addReg(SSPCopyReg)
35478       .addReg(SSPCopyReg);
35479   BuildMI(checkSspMBB, MIMD, TII->get(X86::JCC_1))
35480       .addMBB(sinkMBB)
35481       .addImm(X86::COND_E);
35482   checkSspMBB->addSuccessor(sinkMBB);
35483   checkSspMBB->addSuccessor(fallMBB);
35484 
35485   // Reload the previously saved SSP register value.
35486   Register PrevSSPReg = MRI.createVirtualRegister(PtrRC);
35487   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
35488   const int64_t SPPOffset = 3 * PVT.getStoreSize();
35489   MachineInstrBuilder MIB =
35490       BuildMI(fallMBB, MIMD, TII->get(PtrLoadOpc), PrevSSPReg);
35491   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35492     const MachineOperand &MO = MI.getOperand(i);
35493     if (i == X86::AddrDisp)
35494       MIB.addDisp(MO, SPPOffset);
35495     else if (MO.isReg()) // Don't add the whole operand, we don't want to
35496                          // preserve kill flags.
35497       MIB.addReg(MO.getReg());
35498     else
35499       MIB.add(MO);
35500   }
35501   MIB.setMemRefs(MMOs);
35502 
35503   // Subtract the current SSP from the previous SSP.
35504   Register SspSubReg = MRI.createVirtualRegister(PtrRC);
35505   unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;
35506   BuildMI(fallMBB, MIMD, TII->get(SubRROpc), SspSubReg)
35507       .addReg(PrevSSPReg)
35508       .addReg(SSPCopyReg);
35509 
35510   // Jump to sink in case PrevSSPReg <= SSPCopyReg.
35511   BuildMI(fallMBB, MIMD, TII->get(X86::JCC_1))
35512       .addMBB(sinkMBB)
35513       .addImm(X86::COND_BE);
35514   fallMBB->addSuccessor(sinkMBB);
35515   fallMBB->addSuccessor(fixShadowMBB);
35516 
35517   // Shift right by 2/3 for 32/64 because incssp multiplies the argument by 4/8.
35518   unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;
35519   unsigned Offset = (PVT == MVT::i64) ? 3 : 2;
35520   Register SspFirstShrReg = MRI.createVirtualRegister(PtrRC);
35521   BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspFirstShrReg)
35522       .addReg(SspSubReg)
35523       .addImm(Offset);
35524 
35525   // Increase SSP when looking only on the lower 8 bits of the delta.
35526   unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;
35527   BuildMI(fixShadowMBB, MIMD, TII->get(IncsspOpc)).addReg(SspFirstShrReg);
35528 
35529   // Reset the lower 8 bits.
35530   Register SspSecondShrReg = MRI.createVirtualRegister(PtrRC);
35531   BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspSecondShrReg)
35532       .addReg(SspFirstShrReg)
35533       .addImm(8);
35534 
35535   // Jump if the result of the shift is zero.
35536   BuildMI(fixShadowMBB, MIMD, TII->get(X86::JCC_1))
35537       .addMBB(sinkMBB)
35538       .addImm(X86::COND_E);
35539   fixShadowMBB->addSuccessor(sinkMBB);
35540   fixShadowMBB->addSuccessor(fixShadowLoopPrepareMBB);
35541 
35542   // Do a single shift left.
35543   unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64ri : X86::SHL32ri;
35544   Register SspAfterShlReg = MRI.createVirtualRegister(PtrRC);
35545   BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(ShlR1Opc), SspAfterShlReg)
35546       .addReg(SspSecondShrReg)
35547       .addImm(1);
35548 
35549   // Save the value 128 to a register (will be used next with incssp).
35550   Register Value128InReg = MRI.createVirtualRegister(PtrRC);
35551   unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;
35552   BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(MovRIOpc), Value128InReg)
35553       .addImm(128);
35554   fixShadowLoopPrepareMBB->addSuccessor(fixShadowLoopMBB);
35555 
35556   // Since incssp only looks at the lower 8 bits, we might need to do several
35557   // iterations of incssp until we finish fixing the shadow stack.
35558   Register DecReg = MRI.createVirtualRegister(PtrRC);
35559   Register CounterReg = MRI.createVirtualRegister(PtrRC);
35560   BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::PHI), CounterReg)
35561       .addReg(SspAfterShlReg)
35562       .addMBB(fixShadowLoopPrepareMBB)
35563       .addReg(DecReg)
35564       .addMBB(fixShadowLoopMBB);
35565 
35566   // Every iteration we increase the SSP by 128.
35567   BuildMI(fixShadowLoopMBB, MIMD, TII->get(IncsspOpc)).addReg(Value128InReg);
35568 
35569   // Every iteration we decrement the counter by 1.
35570   unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;
35571   BuildMI(fixShadowLoopMBB, MIMD, TII->get(DecROpc), DecReg).addReg(CounterReg);
35572 
35573   // Jump if the counter is not zero yet.
35574   BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::JCC_1))
35575       .addMBB(fixShadowLoopMBB)
35576       .addImm(X86::COND_NE);
35577   fixShadowLoopMBB->addSuccessor(sinkMBB);
35578   fixShadowLoopMBB->addSuccessor(fixShadowLoopMBB);
35579 
35580   return sinkMBB;
35581 }
35582 
35583 MachineBasicBlock *
35584 X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
35585                                      MachineBasicBlock *MBB) const {
35586   const MIMetadata MIMD(MI);
35587   MachineFunction *MF = MBB->getParent();
35588   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35589   MachineRegisterInfo &MRI = MF->getRegInfo();
35590 
35591   // Memory Reference
35592   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35593                                            MI.memoperands_end());
35594 
35595   MVT PVT = getPointerTy(MF->getDataLayout());
35596   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
35597          "Invalid Pointer Size!");
35598 
35599   const TargetRegisterClass *RC =
35600     (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
35601   Register Tmp = MRI.createVirtualRegister(RC);
35602   // Since FP is only updated here but NOT referenced, it's treated as GPR.
35603   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
35604   Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
35605   Register SP = RegInfo->getStackRegister();
35606 
35607   MachineInstrBuilder MIB;
35608 
35609   const int64_t LabelOffset = 1 * PVT.getStoreSize();
35610   const int64_t SPOffset = 2 * PVT.getStoreSize();
35611 
35612   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
35613   unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
35614 
35615   MachineBasicBlock *thisMBB = MBB;
35616 
35617   // When CET and shadow stack is enabled, we need to fix the Shadow Stack.
35618   if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
35619     thisMBB = emitLongJmpShadowStackFix(MI, thisMBB);
35620   }
35621 
35622   // Reload FP
35623   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), FP);
35624   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35625     const MachineOperand &MO = MI.getOperand(i);
35626     if (MO.isReg()) // Don't add the whole operand, we don't want to
35627                     // preserve kill flags.
35628       MIB.addReg(MO.getReg());
35629     else
35630       MIB.add(MO);
35631   }
35632   MIB.setMemRefs(MMOs);
35633 
35634   // Reload IP
35635   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp);
35636   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35637     const MachineOperand &MO = MI.getOperand(i);
35638     if (i == X86::AddrDisp)
35639       MIB.addDisp(MO, LabelOffset);
35640     else if (MO.isReg()) // Don't add the whole operand, we don't want to
35641                          // preserve kill flags.
35642       MIB.addReg(MO.getReg());
35643     else
35644       MIB.add(MO);
35645   }
35646   MIB.setMemRefs(MMOs);
35647 
35648   // Reload SP
35649   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), SP);
35650   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35651     if (i == X86::AddrDisp)
35652       MIB.addDisp(MI.getOperand(i), SPOffset);
35653     else
35654       MIB.add(MI.getOperand(i)); // We can preserve the kill flags here, it's
35655                                  // the last instruction of the expansion.
35656   }
35657   MIB.setMemRefs(MMOs);
35658 
35659   // Jump
35660   BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp);
35661 
35662   MI.eraseFromParent();
35663   return thisMBB;
35664 }
35665 
35666 void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
35667                                                MachineBasicBlock *MBB,
35668                                                MachineBasicBlock *DispatchBB,
35669                                                int FI) const {
35670   const MIMetadata MIMD(MI);
35671   MachineFunction *MF = MBB->getParent();
35672   MachineRegisterInfo *MRI = &MF->getRegInfo();
35673   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35674 
35675   MVT PVT = getPointerTy(MF->getDataLayout());
35676   assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
35677 
35678   unsigned Op = 0;
35679   unsigned VR = 0;
35680 
35681   bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
35682                      !isPositionIndependent();
35683 
35684   if (UseImmLabel) {
35685     Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
35686   } else {
35687     const TargetRegisterClass *TRC =
35688         (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
35689     VR = MRI->createVirtualRegister(TRC);
35690     Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35691 
35692     if (Subtarget.is64Bit())
35693       BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA64r), VR)
35694           .addReg(X86::RIP)
35695           .addImm(1)
35696           .addReg(0)
35697           .addMBB(DispatchBB)
35698           .addReg(0);
35699     else
35700       BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA32r), VR)
35701           .addReg(0) /* TII->getGlobalBaseReg(MF) */
35702           .addImm(1)
35703           .addReg(0)
35704           .addMBB(DispatchBB, Subtarget.classifyBlockAddressReference())
35705           .addReg(0);
35706   }
35707 
35708   MachineInstrBuilder MIB = BuildMI(*MBB, MI, MIMD, TII->get(Op));
35709   addFrameReference(MIB, FI, Subtarget.is64Bit() ? 56 : 36);
35710   if (UseImmLabel)
35711     MIB.addMBB(DispatchBB);
35712   else
35713     MIB.addReg(VR);
35714 }
35715 
35716 MachineBasicBlock *
35717 X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
35718                                          MachineBasicBlock *BB) const {
35719   const MIMetadata MIMD(MI);
35720   MachineFunction *MF = BB->getParent();
35721   MachineRegisterInfo *MRI = &MF->getRegInfo();
35722   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35723   int FI = MF->getFrameInfo().getFunctionContextIndex();
35724 
35725   // Get a mapping of the call site numbers to all of the landing pads they're
35726   // associated with.
35727   DenseMap<unsigned, SmallVector<MachineBasicBlock *, 2>> CallSiteNumToLPad;
35728   unsigned MaxCSNum = 0;
35729   for (auto &MBB : *MF) {
35730     if (!MBB.isEHPad())
35731       continue;
35732 
35733     MCSymbol *Sym = nullptr;
35734     for (const auto &MI : MBB) {
35735       if (MI.isDebugInstr())
35736         continue;
35737 
35738       assert(MI.isEHLabel() && "expected EH_LABEL");
35739       Sym = MI.getOperand(0).getMCSymbol();
35740       break;
35741     }
35742 
35743     if (!MF->hasCallSiteLandingPad(Sym))
35744       continue;
35745 
35746     for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) {
35747       CallSiteNumToLPad[CSI].push_back(&MBB);
35748       MaxCSNum = std::max(MaxCSNum, CSI);
35749     }
35750   }
35751 
35752   // Get an ordered list of the machine basic blocks for the jump table.
35753   std::vector<MachineBasicBlock *> LPadList;
35754   SmallPtrSet<MachineBasicBlock *, 32> InvokeBBs;
35755   LPadList.reserve(CallSiteNumToLPad.size());
35756 
35757   for (unsigned CSI = 1; CSI <= MaxCSNum; ++CSI) {
35758     for (auto &LP : CallSiteNumToLPad[CSI]) {
35759       LPadList.push_back(LP);
35760       InvokeBBs.insert(LP->pred_begin(), LP->pred_end());
35761     }
35762   }
35763 
35764   assert(!LPadList.empty() &&
35765          "No landing pad destinations for the dispatch jump table!");
35766 
35767   // Create the MBBs for the dispatch code.
35768 
35769   // Shove the dispatch's address into the return slot in the function context.
35770   MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
35771   DispatchBB->setIsEHPad(true);
35772 
35773   MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
35774   BuildMI(TrapBB, MIMD, TII->get(X86::TRAP));
35775   DispatchBB->addSuccessor(TrapBB);
35776 
35777   MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
35778   DispatchBB->addSuccessor(DispContBB);
35779 
35780   // Insert MBBs.
35781   MF->push_back(DispatchBB);
35782   MF->push_back(DispContBB);
35783   MF->push_back(TrapBB);
35784 
35785   // Insert code into the entry block that creates and registers the function
35786   // context.
35787   SetupEntryBlockForSjLj(MI, BB, DispatchBB, FI);
35788 
35789   // Create the jump table and associated information
35790   unsigned JTE = getJumpTableEncoding();
35791   MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTE);
35792   unsigned MJTI = JTI->createJumpTableIndex(LPadList);
35793 
35794   const X86RegisterInfo &RI = TII->getRegisterInfo();
35795   // Add a register mask with no preserved registers.  This results in all
35796   // registers being marked as clobbered.
35797   if (RI.hasBasePointer(*MF)) {
35798     const bool FPIs64Bit =
35799         Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
35800     X86MachineFunctionInfo *MFI = MF->getInfo<X86MachineFunctionInfo>();
35801     MFI->setRestoreBasePointer(MF);
35802 
35803     Register FP = RI.getFrameRegister(*MF);
35804     Register BP = RI.getBaseRegister();
35805     unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
35806     addRegOffset(BuildMI(DispatchBB, MIMD, TII->get(Op), BP), FP, true,
35807                  MFI->getRestoreBasePointerOffset())
35808         .addRegMask(RI.getNoPreservedMask());
35809   } else {
35810     BuildMI(DispatchBB, MIMD, TII->get(X86::NOOP))
35811         .addRegMask(RI.getNoPreservedMask());
35812   }
35813 
35814   // IReg is used as an index in a memory operand and therefore can't be SP
35815   Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass);
35816   addFrameReference(BuildMI(DispatchBB, MIMD, TII->get(X86::MOV32rm), IReg), FI,
35817                     Subtarget.is64Bit() ? 8 : 4);
35818   BuildMI(DispatchBB, MIMD, TII->get(X86::CMP32ri))
35819       .addReg(IReg)
35820       .addImm(LPadList.size());
35821   BuildMI(DispatchBB, MIMD, TII->get(X86::JCC_1))
35822       .addMBB(TrapBB)
35823       .addImm(X86::COND_AE);
35824 
35825   if (Subtarget.is64Bit()) {
35826     Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass);
35827     Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
35828 
35829     // leaq .LJTI0_0(%rip), BReg
35830     BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), BReg)
35831         .addReg(X86::RIP)
35832         .addImm(1)
35833         .addReg(0)
35834         .addJumpTableIndex(MJTI)
35835         .addReg(0);
35836     // movzx IReg64, IReg
35837     BuildMI(DispContBB, MIMD, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64)
35838         .addImm(0)
35839         .addReg(IReg)
35840         .addImm(X86::sub_32bit);
35841 
35842     switch (JTE) {
35843     case MachineJumpTableInfo::EK_BlockAddress:
35844       // jmpq *(BReg,IReg64,8)
35845       BuildMI(DispContBB, MIMD, TII->get(X86::JMP64m))
35846           .addReg(BReg)
35847           .addImm(8)
35848           .addReg(IReg64)
35849           .addImm(0)
35850           .addReg(0);
35851       break;
35852     case MachineJumpTableInfo::EK_LabelDifference32: {
35853       Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass);
35854       Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
35855       Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
35856 
35857       // movl (BReg,IReg64,4), OReg
35858       BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg)
35859           .addReg(BReg)
35860           .addImm(4)
35861           .addReg(IReg64)
35862           .addImm(0)
35863           .addReg(0);
35864       // movsx OReg64, OReg
35865       BuildMI(DispContBB, MIMD, TII->get(X86::MOVSX64rr32), OReg64)
35866           .addReg(OReg);
35867       // addq BReg, OReg64, TReg
35868       BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg)
35869           .addReg(OReg64)
35870           .addReg(BReg);
35871       // jmpq *TReg
35872       BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
35873       break;
35874     }
35875     default:
35876       llvm_unreachable("Unexpected jump table encoding");
35877     }
35878   } else {
35879     // jmpl *.LJTI0_0(,IReg,4)
35880     BuildMI(DispContBB, MIMD, TII->get(X86::JMP32m))
35881         .addReg(0)
35882         .addImm(4)
35883         .addReg(IReg)
35884         .addJumpTableIndex(MJTI)
35885         .addReg(0);
35886   }
35887 
35888   // Add the jump table entries as successors to the MBB.
35889   SmallPtrSet<MachineBasicBlock *, 8> SeenMBBs;
35890   for (auto &LP : LPadList)
35891     if (SeenMBBs.insert(LP).second)
35892       DispContBB->addSuccessor(LP);
35893 
35894   // N.B. the order the invoke BBs are processed in doesn't matter here.
35895   SmallVector<MachineBasicBlock *, 64> MBBLPads;
35896   const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs();
35897   for (MachineBasicBlock *MBB : InvokeBBs) {
35898     // Remove the landing pad successor from the invoke block and replace it
35899     // with the new dispatch block.
35900     // Keep a copy of Successors since it's modified inside the loop.
35901     SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(),
35902                                                    MBB->succ_rend());
35903     // FIXME: Avoid quadratic complexity.
35904     for (auto *MBBS : Successors) {
35905       if (MBBS->isEHPad()) {
35906         MBB->removeSuccessor(MBBS);
35907         MBBLPads.push_back(MBBS);
35908       }
35909     }
35910 
35911     MBB->addSuccessor(DispatchBB);
35912 
35913     // Find the invoke call and mark all of the callee-saved registers as
35914     // 'implicit defined' so that they're spilled.  This prevents code from
35915     // moving instructions to before the EH block, where they will never be
35916     // executed.
35917     for (auto &II : reverse(*MBB)) {
35918       if (!II.isCall())
35919         continue;
35920 
35921       DenseMap<unsigned, bool> DefRegs;
35922       for (auto &MOp : II.operands())
35923         if (MOp.isReg())
35924           DefRegs[MOp.getReg()] = true;
35925 
35926       MachineInstrBuilder MIB(*MF, &II);
35927       for (unsigned RegIdx = 0; SavedRegs[RegIdx]; ++RegIdx) {
35928         unsigned Reg = SavedRegs[RegIdx];
35929         if (!DefRegs[Reg])
35930           MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
35931       }
35932 
35933       break;
35934     }
35935   }
35936 
35937   // Mark all former landing pads as non-landing pads.  The dispatch is the only
35938   // landing pad now.
35939   for (auto &LP : MBBLPads)
35940     LP->setIsEHPad(false);
35941 
35942   // The instruction is gone now.
35943   MI.eraseFromParent();
35944   return BB;
35945 }
35946 
35947 MachineBasicBlock *
35948 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
35949                                                MachineBasicBlock *BB) const {
35950   MachineFunction *MF = BB->getParent();
35951   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35952   const MIMetadata MIMD(MI);
35953 
35954   auto TMMImmToTMMReg = [](unsigned Imm) {
35955     assert (Imm < 8 && "Illegal tmm index");
35956     return X86::TMM0 + Imm;
35957   };
35958   switch (MI.getOpcode()) {
35959   default: llvm_unreachable("Unexpected instr type to insert");
35960   case X86::TLS_addr32:
35961   case X86::TLS_addr64:
35962   case X86::TLS_addrX32:
35963   case X86::TLS_base_addr32:
35964   case X86::TLS_base_addr64:
35965   case X86::TLS_base_addrX32:
35966     return EmitLoweredTLSAddr(MI, BB);
35967   case X86::INDIRECT_THUNK_CALL32:
35968   case X86::INDIRECT_THUNK_CALL64:
35969   case X86::INDIRECT_THUNK_TCRETURN32:
35970   case X86::INDIRECT_THUNK_TCRETURN64:
35971     return EmitLoweredIndirectThunk(MI, BB);
35972   case X86::CATCHRET:
35973     return EmitLoweredCatchRet(MI, BB);
35974   case X86::SEG_ALLOCA_32:
35975   case X86::SEG_ALLOCA_64:
35976     return EmitLoweredSegAlloca(MI, BB);
35977   case X86::PROBED_ALLOCA_32:
35978   case X86::PROBED_ALLOCA_64:
35979     return EmitLoweredProbedAlloca(MI, BB);
35980   case X86::TLSCall_32:
35981   case X86::TLSCall_64:
35982     return EmitLoweredTLSCall(MI, BB);
35983   case X86::CMOV_FR16:
35984   case X86::CMOV_FR16X:
35985   case X86::CMOV_FR32:
35986   case X86::CMOV_FR32X:
35987   case X86::CMOV_FR64:
35988   case X86::CMOV_FR64X:
35989   case X86::CMOV_GR8:
35990   case X86::CMOV_GR16:
35991   case X86::CMOV_GR32:
35992   case X86::CMOV_RFP32:
35993   case X86::CMOV_RFP64:
35994   case X86::CMOV_RFP80:
35995   case X86::CMOV_VR64:
35996   case X86::CMOV_VR128:
35997   case X86::CMOV_VR128X:
35998   case X86::CMOV_VR256:
35999   case X86::CMOV_VR256X:
36000   case X86::CMOV_VR512:
36001   case X86::CMOV_VK1:
36002   case X86::CMOV_VK2:
36003   case X86::CMOV_VK4:
36004   case X86::CMOV_VK8:
36005   case X86::CMOV_VK16:
36006   case X86::CMOV_VK32:
36007   case X86::CMOV_VK64:
36008     return EmitLoweredSelect(MI, BB);
36009 
36010   case X86::FP80_ADDr:
36011   case X86::FP80_ADDm32: {
36012     // Change the floating point control register to use double extended
36013     // precision when performing the addition.
36014     int OrigCWFrameIdx =
36015         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36016     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36017                       OrigCWFrameIdx);
36018 
36019     // Load the old value of the control word...
36020     Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36021     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36022                       OrigCWFrameIdx);
36023 
36024     // OR 0b11 into bit 8 and 9. 0b11 is the encoding for double extended
36025     // precision.
36026     Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36027     BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36028         .addReg(OldCW, RegState::Kill)
36029         .addImm(0x300);
36030 
36031     // Extract to 16 bits.
36032     Register NewCW16 =
36033         MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36034     BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36035         .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36036 
36037     // Prepare memory for FLDCW.
36038     int NewCWFrameIdx =
36039         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36040     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36041                       NewCWFrameIdx)
36042         .addReg(NewCW16, RegState::Kill);
36043 
36044     // Reload the modified control word now...
36045     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36046                       NewCWFrameIdx);
36047 
36048     // Do the addition.
36049     if (MI.getOpcode() == X86::FP80_ADDr) {
36050       BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80))
36051           .add(MI.getOperand(0))
36052           .add(MI.getOperand(1))
36053           .add(MI.getOperand(2));
36054     } else {
36055       BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80m32))
36056           .add(MI.getOperand(0))
36057           .add(MI.getOperand(1))
36058           .add(MI.getOperand(2))
36059           .add(MI.getOperand(3))
36060           .add(MI.getOperand(4))
36061           .add(MI.getOperand(5))
36062           .add(MI.getOperand(6));
36063     }
36064 
36065     // Reload the original control word now.
36066     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36067                       OrigCWFrameIdx);
36068 
36069     MI.eraseFromParent(); // The pseudo instruction is gone now.
36070     return BB;
36071   }
36072 
36073   case X86::FP32_TO_INT16_IN_MEM:
36074   case X86::FP32_TO_INT32_IN_MEM:
36075   case X86::FP32_TO_INT64_IN_MEM:
36076   case X86::FP64_TO_INT16_IN_MEM:
36077   case X86::FP64_TO_INT32_IN_MEM:
36078   case X86::FP64_TO_INT64_IN_MEM:
36079   case X86::FP80_TO_INT16_IN_MEM:
36080   case X86::FP80_TO_INT32_IN_MEM:
36081   case X86::FP80_TO_INT64_IN_MEM: {
36082     // Change the floating point control register to use "round towards zero"
36083     // mode when truncating to an integer value.
36084     int OrigCWFrameIdx =
36085         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36086     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36087                       OrigCWFrameIdx);
36088 
36089     // Load the old value of the control word...
36090     Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36091     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36092                       OrigCWFrameIdx);
36093 
36094     // OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
36095     Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36096     BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36097       .addReg(OldCW, RegState::Kill).addImm(0xC00);
36098 
36099     // Extract to 16 bits.
36100     Register NewCW16 =
36101         MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36102     BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36103       .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36104 
36105     // Prepare memory for FLDCW.
36106     int NewCWFrameIdx =
36107         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36108     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36109                       NewCWFrameIdx)
36110       .addReg(NewCW16, RegState::Kill);
36111 
36112     // Reload the modified control word now...
36113     addFrameReference(BuildMI(*BB, MI, MIMD,
36114                               TII->get(X86::FLDCW16m)), NewCWFrameIdx);
36115 
36116     // Get the X86 opcode to use.
36117     unsigned Opc;
36118     switch (MI.getOpcode()) {
36119     default: llvm_unreachable("illegal opcode!");
36120     case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
36121     case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
36122     case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
36123     case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
36124     case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
36125     case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
36126     case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
36127     case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
36128     case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
36129     }
36130 
36131     X86AddressMode AM = getAddressFromInstr(&MI, 0);
36132     addFullAddress(BuildMI(*BB, MI, MIMD, TII->get(Opc)), AM)
36133         .addReg(MI.getOperand(X86::AddrNumOperands).getReg());
36134 
36135     // Reload the original control word now.
36136     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36137                       OrigCWFrameIdx);
36138 
36139     MI.eraseFromParent(); // The pseudo instruction is gone now.
36140     return BB;
36141   }
36142 
36143   // xbegin
36144   case X86::XBEGIN:
36145     return emitXBegin(MI, BB, Subtarget.getInstrInfo());
36146 
36147   case X86::VAARG_64:
36148   case X86::VAARG_X32:
36149     return EmitVAARGWithCustomInserter(MI, BB);
36150 
36151   case X86::EH_SjLj_SetJmp32:
36152   case X86::EH_SjLj_SetJmp64:
36153     return emitEHSjLjSetJmp(MI, BB);
36154 
36155   case X86::EH_SjLj_LongJmp32:
36156   case X86::EH_SjLj_LongJmp64:
36157     return emitEHSjLjLongJmp(MI, BB);
36158 
36159   case X86::Int_eh_sjlj_setup_dispatch:
36160     return EmitSjLjDispatchBlock(MI, BB);
36161 
36162   case TargetOpcode::STATEPOINT:
36163     // As an implementation detail, STATEPOINT shares the STACKMAP format at
36164     // this point in the process.  We diverge later.
36165     return emitPatchPoint(MI, BB);
36166 
36167   case TargetOpcode::STACKMAP:
36168   case TargetOpcode::PATCHPOINT:
36169     return emitPatchPoint(MI, BB);
36170 
36171   case TargetOpcode::PATCHABLE_EVENT_CALL:
36172   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
36173     return BB;
36174 
36175   case X86::LCMPXCHG8B: {
36176     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36177     // In addition to 4 E[ABCD] registers implied by encoding, CMPXCHG8B
36178     // requires a memory operand. If it happens that current architecture is
36179     // i686 and for current function we need a base pointer
36180     // - which is ESI for i686 - register allocator would not be able to
36181     // allocate registers for an address in form of X(%reg, %reg, Y)
36182     // - there never would be enough unreserved registers during regalloc
36183     // (without the need for base ptr the only option would be X(%edi, %esi, Y).
36184     // We are giving a hand to register allocator by precomputing the address in
36185     // a new vreg using LEA.
36186 
36187     // If it is not i686 or there is no base pointer - nothing to do here.
36188     if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF))
36189       return BB;
36190 
36191     // Even though this code does not necessarily needs the base pointer to
36192     // be ESI, we check for that. The reason: if this assert fails, there are
36193     // some changes happened in the compiler base pointer handling, which most
36194     // probably have to be addressed somehow here.
36195     assert(TRI->getBaseRegister() == X86::ESI &&
36196            "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
36197            "base pointer in mind");
36198 
36199     MachineRegisterInfo &MRI = MF->getRegInfo();
36200     MVT SPTy = getPointerTy(MF->getDataLayout());
36201     const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
36202     Register computedAddrVReg = MRI.createVirtualRegister(AddrRegClass);
36203 
36204     X86AddressMode AM = getAddressFromInstr(&MI, 0);
36205     // Regalloc does not need any help when the memory operand of CMPXCHG8B
36206     // does not use index register.
36207     if (AM.IndexReg == X86::NoRegister)
36208       return BB;
36209 
36210     // After X86TargetLowering::ReplaceNodeResults CMPXCHG8B is glued to its
36211     // four operand definitions that are E[ABCD] registers. We skip them and
36212     // then insert the LEA.
36213     MachineBasicBlock::reverse_iterator RMBBI(MI.getReverseIterator());
36214     while (RMBBI != BB->rend() && (RMBBI->definesRegister(X86::EAX) ||
36215                                    RMBBI->definesRegister(X86::EBX) ||
36216                                    RMBBI->definesRegister(X86::ECX) ||
36217                                    RMBBI->definesRegister(X86::EDX))) {
36218       ++RMBBI;
36219     }
36220     MachineBasicBlock::iterator MBBI(RMBBI);
36221     addFullAddress(
36222         BuildMI(*BB, *MBBI, MIMD, TII->get(X86::LEA32r), computedAddrVReg), AM);
36223 
36224     setDirectAddressInInstr(&MI, 0, computedAddrVReg);
36225 
36226     return BB;
36227   }
36228   case X86::LCMPXCHG16B_NO_RBX: {
36229     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36230     Register BasePtr = TRI->getBaseRegister();
36231     if (TRI->hasBasePointer(*MF) &&
36232         (BasePtr == X86::RBX || BasePtr == X86::EBX)) {
36233       if (!BB->isLiveIn(BasePtr))
36234         BB->addLiveIn(BasePtr);
36235       // Save RBX into a virtual register.
36236       Register SaveRBX =
36237           MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36238       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36239           .addReg(X86::RBX);
36240       Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36241       MachineInstrBuilder MIB =
36242           BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst);
36243       for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36244         MIB.add(MI.getOperand(Idx));
36245       MIB.add(MI.getOperand(X86::AddrNumOperands));
36246       MIB.addReg(SaveRBX);
36247     } else {
36248       // Simple case, just copy the virtual register to RBX.
36249       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::RBX)
36250           .add(MI.getOperand(X86::AddrNumOperands));
36251       MachineInstrBuilder MIB =
36252           BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B));
36253       for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36254         MIB.add(MI.getOperand(Idx));
36255     }
36256     MI.eraseFromParent();
36257     return BB;
36258   }
36259   case X86::MWAITX: {
36260     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36261     Register BasePtr = TRI->getBaseRegister();
36262     bool IsRBX = (BasePtr == X86::RBX || BasePtr == X86::EBX);
36263     // If no need to save the base pointer, we generate MWAITXrrr,
36264     // else we generate pseudo MWAITX_SAVE_RBX.
36265     if (!IsRBX || !TRI->hasBasePointer(*MF)) {
36266       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36267           .addReg(MI.getOperand(0).getReg());
36268       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36269           .addReg(MI.getOperand(1).getReg());
36270       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EBX)
36271           .addReg(MI.getOperand(2).getReg());
36272       BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITXrrr));
36273       MI.eraseFromParent();
36274     } else {
36275       if (!BB->isLiveIn(BasePtr)) {
36276         BB->addLiveIn(BasePtr);
36277       }
36278       // Parameters can be copied into ECX and EAX but not EBX yet.
36279       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36280           .addReg(MI.getOperand(0).getReg());
36281       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36282           .addReg(MI.getOperand(1).getReg());
36283       assert(Subtarget.is64Bit() && "Expected 64-bit mode!");
36284       // Save RBX into a virtual register.
36285       Register SaveRBX =
36286           MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36287       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36288           .addReg(X86::RBX);
36289       // Generate mwaitx pseudo.
36290       Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36291       BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITX_SAVE_RBX))
36292           .addDef(Dst) // Destination tied in with SaveRBX.
36293           .addReg(MI.getOperand(2).getReg()) // input value of EBX.
36294           .addUse(SaveRBX);                  // Save of base pointer.
36295       MI.eraseFromParent();
36296     }
36297     return BB;
36298   }
36299   case TargetOpcode::PREALLOCATED_SETUP: {
36300     assert(Subtarget.is32Bit() && "preallocated only used in 32-bit");
36301     auto MFI = MF->getInfo<X86MachineFunctionInfo>();
36302     MFI->setHasPreallocatedCall(true);
36303     int64_t PreallocatedId = MI.getOperand(0).getImm();
36304     size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId);
36305     assert(StackAdjustment != 0 && "0 stack adjustment");
36306     LLVM_DEBUG(dbgs() << "PREALLOCATED_SETUP stack adjustment "
36307                       << StackAdjustment << "\n");
36308     BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP)
36309         .addReg(X86::ESP)
36310         .addImm(StackAdjustment);
36311     MI.eraseFromParent();
36312     return BB;
36313   }
36314   case TargetOpcode::PREALLOCATED_ARG: {
36315     assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit");
36316     int64_t PreallocatedId = MI.getOperand(1).getImm();
36317     int64_t ArgIdx = MI.getOperand(2).getImm();
36318     auto MFI = MF->getInfo<X86MachineFunctionInfo>();
36319     size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx];
36320     LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx
36321                       << ", arg offset " << ArgOffset << "\n");
36322     // stack pointer + offset
36323     addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r),
36324                          MI.getOperand(0).getReg()),
36325                  X86::ESP, false, ArgOffset);
36326     MI.eraseFromParent();
36327     return BB;
36328   }
36329   case X86::PTDPBSSD:
36330   case X86::PTDPBSUD:
36331   case X86::PTDPBUSD:
36332   case X86::PTDPBUUD:
36333   case X86::PTDPBF16PS:
36334   case X86::PTDPFP16PS: {
36335     unsigned Opc;
36336     switch (MI.getOpcode()) {
36337     default: llvm_unreachable("illegal opcode!");
36338     case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
36339     case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
36340     case X86::PTDPBUSD: Opc = X86::TDPBUSD; break;
36341     case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
36342     case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
36343     case X86::PTDPFP16PS: Opc = X86::TDPFP16PS; break;
36344     }
36345 
36346     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36347     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
36348     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
36349     MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
36350     MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
36351 
36352     MI.eraseFromParent(); // The pseudo is gone now.
36353     return BB;
36354   }
36355   case X86::PTILEZERO: {
36356     unsigned Imm = MI.getOperand(0).getImm();
36357     BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
36358     MI.eraseFromParent(); // The pseudo is gone now.
36359     return BB;
36360   }
36361   case X86::PTILELOADD:
36362   case X86::PTILELOADDT1:
36363   case X86::PTILESTORED: {
36364     unsigned Opc;
36365     switch (MI.getOpcode()) {
36366     default: llvm_unreachable("illegal opcode!");
36367     case X86::PTILELOADD:   Opc = X86::TILELOADD;   break;
36368     case X86::PTILELOADDT1: Opc = X86::TILELOADDT1; break;
36369     case X86::PTILESTORED:  Opc = X86::TILESTORED;  break;
36370     }
36371 
36372     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36373     unsigned CurOp = 0;
36374     if (Opc != X86::TILESTORED)
36375       MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
36376                  RegState::Define);
36377 
36378     MIB.add(MI.getOperand(CurOp++)); // base
36379     MIB.add(MI.getOperand(CurOp++)); // scale
36380     MIB.add(MI.getOperand(CurOp++)); // index -- stride
36381     MIB.add(MI.getOperand(CurOp++)); // displacement
36382     MIB.add(MI.getOperand(CurOp++)); // segment
36383 
36384     if (Opc == X86::TILESTORED)
36385       MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
36386                  RegState::Undef);
36387 
36388     MI.eraseFromParent(); // The pseudo is gone now.
36389     return BB;
36390   }
36391   case X86::PTCMMIMFP16PS:
36392   case X86::PTCMMRLFP16PS: {
36393     const MIMetadata MIMD(MI);
36394     unsigned Opc;
36395     switch (MI.getOpcode()) {
36396     default: llvm_unreachable("Unexpected instruction!");
36397     case X86::PTCMMIMFP16PS:     Opc = X86::TCMMIMFP16PS;     break;
36398     case X86::PTCMMRLFP16PS:     Opc = X86::TCMMRLFP16PS;     break;
36399     }
36400     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36401     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
36402     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
36403     MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
36404     MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
36405     MI.eraseFromParent(); // The pseudo is gone now.
36406     return BB;
36407   }
36408   }
36409 }
36410 
36411 //===----------------------------------------------------------------------===//
36412 //                           X86 Optimization Hooks
36413 //===----------------------------------------------------------------------===//
36414 
36415 bool
36416 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
36417                                                 const APInt &DemandedBits,
36418                                                 const APInt &DemandedElts,
36419                                                 TargetLoweringOpt &TLO) const {
36420   EVT VT = Op.getValueType();
36421   unsigned Opcode = Op.getOpcode();
36422   unsigned EltSize = VT.getScalarSizeInBits();
36423 
36424   if (VT.isVector()) {
36425     // If the constant is only all signbits in the active bits, then we should
36426     // extend it to the entire constant to allow it act as a boolean constant
36427     // vector.
36428     auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
36429       if (!ISD::isBuildVectorOfConstantSDNodes(V.getNode()))
36430         return false;
36431       for (unsigned i = 0, e = V.getNumOperands(); i != e; ++i) {
36432         if (!DemandedElts[i] || V.getOperand(i).isUndef())
36433           continue;
36434         const APInt &Val = V.getConstantOperandAPInt(i);
36435         if (Val.getBitWidth() > Val.getNumSignBits() &&
36436             Val.trunc(ActiveBits).getNumSignBits() == ActiveBits)
36437           return true;
36438       }
36439       return false;
36440     };
36441     // For vectors - if we have a constant, then try to sign extend.
36442     // TODO: Handle AND cases.
36443     unsigned ActiveBits = DemandedBits.getActiveBits();
36444     if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) &&
36445         (Opcode == ISD::OR || Opcode == ISD::XOR || Opcode == X86ISD::ANDNP) &&
36446         NeedsSignExtension(Op.getOperand(1), ActiveBits)) {
36447       EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits);
36448       EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT,
36449                                    VT.getVectorNumElements());
36450       SDValue NewC =
36451           TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT,
36452                           Op.getOperand(1), TLO.DAG.getValueType(ExtVT));
36453       SDValue NewOp =
36454           TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC);
36455       return TLO.CombineTo(Op, NewOp);
36456     }
36457     return false;
36458   }
36459 
36460   // Only optimize Ands to prevent shrinking a constant that could be
36461   // matched by movzx.
36462   if (Opcode != ISD::AND)
36463     return false;
36464 
36465   // Make sure the RHS really is a constant.
36466   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
36467   if (!C)
36468     return false;
36469 
36470   const APInt &Mask = C->getAPIntValue();
36471 
36472   // Clear all non-demanded bits initially.
36473   APInt ShrunkMask = Mask & DemandedBits;
36474 
36475   // Find the width of the shrunk mask.
36476   unsigned Width = ShrunkMask.getActiveBits();
36477 
36478   // If the mask is all 0s there's nothing to do here.
36479   if (Width == 0)
36480     return false;
36481 
36482   // Find the next power of 2 width, rounding up to a byte.
36483   Width = llvm::bit_ceil(std::max(Width, 8U));
36484   // Truncate the width to size to handle illegal types.
36485   Width = std::min(Width, EltSize);
36486 
36487   // Calculate a possible zero extend mask for this constant.
36488   APInt ZeroExtendMask = APInt::getLowBitsSet(EltSize, Width);
36489 
36490   // If we aren't changing the mask, just return true to keep it and prevent
36491   // the caller from optimizing.
36492   if (ZeroExtendMask == Mask)
36493     return true;
36494 
36495   // Make sure the new mask can be represented by a combination of mask bits
36496   // and non-demanded bits.
36497   if (!ZeroExtendMask.isSubsetOf(Mask | ~DemandedBits))
36498     return false;
36499 
36500   // Replace the constant with the zero extend mask.
36501   SDLoc DL(Op);
36502   SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
36503   SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
36504   return TLO.CombineTo(Op, NewOp);
36505 }
36506 
36507 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
36508                                                       KnownBits &Known,
36509                                                       const APInt &DemandedElts,
36510                                                       const SelectionDAG &DAG,
36511                                                       unsigned Depth) const {
36512   unsigned BitWidth = Known.getBitWidth();
36513   unsigned NumElts = DemandedElts.getBitWidth();
36514   unsigned Opc = Op.getOpcode();
36515   EVT VT = Op.getValueType();
36516   assert((Opc >= ISD::BUILTIN_OP_END ||
36517           Opc == ISD::INTRINSIC_WO_CHAIN ||
36518           Opc == ISD::INTRINSIC_W_CHAIN ||
36519           Opc == ISD::INTRINSIC_VOID) &&
36520          "Should use MaskedValueIsZero if you don't know whether Op"
36521          " is a target node!");
36522 
36523   Known.resetAll();
36524   switch (Opc) {
36525   default: break;
36526   case X86ISD::MUL_IMM: {
36527     KnownBits Known2;
36528     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36529     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36530     Known = KnownBits::mul(Known, Known2);
36531     break;
36532   }
36533   case X86ISD::SETCC:
36534     Known.Zero.setBitsFrom(1);
36535     break;
36536   case X86ISD::MOVMSK: {
36537     unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
36538     Known.Zero.setBitsFrom(NumLoBits);
36539     break;
36540   }
36541   case X86ISD::PEXTRB:
36542   case X86ISD::PEXTRW: {
36543     SDValue Src = Op.getOperand(0);
36544     EVT SrcVT = Src.getValueType();
36545     APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
36546                                             Op.getConstantOperandVal(1));
36547     Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
36548     Known = Known.anyextOrTrunc(BitWidth);
36549     Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits());
36550     break;
36551   }
36552   case X86ISD::VSRAI:
36553   case X86ISD::VSHLI:
36554   case X86ISD::VSRLI: {
36555     unsigned ShAmt = Op.getConstantOperandVal(1);
36556     if (ShAmt >= VT.getScalarSizeInBits()) {
36557       // Out of range logical bit shifts are guaranteed to be zero.
36558       // Out of range arithmetic bit shifts splat the sign bit.
36559       if (Opc != X86ISD::VSRAI) {
36560         Known.setAllZero();
36561         break;
36562       }
36563 
36564       ShAmt = VT.getScalarSizeInBits() - 1;
36565     }
36566 
36567     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36568     if (Opc == X86ISD::VSHLI) {
36569       Known.Zero <<= ShAmt;
36570       Known.One <<= ShAmt;
36571       // Low bits are known zero.
36572       Known.Zero.setLowBits(ShAmt);
36573     } else if (Opc == X86ISD::VSRLI) {
36574       Known.Zero.lshrInPlace(ShAmt);
36575       Known.One.lshrInPlace(ShAmt);
36576       // High bits are known zero.
36577       Known.Zero.setHighBits(ShAmt);
36578     } else {
36579       Known.Zero.ashrInPlace(ShAmt);
36580       Known.One.ashrInPlace(ShAmt);
36581     }
36582     break;
36583   }
36584   case X86ISD::PACKUS: {
36585     // PACKUS is just a truncation if the upper half is zero.
36586     APInt DemandedLHS, DemandedRHS;
36587     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
36588 
36589     Known.One = APInt::getAllOnes(BitWidth * 2);
36590     Known.Zero = APInt::getAllOnes(BitWidth * 2);
36591 
36592     KnownBits Known2;
36593     if (!!DemandedLHS) {
36594       Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
36595       Known = Known.intersectWith(Known2);
36596     }
36597     if (!!DemandedRHS) {
36598       Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
36599       Known = Known.intersectWith(Known2);
36600     }
36601 
36602     if (Known.countMinLeadingZeros() < BitWidth)
36603       Known.resetAll();
36604     Known = Known.trunc(BitWidth);
36605     break;
36606   }
36607   case X86ISD::VBROADCAST: {
36608     SDValue Src = Op.getOperand(0);
36609     if (!Src.getSimpleValueType().isVector()) {
36610       Known = DAG.computeKnownBits(Src, Depth + 1);
36611       return;
36612     }
36613     break;
36614   }
36615   case X86ISD::AND: {
36616     if (Op.getResNo() == 0) {
36617       KnownBits Known2;
36618       Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36619       Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36620       Known &= Known2;
36621     }
36622     break;
36623   }
36624   case X86ISD::ANDNP: {
36625     KnownBits Known2;
36626     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36627     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36628 
36629     // ANDNP = (~X & Y);
36630     Known.One &= Known2.Zero;
36631     Known.Zero |= Known2.One;
36632     break;
36633   }
36634   case X86ISD::FOR: {
36635     KnownBits Known2;
36636     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36637     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36638 
36639     Known |= Known2;
36640     break;
36641   }
36642   case X86ISD::PSADBW: {
36643     assert(VT.getScalarType() == MVT::i64 &&
36644            Op.getOperand(0).getValueType().getScalarType() == MVT::i8 &&
36645            "Unexpected PSADBW types");
36646 
36647     // PSADBW - fills low 16 bits and zeros upper 48 bits of each i64 result.
36648     Known.Zero.setBitsFrom(16);
36649     break;
36650   }
36651   case X86ISD::PCMPGT:
36652   case X86ISD::PCMPEQ: {
36653     KnownBits KnownLhs =
36654         DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36655     KnownBits KnownRhs =
36656         DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36657     std::optional<bool> Res = Opc == X86ISD::PCMPEQ
36658                                   ? KnownBits::eq(KnownLhs, KnownRhs)
36659                                   : KnownBits::sgt(KnownLhs, KnownRhs);
36660     if (Res) {
36661       if (*Res)
36662         Known.setAllOnes();
36663       else
36664         Known.setAllZero();
36665     }
36666     break;
36667   }
36668   case X86ISD::PMULUDQ: {
36669     KnownBits Known2;
36670     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36671     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36672 
36673     Known = Known.trunc(BitWidth / 2).zext(BitWidth);
36674     Known2 = Known2.trunc(BitWidth / 2).zext(BitWidth);
36675     Known = KnownBits::mul(Known, Known2);
36676     break;
36677   }
36678   case X86ISD::CMOV: {
36679     Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
36680     // If we don't know any bits, early out.
36681     if (Known.isUnknown())
36682       break;
36683     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
36684 
36685     // Only known if known in both the LHS and RHS.
36686     Known = Known.intersectWith(Known2);
36687     break;
36688   }
36689   case X86ISD::BEXTR:
36690   case X86ISD::BEXTRI: {
36691     SDValue Op0 = Op.getOperand(0);
36692     SDValue Op1 = Op.getOperand(1);
36693 
36694     if (auto* Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
36695       unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
36696       unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
36697 
36698       // If the length is 0, the result is 0.
36699       if (Length == 0) {
36700         Known.setAllZero();
36701         break;
36702       }
36703 
36704       if ((Shift + Length) <= BitWidth) {
36705         Known = DAG.computeKnownBits(Op0, Depth + 1);
36706         Known = Known.extractBits(Length, Shift);
36707         Known = Known.zextOrTrunc(BitWidth);
36708       }
36709     }
36710     break;
36711   }
36712   case X86ISD::PDEP: {
36713     KnownBits Known2;
36714     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36715     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36716     // Zeros are retained from the mask operand. But not ones.
36717     Known.One.clearAllBits();
36718     // The result will have at least as many trailing zeros as the non-mask
36719     // operand since bits can only map to the same or higher bit position.
36720     Known.Zero.setLowBits(Known2.countMinTrailingZeros());
36721     break;
36722   }
36723   case X86ISD::PEXT: {
36724     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36725     // The result has as many leading zeros as the number of zeroes in the mask.
36726     unsigned Count = Known.Zero.popcount();
36727     Known.Zero = APInt::getHighBitsSet(BitWidth, Count);
36728     Known.One.clearAllBits();
36729     break;
36730   }
36731   case X86ISD::VTRUNC:
36732   case X86ISD::VTRUNCS:
36733   case X86ISD::VTRUNCUS:
36734   case X86ISD::CVTSI2P:
36735   case X86ISD::CVTUI2P:
36736   case X86ISD::CVTP2SI:
36737   case X86ISD::CVTP2UI:
36738   case X86ISD::MCVTP2SI:
36739   case X86ISD::MCVTP2UI:
36740   case X86ISD::CVTTP2SI:
36741   case X86ISD::CVTTP2UI:
36742   case X86ISD::MCVTTP2SI:
36743   case X86ISD::MCVTTP2UI:
36744   case X86ISD::MCVTSI2P:
36745   case X86ISD::MCVTUI2P:
36746   case X86ISD::VFPROUND:
36747   case X86ISD::VMFPROUND:
36748   case X86ISD::CVTPS2PH:
36749   case X86ISD::MCVTPS2PH: {
36750     // Truncations/Conversions - upper elements are known zero.
36751     EVT SrcVT = Op.getOperand(0).getValueType();
36752     if (SrcVT.isVector()) {
36753       unsigned NumSrcElts = SrcVT.getVectorNumElements();
36754       if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
36755         Known.setAllZero();
36756     }
36757     break;
36758   }
36759   case X86ISD::STRICT_CVTTP2SI:
36760   case X86ISD::STRICT_CVTTP2UI:
36761   case X86ISD::STRICT_CVTSI2P:
36762   case X86ISD::STRICT_CVTUI2P:
36763   case X86ISD::STRICT_VFPROUND:
36764   case X86ISD::STRICT_CVTPS2PH: {
36765     // Strict Conversions - upper elements are known zero.
36766     EVT SrcVT = Op.getOperand(1).getValueType();
36767     if (SrcVT.isVector()) {
36768       unsigned NumSrcElts = SrcVT.getVectorNumElements();
36769       if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
36770         Known.setAllZero();
36771     }
36772     break;
36773   }
36774   case X86ISD::MOVQ2DQ: {
36775     // Move from MMX to XMM. Upper half of XMM should be 0.
36776     if (DemandedElts.countr_zero() >= (NumElts / 2))
36777       Known.setAllZero();
36778     break;
36779   }
36780   case X86ISD::VBROADCAST_LOAD: {
36781     APInt UndefElts;
36782     SmallVector<APInt, 16> EltBits;
36783     if (getTargetConstantBitsFromNode(Op, BitWidth, UndefElts, EltBits,
36784                                       /*AllowWholeUndefs*/ false,
36785                                       /*AllowPartialUndefs*/ false)) {
36786       Known.Zero.setAllBits();
36787       Known.One.setAllBits();
36788       for (unsigned I = 0; I != NumElts; ++I) {
36789         if (!DemandedElts[I])
36790           continue;
36791         if (UndefElts[I]) {
36792           Known.resetAll();
36793           break;
36794         }
36795         KnownBits Known2 = KnownBits::makeConstant(EltBits[I]);
36796         Known = Known.intersectWith(Known2);
36797       }
36798       return;
36799     }
36800     break;
36801   }
36802   }
36803 
36804   // Handle target shuffles.
36805   // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
36806   if (isTargetShuffle(Opc)) {
36807     SmallVector<int, 64> Mask;
36808     SmallVector<SDValue, 2> Ops;
36809     if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) {
36810       unsigned NumOps = Ops.size();
36811       unsigned NumElts = VT.getVectorNumElements();
36812       if (Mask.size() == NumElts) {
36813         SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
36814         Known.Zero.setAllBits(); Known.One.setAllBits();
36815         for (unsigned i = 0; i != NumElts; ++i) {
36816           if (!DemandedElts[i])
36817             continue;
36818           int M = Mask[i];
36819           if (M == SM_SentinelUndef) {
36820             // For UNDEF elements, we don't know anything about the common state
36821             // of the shuffle result.
36822             Known.resetAll();
36823             break;
36824           }
36825           if (M == SM_SentinelZero) {
36826             Known.One.clearAllBits();
36827             continue;
36828           }
36829           assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
36830                  "Shuffle index out of range");
36831 
36832           unsigned OpIdx = (unsigned)M / NumElts;
36833           unsigned EltIdx = (unsigned)M % NumElts;
36834           if (Ops[OpIdx].getValueType() != VT) {
36835             // TODO - handle target shuffle ops with different value types.
36836             Known.resetAll();
36837             break;
36838           }
36839           DemandedOps[OpIdx].setBit(EltIdx);
36840         }
36841         // Known bits are the values that are shared by every demanded element.
36842         for (unsigned i = 0; i != NumOps && !Known.isUnknown(); ++i) {
36843           if (!DemandedOps[i])
36844             continue;
36845           KnownBits Known2 =
36846               DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
36847           Known = Known.intersectWith(Known2);
36848         }
36849       }
36850     }
36851   }
36852 }
36853 
36854 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
36855     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
36856     unsigned Depth) const {
36857   EVT VT = Op.getValueType();
36858   unsigned VTBits = VT.getScalarSizeInBits();
36859   unsigned Opcode = Op.getOpcode();
36860   switch (Opcode) {
36861   case X86ISD::SETCC_CARRY:
36862     // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
36863     return VTBits;
36864 
36865   case X86ISD::VTRUNC: {
36866     SDValue Src = Op.getOperand(0);
36867     MVT SrcVT = Src.getSimpleValueType();
36868     unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
36869     assert(VTBits < NumSrcBits && "Illegal truncation input type");
36870     APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
36871     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
36872     if (Tmp > (NumSrcBits - VTBits))
36873       return Tmp - (NumSrcBits - VTBits);
36874     return 1;
36875   }
36876 
36877   case X86ISD::PACKSS: {
36878     // PACKSS is just a truncation if the sign bits extend to the packed size.
36879     APInt DemandedLHS, DemandedRHS;
36880     getPackDemandedElts(Op.getValueType(), DemandedElts, DemandedLHS,
36881                         DemandedRHS);
36882 
36883     // Helper to detect PACKSSDW(BITCAST(PACKSSDW(X)),BITCAST(PACKSSDW(Y)))
36884     // patterns often used to compact vXi64 allsignbit patterns.
36885     auto NumSignBitsPACKSS = [&](SDValue V, const APInt &Elts) -> unsigned {
36886       SDValue BC = peekThroughBitcasts(V);
36887       if (BC.getOpcode() == X86ISD::PACKSS &&
36888           BC.getScalarValueSizeInBits() == 16 &&
36889           V.getScalarValueSizeInBits() == 32) {
36890         SDValue BC0 = peekThroughBitcasts(BC.getOperand(0));
36891         SDValue BC1 = peekThroughBitcasts(BC.getOperand(1));
36892         if (BC0.getScalarValueSizeInBits() == 64 &&
36893             BC1.getScalarValueSizeInBits() == 64 &&
36894             DAG.ComputeNumSignBits(BC0, Depth + 1) == 64 &&
36895             DAG.ComputeNumSignBits(BC1, Depth + 1) == 64)
36896           return 32;
36897       }
36898       return DAG.ComputeNumSignBits(V, Elts, Depth + 1);
36899     };
36900 
36901     unsigned SrcBits = Op.getOperand(0).getScalarValueSizeInBits();
36902     unsigned Tmp0 = SrcBits, Tmp1 = SrcBits;
36903     if (!!DemandedLHS)
36904       Tmp0 = NumSignBitsPACKSS(Op.getOperand(0), DemandedLHS);
36905     if (!!DemandedRHS)
36906       Tmp1 = NumSignBitsPACKSS(Op.getOperand(1), DemandedRHS);
36907     unsigned Tmp = std::min(Tmp0, Tmp1);
36908     if (Tmp > (SrcBits - VTBits))
36909       return Tmp - (SrcBits - VTBits);
36910     return 1;
36911   }
36912 
36913   case X86ISD::VBROADCAST: {
36914     SDValue Src = Op.getOperand(0);
36915     if (!Src.getSimpleValueType().isVector())
36916       return DAG.ComputeNumSignBits(Src, Depth + 1);
36917     break;
36918   }
36919 
36920   case X86ISD::VSHLI: {
36921     SDValue Src = Op.getOperand(0);
36922     const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
36923     if (ShiftVal.uge(VTBits))
36924       return VTBits; // Shifted all bits out --> zero.
36925     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
36926     if (ShiftVal.uge(Tmp))
36927       return 1; // Shifted all sign bits out --> unknown.
36928     return Tmp - ShiftVal.getZExtValue();
36929   }
36930 
36931   case X86ISD::VSRAI: {
36932     SDValue Src = Op.getOperand(0);
36933     APInt ShiftVal = Op.getConstantOperandAPInt(1);
36934     if (ShiftVal.uge(VTBits - 1))
36935       return VTBits; // Sign splat.
36936     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
36937     ShiftVal += Tmp;
36938     return ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
36939   }
36940 
36941   case X86ISD::FSETCC:
36942     // cmpss/cmpsd return zero/all-bits result values in the bottom element.
36943     if (VT == MVT::f32 || VT == MVT::f64 ||
36944         ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1))
36945       return VTBits;
36946     break;
36947 
36948   case X86ISD::PCMPGT:
36949   case X86ISD::PCMPEQ:
36950   case X86ISD::CMPP:
36951   case X86ISD::VPCOM:
36952   case X86ISD::VPCOMU:
36953     // Vector compares return zero/all-bits result values.
36954     return VTBits;
36955 
36956   case X86ISD::ANDNP: {
36957     unsigned Tmp0 =
36958         DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
36959     if (Tmp0 == 1) return 1; // Early out.
36960     unsigned Tmp1 =
36961         DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
36962     return std::min(Tmp0, Tmp1);
36963   }
36964 
36965   case X86ISD::CMOV: {
36966     unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
36967     if (Tmp0 == 1) return 1;  // Early out.
36968     unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
36969     return std::min(Tmp0, Tmp1);
36970   }
36971   }
36972 
36973   // Handle target shuffles.
36974   // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
36975   if (isTargetShuffle(Opcode)) {
36976     SmallVector<int, 64> Mask;
36977     SmallVector<SDValue, 2> Ops;
36978     if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) {
36979       unsigned NumOps = Ops.size();
36980       unsigned NumElts = VT.getVectorNumElements();
36981       if (Mask.size() == NumElts) {
36982         SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
36983         for (unsigned i = 0; i != NumElts; ++i) {
36984           if (!DemandedElts[i])
36985             continue;
36986           int M = Mask[i];
36987           if (M == SM_SentinelUndef) {
36988             // For UNDEF elements, we don't know anything about the common state
36989             // of the shuffle result.
36990             return 1;
36991           } else if (M == SM_SentinelZero) {
36992             // Zero = all sign bits.
36993             continue;
36994           }
36995           assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
36996                  "Shuffle index out of range");
36997 
36998           unsigned OpIdx = (unsigned)M / NumElts;
36999           unsigned EltIdx = (unsigned)M % NumElts;
37000           if (Ops[OpIdx].getValueType() != VT) {
37001             // TODO - handle target shuffle ops with different value types.
37002             return 1;
37003           }
37004           DemandedOps[OpIdx].setBit(EltIdx);
37005         }
37006         unsigned Tmp0 = VTBits;
37007         for (unsigned i = 0; i != NumOps && Tmp0 > 1; ++i) {
37008           if (!DemandedOps[i])
37009             continue;
37010           unsigned Tmp1 =
37011               DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1);
37012           Tmp0 = std::min(Tmp0, Tmp1);
37013         }
37014         return Tmp0;
37015       }
37016     }
37017   }
37018 
37019   // Fallback case.
37020   return 1;
37021 }
37022 
37023 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
37024   if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
37025     return N->getOperand(0);
37026   return N;
37027 }
37028 
37029 // Helper to look for a normal load that can be narrowed into a vzload with the
37030 // specified VT and memory VT. Returns SDValue() on failure.
37031 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,
37032                                   SelectionDAG &DAG) {
37033   // Can't if the load is volatile or atomic.
37034   if (!LN->isSimple())
37035     return SDValue();
37036 
37037   SDVTList Tys = DAG.getVTList(VT, MVT::Other);
37038   SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
37039   return DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, SDLoc(LN), Tys, Ops, MemVT,
37040                                  LN->getPointerInfo(), LN->getOriginalAlign(),
37041                                  LN->getMemOperand()->getFlags());
37042 }
37043 
37044 // Attempt to match a combined shuffle mask against supported unary shuffle
37045 // instructions.
37046 // TODO: Investigate sharing more of this with shuffle lowering.
37047 static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
37048                               bool AllowFloatDomain, bool AllowIntDomain,
37049                               SDValue V1, const SelectionDAG &DAG,
37050                               const X86Subtarget &Subtarget, unsigned &Shuffle,
37051                               MVT &SrcVT, MVT &DstVT) {
37052   unsigned NumMaskElts = Mask.size();
37053   unsigned MaskEltSize = MaskVT.getScalarSizeInBits();
37054 
37055   // Match against a VZEXT_MOVL vXi32 and vXi16 zero-extending instruction.
37056   if (Mask[0] == 0 &&
37057       (MaskEltSize == 32 || (MaskEltSize == 16 && Subtarget.hasFP16()))) {
37058     if ((isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) ||
37059         (V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
37060          isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) {
37061       Shuffle = X86ISD::VZEXT_MOVL;
37062       if (MaskEltSize == 16)
37063         SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37064       else
37065         SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37066       return true;
37067     }
37068   }
37069 
37070   // Match against a ANY/SIGN/ZERO_EXTEND_VECTOR_INREG instruction.
37071   // TODO: Add 512-bit vector support (split AVX512F and AVX512BW).
37072   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) ||
37073                          (MaskVT.is256BitVector() && Subtarget.hasInt256()))) {
37074     unsigned MaxScale = 64 / MaskEltSize;
37075     bool UseSign = V1.getScalarValueSizeInBits() == MaskEltSize &&
37076                    DAG.ComputeNumSignBits(V1) == MaskEltSize;
37077     for (unsigned Scale = 2; Scale <= MaxScale; Scale *= 2) {
37078       bool MatchAny = true;
37079       bool MatchZero = true;
37080       bool MatchSign = UseSign;
37081       unsigned NumDstElts = NumMaskElts / Scale;
37082       for (unsigned i = 0;
37083            i != NumDstElts && (MatchAny || MatchSign || MatchZero); ++i) {
37084         if (!isUndefOrEqual(Mask[i * Scale], (int)i)) {
37085           MatchAny = MatchSign = MatchZero = false;
37086           break;
37087         }
37088         unsigned Pos = (i * Scale) + 1;
37089         unsigned Len = Scale - 1;
37090         MatchAny &= isUndefInRange(Mask, Pos, Len);
37091         MatchZero &= isUndefOrZeroInRange(Mask, Pos, Len);
37092         MatchSign &= isUndefOrEqualInRange(Mask, (int)i, Pos, Len);
37093       }
37094       if (MatchAny || MatchSign || MatchZero) {
37095         assert((MatchSign || MatchZero) &&
37096                "Failed to match sext/zext but matched aext?");
37097         unsigned SrcSize = std::max(128u, NumDstElts * MaskEltSize);
37098         MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType()
37099                                           : MVT::getIntegerVT(MaskEltSize);
37100         SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize);
37101 
37102         Shuffle = unsigned(
37103             MatchAny ? ISD::ANY_EXTEND
37104                      : (MatchSign ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND));
37105         if (SrcVT.getVectorNumElements() != NumDstElts)
37106           Shuffle = DAG.getOpcode_EXTEND_VECTOR_INREG(Shuffle);
37107 
37108         DstVT = MVT::getIntegerVT(Scale * MaskEltSize);
37109         DstVT = MVT::getVectorVT(DstVT, NumDstElts);
37110         return true;
37111       }
37112     }
37113   }
37114 
37115   // Match against a VZEXT_MOVL instruction, SSE1 only supports 32-bits (MOVSS).
37116   if (((MaskEltSize == 32) || (MaskEltSize == 64 && Subtarget.hasSSE2()) ||
37117        (MaskEltSize == 16 && Subtarget.hasFP16())) &&
37118       isUndefOrEqual(Mask[0], 0) &&
37119       isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) {
37120     Shuffle = X86ISD::VZEXT_MOVL;
37121     if (MaskEltSize == 16)
37122       SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37123     else
37124       SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37125     return true;
37126   }
37127 
37128   // Check if we have SSE3 which will let us use MOVDDUP etc. The
37129   // instructions are no slower than UNPCKLPD but has the option to
37130   // fold the input operand into even an unaligned memory load.
37131   if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) {
37132     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG, V1)) {
37133       Shuffle = X86ISD::MOVDDUP;
37134       SrcVT = DstVT = MVT::v2f64;
37135       return true;
37136     }
37137     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
37138       Shuffle = X86ISD::MOVSLDUP;
37139       SrcVT = DstVT = MVT::v4f32;
37140       return true;
37141     }
37142     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, DAG, V1)) {
37143       Shuffle = X86ISD::MOVSHDUP;
37144       SrcVT = DstVT = MVT::v4f32;
37145       return true;
37146     }
37147   }
37148 
37149   if (MaskVT.is256BitVector() && AllowFloatDomain) {
37150     assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles");
37151     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
37152       Shuffle = X86ISD::MOVDDUP;
37153       SrcVT = DstVT = MVT::v4f64;
37154       return true;
37155     }
37156     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
37157                                   V1)) {
37158       Shuffle = X86ISD::MOVSLDUP;
37159       SrcVT = DstVT = MVT::v8f32;
37160       return true;
37161     }
37162     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, DAG,
37163                                   V1)) {
37164       Shuffle = X86ISD::MOVSHDUP;
37165       SrcVT = DstVT = MVT::v8f32;
37166       return true;
37167     }
37168   }
37169 
37170   if (MaskVT.is512BitVector() && AllowFloatDomain) {
37171     assert(Subtarget.hasAVX512() &&
37172            "AVX512 required for 512-bit vector shuffles");
37173     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
37174                                   V1)) {
37175       Shuffle = X86ISD::MOVDDUP;
37176       SrcVT = DstVT = MVT::v8f64;
37177       return true;
37178     }
37179     if (isTargetShuffleEquivalent(
37180             MaskVT, Mask,
37181             {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, DAG, V1)) {
37182       Shuffle = X86ISD::MOVSLDUP;
37183       SrcVT = DstVT = MVT::v16f32;
37184       return true;
37185     }
37186     if (isTargetShuffleEquivalent(
37187             MaskVT, Mask,
37188             {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, DAG, V1)) {
37189       Shuffle = X86ISD::MOVSHDUP;
37190       SrcVT = DstVT = MVT::v16f32;
37191       return true;
37192     }
37193   }
37194 
37195   return false;
37196 }
37197 
37198 // Attempt to match a combined shuffle mask against supported unary immediate
37199 // permute instructions.
37200 // TODO: Investigate sharing more of this with shuffle lowering.
37201 static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
37202                                      const APInt &Zeroable,
37203                                      bool AllowFloatDomain, bool AllowIntDomain,
37204                                      const SelectionDAG &DAG,
37205                                      const X86Subtarget &Subtarget,
37206                                      unsigned &Shuffle, MVT &ShuffleVT,
37207                                      unsigned &PermuteImm) {
37208   unsigned NumMaskElts = Mask.size();
37209   unsigned InputSizeInBits = MaskVT.getSizeInBits();
37210   unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts;
37211   MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits);
37212   bool ContainsZeros = isAnyZero(Mask);
37213 
37214   // Handle VPERMI/VPERMILPD vXi64/vXi64 patterns.
37215   if (!ContainsZeros && MaskScalarSizeInBits == 64) {
37216     // Check for lane crossing permutes.
37217     if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) {
37218       // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+).
37219       if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) {
37220         Shuffle = X86ISD::VPERMI;
37221         ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64);
37222         PermuteImm = getV4X86ShuffleImm(Mask);
37223         return true;
37224       }
37225       if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) {
37226         SmallVector<int, 4> RepeatedMask;
37227         if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) {
37228           Shuffle = X86ISD::VPERMI;
37229           ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64);
37230           PermuteImm = getV4X86ShuffleImm(RepeatedMask);
37231           return true;
37232         }
37233       }
37234     } else if (AllowFloatDomain && Subtarget.hasAVX()) {
37235       // VPERMILPD can permute with a non-repeating shuffle.
37236       Shuffle = X86ISD::VPERMILPI;
37237       ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size());
37238       PermuteImm = 0;
37239       for (int i = 0, e = Mask.size(); i != e; ++i) {
37240         int M = Mask[i];
37241         if (M == SM_SentinelUndef)
37242           continue;
37243         assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index");
37244         PermuteImm |= (M & 1) << i;
37245       }
37246       return true;
37247     }
37248   }
37249 
37250   // We are checking for shuffle match or shift match. Loop twice so we can
37251   // order which we try and match first depending on target preference.
37252   for (unsigned Order = 0; Order < 2; ++Order) {
37253     if (Subtarget.preferLowerShuffleAsShift() ? (Order == 1) : (Order == 0)) {
37254       // Handle PSHUFD/VPERMILPI vXi32/vXf32 repeated patterns.
37255       // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we
37256       // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here).
37257       if ((MaskScalarSizeInBits == 64 || MaskScalarSizeInBits == 32) &&
37258           !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) {
37259         SmallVector<int, 4> RepeatedMask;
37260         if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
37261           // Narrow the repeated mask to create 32-bit element permutes.
37262           SmallVector<int, 4> WordMask = RepeatedMask;
37263           if (MaskScalarSizeInBits == 64)
37264             narrowShuffleMaskElts(2, RepeatedMask, WordMask);
37265 
37266           Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
37267           ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);
37268           ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32);
37269           PermuteImm = getV4X86ShuffleImm(WordMask);
37270           return true;
37271         }
37272       }
37273 
37274       // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns.
37275       if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16 &&
37276           ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37277            (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37278            (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
37279         SmallVector<int, 4> RepeatedMask;
37280         if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
37281           ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4);
37282           ArrayRef<int> HiMask(RepeatedMask.data() + 4, 4);
37283 
37284           // PSHUFLW: permute lower 4 elements only.
37285           if (isUndefOrInRange(LoMask, 0, 4) &&
37286               isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
37287             Shuffle = X86ISD::PSHUFLW;
37288             ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
37289             PermuteImm = getV4X86ShuffleImm(LoMask);
37290             return true;
37291           }
37292 
37293           // PSHUFHW: permute upper 4 elements only.
37294           if (isUndefOrInRange(HiMask, 4, 8) &&
37295               isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
37296             // Offset the HiMask so that we can create the shuffle immediate.
37297             int OffsetHiMask[4];
37298             for (int i = 0; i != 4; ++i)
37299               OffsetHiMask[i] = (HiMask[i] < 0 ? HiMask[i] : HiMask[i] - 4);
37300 
37301             Shuffle = X86ISD::PSHUFHW;
37302             ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
37303             PermuteImm = getV4X86ShuffleImm(OffsetHiMask);
37304             return true;
37305           }
37306         }
37307       }
37308     } else {
37309       // Attempt to match against bit rotates.
37310       if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits < 64 &&
37311           ((MaskVT.is128BitVector() && Subtarget.hasXOP()) ||
37312            Subtarget.hasAVX512())) {
37313         int RotateAmt = matchShuffleAsBitRotate(ShuffleVT, MaskScalarSizeInBits,
37314                                                 Subtarget, Mask);
37315         if (0 < RotateAmt) {
37316           Shuffle = X86ISD::VROTLI;
37317           PermuteImm = (unsigned)RotateAmt;
37318           return true;
37319         }
37320       }
37321     }
37322     // Attempt to match against byte/bit shifts.
37323     if (AllowIntDomain &&
37324         ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37325          (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37326          (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37327       int ShiftAmt =
37328           matchShuffleAsShift(ShuffleVT, Shuffle, MaskScalarSizeInBits, Mask, 0,
37329                               Zeroable, Subtarget);
37330       if (0 < ShiftAmt && (!ShuffleVT.is512BitVector() || Subtarget.hasBWI() ||
37331                            32 <= ShuffleVT.getScalarSizeInBits())) {
37332         // Byte shifts can be slower so only match them on second attempt.
37333         if (Order == 0 &&
37334             (Shuffle == X86ISD::VSHLDQ || Shuffle == X86ISD::VSRLDQ))
37335           continue;
37336 
37337         PermuteImm = (unsigned)ShiftAmt;
37338         return true;
37339       }
37340 
37341     }
37342   }
37343 
37344   return false;
37345 }
37346 
37347 // Attempt to match a combined unary shuffle mask against supported binary
37348 // shuffle instructions.
37349 // TODO: Investigate sharing more of this with shuffle lowering.
37350 static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
37351                                bool AllowFloatDomain, bool AllowIntDomain,
37352                                SDValue &V1, SDValue &V2, const SDLoc &DL,
37353                                SelectionDAG &DAG, const X86Subtarget &Subtarget,
37354                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
37355                                bool IsUnary) {
37356   unsigned NumMaskElts = Mask.size();
37357   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
37358   unsigned SizeInBits = MaskVT.getSizeInBits();
37359 
37360   if (MaskVT.is128BitVector()) {
37361     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG) &&
37362         AllowFloatDomain) {
37363       V2 = V1;
37364       V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);
37365       Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS;
37366       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
37367       return true;
37368     }
37369     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}, DAG) &&
37370         AllowFloatDomain) {
37371       V2 = V1;
37372       Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS;
37373       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
37374       return true;
37375     }
37376     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}, DAG) &&
37377         Subtarget.hasSSE2() && (AllowFloatDomain || !Subtarget.hasSSE41())) {
37378       std::swap(V1, V2);
37379       Shuffle = X86ISD::MOVSD;
37380       SrcVT = DstVT = MVT::v2f64;
37381       return true;
37382     }
37383     if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG) &&
37384         (AllowFloatDomain || !Subtarget.hasSSE41())) {
37385       Shuffle = X86ISD::MOVSS;
37386       SrcVT = DstVT = MVT::v4f32;
37387       return true;
37388     }
37389     if (isTargetShuffleEquivalent(MaskVT, Mask, {8, 1, 2, 3, 4, 5, 6, 7},
37390                                   DAG) &&
37391         Subtarget.hasFP16()) {
37392       Shuffle = X86ISD::MOVSH;
37393       SrcVT = DstVT = MVT::v8f16;
37394       return true;
37395     }
37396   }
37397 
37398   // Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
37399   if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
37400       ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
37401       ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
37402     if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
37403                              Subtarget)) {
37404       DstVT = MaskVT;
37405       return true;
37406     }
37407   }
37408   // TODO: Can we handle this inside matchShuffleWithPACK?
37409   if (MaskVT == MVT::v4i32 && Subtarget.hasSSE2() &&
37410       isTargetShuffleEquivalent(MaskVT, Mask, {0, 2, 4, 6}, DAG) &&
37411       V1.getScalarValueSizeInBits() == 64 &&
37412       V2.getScalarValueSizeInBits() == 64) {
37413     // Use (SSE41) PACKUSWD if the leading zerobits goto the lowest 16-bits.
37414     unsigned MinLZV1 = DAG.computeKnownBits(V1).countMinLeadingZeros();
37415     unsigned MinLZV2 = DAG.computeKnownBits(V2).countMinLeadingZeros();
37416     if (Subtarget.hasSSE41() && MinLZV1 >= 48 && MinLZV2 >= 48) {
37417       SrcVT = MVT::v4i32;
37418       DstVT = MVT::v8i16;
37419       Shuffle = X86ISD::PACKUS;
37420       return true;
37421     }
37422     // Use PACKUSBW if the leading zerobits goto the lowest 8-bits.
37423     if (MinLZV1 >= 56 && MinLZV2 >= 56) {
37424       SrcVT = MVT::v8i16;
37425       DstVT = MVT::v16i8;
37426       Shuffle = X86ISD::PACKUS;
37427       return true;
37428     }
37429     // Use PACKSSWD if the signbits extend to the lowest 16-bits.
37430     if (DAG.ComputeNumSignBits(V1) > 48 && DAG.ComputeNumSignBits(V2) > 48) {
37431       SrcVT = MVT::v4i32;
37432       DstVT = MVT::v8i16;
37433       Shuffle = X86ISD::PACKSS;
37434       return true;
37435     }
37436   }
37437 
37438   // Attempt to match against either a unary or binary UNPCKL/UNPCKH shuffle.
37439   if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) ||
37440       (MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37441       (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) ||
37442       (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37443       (MaskVT.is512BitVector() && Subtarget.hasAVX512() &&
37444        (32 <= EltSizeInBits || Subtarget.hasBWI()))) {
37445     if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
37446                               Subtarget)) {
37447       SrcVT = DstVT = MaskVT;
37448       if (MaskVT.is256BitVector() && !Subtarget.hasAVX2())
37449         SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64);
37450       return true;
37451     }
37452   }
37453 
37454   // Attempt to match against a OR if we're performing a blend shuffle and the
37455   // non-blended source element is zero in each case.
37456   // TODO: Handle cases where V1/V2 sizes doesn't match SizeInBits.
37457   if (SizeInBits == V1.getValueSizeInBits() &&
37458       SizeInBits == V2.getValueSizeInBits() &&
37459       (EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
37460       (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
37461     bool IsBlend = true;
37462     unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
37463     unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
37464     unsigned Scale1 = NumV1Elts / NumMaskElts;
37465     unsigned Scale2 = NumV2Elts / NumMaskElts;
37466     APInt DemandedZeroV1 = APInt::getZero(NumV1Elts);
37467     APInt DemandedZeroV2 = APInt::getZero(NumV2Elts);
37468     for (unsigned i = 0; i != NumMaskElts; ++i) {
37469       int M = Mask[i];
37470       if (M == SM_SentinelUndef)
37471         continue;
37472       if (M == SM_SentinelZero) {
37473         DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
37474         DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
37475         continue;
37476       }
37477       if (M == (int)i) {
37478         DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
37479         continue;
37480       }
37481       if (M == (int)(i + NumMaskElts)) {
37482         DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
37483         continue;
37484       }
37485       IsBlend = false;
37486       break;
37487     }
37488     if (IsBlend) {
37489       if (DAG.MaskedVectorIsZero(V1, DemandedZeroV1) &&
37490           DAG.MaskedVectorIsZero(V2, DemandedZeroV2)) {
37491         Shuffle = ISD::OR;
37492         SrcVT = DstVT = MaskVT.changeTypeToInteger();
37493         return true;
37494       }
37495       if (NumV1Elts == NumV2Elts && NumV1Elts == NumMaskElts) {
37496         // FIXME: handle mismatched sizes?
37497         // TODO: investigate if `ISD::OR` handling in
37498         // `TargetLowering::SimplifyDemandedVectorElts` can be improved instead.
37499         auto computeKnownBitsElementWise = [&DAG](SDValue V) {
37500           unsigned NumElts = V.getValueType().getVectorNumElements();
37501           KnownBits Known(NumElts);
37502           for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
37503             APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
37504             KnownBits PeepholeKnown = DAG.computeKnownBits(V, Mask);
37505             if (PeepholeKnown.isZero())
37506               Known.Zero.setBit(EltIdx);
37507             if (PeepholeKnown.isAllOnes())
37508               Known.One.setBit(EltIdx);
37509           }
37510           return Known;
37511         };
37512 
37513         KnownBits V1Known = computeKnownBitsElementWise(V1);
37514         KnownBits V2Known = computeKnownBitsElementWise(V2);
37515 
37516         for (unsigned i = 0; i != NumMaskElts && IsBlend; ++i) {
37517           int M = Mask[i];
37518           if (M == SM_SentinelUndef)
37519             continue;
37520           if (M == SM_SentinelZero) {
37521             IsBlend &= V1Known.Zero[i] && V2Known.Zero[i];
37522             continue;
37523           }
37524           if (M == (int)i) {
37525             IsBlend &= V2Known.Zero[i] || V1Known.One[i];
37526             continue;
37527           }
37528           if (M == (int)(i + NumMaskElts)) {
37529             IsBlend &= V1Known.Zero[i] || V2Known.One[i];
37530             continue;
37531           }
37532           llvm_unreachable("will not get here.");
37533         }
37534         if (IsBlend) {
37535           Shuffle = ISD::OR;
37536           SrcVT = DstVT = MaskVT.changeTypeToInteger();
37537           return true;
37538         }
37539       }
37540     }
37541   }
37542 
37543   return false;
37544 }
37545 
37546 static bool matchBinaryPermuteShuffle(
37547     MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
37548     bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
37549     const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
37550     unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) {
37551   unsigned NumMaskElts = Mask.size();
37552   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
37553 
37554   // Attempt to match against VALIGND/VALIGNQ rotate.
37555   if (AllowIntDomain && (EltSizeInBits == 64 || EltSizeInBits == 32) &&
37556       ((MaskVT.is128BitVector() && Subtarget.hasVLX()) ||
37557        (MaskVT.is256BitVector() && Subtarget.hasVLX()) ||
37558        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37559     if (!isAnyZero(Mask)) {
37560       int Rotation = matchShuffleAsElementRotate(V1, V2, Mask);
37561       if (0 < Rotation) {
37562         Shuffle = X86ISD::VALIGN;
37563         if (EltSizeInBits == 64)
37564           ShuffleVT = MVT::getVectorVT(MVT::i64, MaskVT.getSizeInBits() / 64);
37565         else
37566           ShuffleVT = MVT::getVectorVT(MVT::i32, MaskVT.getSizeInBits() / 32);
37567         PermuteImm = Rotation;
37568         return true;
37569       }
37570     }
37571   }
37572 
37573   // Attempt to match against PALIGNR byte rotate.
37574   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) ||
37575                          (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37576                          (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
37577     int ByteRotation = matchShuffleAsByteRotate(MaskVT, V1, V2, Mask);
37578     if (0 < ByteRotation) {
37579       Shuffle = X86ISD::PALIGNR;
37580       ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);
37581       PermuteImm = ByteRotation;
37582       return true;
37583     }
37584   }
37585 
37586   // Attempt to combine to X86ISD::BLENDI.
37587   if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) ||
37588                             (Subtarget.hasAVX() && MaskVT.is256BitVector()))) ||
37589       (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) {
37590     uint64_t BlendMask = 0;
37591     bool ForceV1Zero = false, ForceV2Zero = false;
37592     SmallVector<int, 8> TargetMask(Mask);
37593     if (matchShuffleAsBlend(MaskVT, V1, V2, TargetMask, Zeroable, ForceV1Zero,
37594                             ForceV2Zero, BlendMask)) {
37595       if (MaskVT == MVT::v16i16) {
37596         // We can only use v16i16 PBLENDW if the lanes are repeated.
37597         SmallVector<int, 8> RepeatedMask;
37598         if (isRepeatedTargetShuffleMask(128, MaskVT, TargetMask,
37599                                         RepeatedMask)) {
37600           assert(RepeatedMask.size() == 8 &&
37601                  "Repeated mask size doesn't match!");
37602           PermuteImm = 0;
37603           for (int i = 0; i < 8; ++i)
37604             if (RepeatedMask[i] >= 8)
37605               PermuteImm |= 1 << i;
37606           V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37607           V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37608           Shuffle = X86ISD::BLENDI;
37609           ShuffleVT = MaskVT;
37610           return true;
37611         }
37612       } else {
37613         V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37614         V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37615         PermuteImm = (unsigned)BlendMask;
37616         Shuffle = X86ISD::BLENDI;
37617         ShuffleVT = MaskVT;
37618         return true;
37619       }
37620     }
37621   }
37622 
37623   // Attempt to combine to INSERTPS, but only if it has elements that need to
37624   // be set to zero.
37625   if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
37626       MaskVT.is128BitVector() && isAnyZero(Mask) &&
37627       matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
37628     Shuffle = X86ISD::INSERTPS;
37629     ShuffleVT = MVT::v4f32;
37630     return true;
37631   }
37632 
37633   // Attempt to combine to SHUFPD.
37634   if (AllowFloatDomain && EltSizeInBits == 64 &&
37635       ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37636        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
37637        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37638     bool ForceV1Zero = false, ForceV2Zero = false;
37639     if (matchShuffleWithSHUFPD(MaskVT, V1, V2, ForceV1Zero, ForceV2Zero,
37640                                PermuteImm, Mask, Zeroable)) {
37641       V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37642       V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37643       Shuffle = X86ISD::SHUFP;
37644       ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
37645       return true;
37646     }
37647   }
37648 
37649   // Attempt to combine to SHUFPS.
37650   if (AllowFloatDomain && EltSizeInBits == 32 &&
37651       ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) ||
37652        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
37653        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37654     SmallVector<int, 4> RepeatedMask;
37655     if (isRepeatedTargetShuffleMask(128, MaskVT, Mask, RepeatedMask)) {
37656       // Match each half of the repeated mask, to determine if its just
37657       // referencing one of the vectors, is zeroable or entirely undef.
37658       auto MatchHalf = [&](unsigned Offset, int &S0, int &S1) {
37659         int M0 = RepeatedMask[Offset];
37660         int M1 = RepeatedMask[Offset + 1];
37661 
37662         if (isUndefInRange(RepeatedMask, Offset, 2)) {
37663           return DAG.getUNDEF(MaskVT);
37664         } else if (isUndefOrZeroInRange(RepeatedMask, Offset, 2)) {
37665           S0 = (SM_SentinelUndef == M0 ? -1 : 0);
37666           S1 = (SM_SentinelUndef == M1 ? -1 : 1);
37667           return getZeroVector(MaskVT, Subtarget, DAG, DL);
37668         } else if (isUndefOrInRange(M0, 0, 4) && isUndefOrInRange(M1, 0, 4)) {
37669           S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
37670           S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
37671           return V1;
37672         } else if (isUndefOrInRange(M0, 4, 8) && isUndefOrInRange(M1, 4, 8)) {
37673           S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
37674           S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
37675           return V2;
37676         }
37677 
37678         return SDValue();
37679       };
37680 
37681       int ShufMask[4] = {-1, -1, -1, -1};
37682       SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
37683       SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
37684 
37685       if (Lo && Hi) {
37686         V1 = Lo;
37687         V2 = Hi;
37688         Shuffle = X86ISD::SHUFP;
37689         ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32);
37690         PermuteImm = getV4X86ShuffleImm(ShufMask);
37691         return true;
37692       }
37693     }
37694   }
37695 
37696   // Attempt to combine to INSERTPS more generally if X86ISD::SHUFP failed.
37697   if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
37698       MaskVT.is128BitVector() &&
37699       matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
37700     Shuffle = X86ISD::INSERTPS;
37701     ShuffleVT = MVT::v4f32;
37702     return true;
37703   }
37704 
37705   return false;
37706 }
37707 
37708 static SDValue combineX86ShuffleChainWithExtract(
37709     ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
37710     bool HasVariableMask, bool AllowVariableCrossLaneMask,
37711     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
37712     const X86Subtarget &Subtarget);
37713 
37714 /// Combine an arbitrary chain of shuffles into a single instruction if
37715 /// possible.
37716 ///
37717 /// This is the leaf of the recursive combine below. When we have found some
37718 /// chain of single-use x86 shuffle instructions and accumulated the combined
37719 /// shuffle mask represented by them, this will try to pattern match that mask
37720 /// into either a single instruction if there is a special purpose instruction
37721 /// for this operation, or into a PSHUFB instruction which is a fully general
37722 /// instruction but should only be used to replace chains over a certain depth.
37723 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
37724                                       ArrayRef<int> BaseMask, int Depth,
37725                                       bool HasVariableMask,
37726                                       bool AllowVariableCrossLaneMask,
37727                                       bool AllowVariablePerLaneMask,
37728                                       SelectionDAG &DAG,
37729                                       const X86Subtarget &Subtarget) {
37730   assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
37731   assert((Inputs.size() == 1 || Inputs.size() == 2) &&
37732          "Unexpected number of shuffle inputs!");
37733 
37734   SDLoc DL(Root);
37735   MVT RootVT = Root.getSimpleValueType();
37736   unsigned RootSizeInBits = RootVT.getSizeInBits();
37737   unsigned NumRootElts = RootVT.getVectorNumElements();
37738 
37739   // Canonicalize shuffle input op to the requested type.
37740   auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) {
37741     if (VT.getSizeInBits() > Op.getValueSizeInBits())
37742       Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits());
37743     else if (VT.getSizeInBits() < Op.getValueSizeInBits())
37744       Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits());
37745     return DAG.getBitcast(VT, Op);
37746   };
37747 
37748   // Find the inputs that enter the chain. Note that multiple uses are OK
37749   // here, we're not going to remove the operands we find.
37750   bool UnaryShuffle = (Inputs.size() == 1);
37751   SDValue V1 = peekThroughBitcasts(Inputs[0]);
37752   SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
37753                              : peekThroughBitcasts(Inputs[1]));
37754 
37755   MVT VT1 = V1.getSimpleValueType();
37756   MVT VT2 = V2.getSimpleValueType();
37757   assert((RootSizeInBits % VT1.getSizeInBits()) == 0 &&
37758          (RootSizeInBits % VT2.getSizeInBits()) == 0 && "Vector size mismatch");
37759 
37760   SDValue Res;
37761 
37762   unsigned NumBaseMaskElts = BaseMask.size();
37763   if (NumBaseMaskElts == 1) {
37764     assert(BaseMask[0] == 0 && "Invalid shuffle index found!");
37765     return CanonicalizeShuffleInput(RootVT, V1);
37766   }
37767 
37768   bool OptForSize = DAG.shouldOptForSize();
37769   unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
37770   bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() ||
37771                      (RootVT.isFloatingPoint() && Depth >= 1) ||
37772                      (RootVT.is256BitVector() && !Subtarget.hasAVX2());
37773 
37774   // Don't combine if we are a AVX512/EVEX target and the mask element size
37775   // is different from the root element size - this would prevent writemasks
37776   // from being reused.
37777   bool IsMaskedShuffle = false;
37778   if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) {
37779     if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT &&
37780         Root->use_begin()->getOperand(0).getScalarValueSizeInBits() == 1) {
37781       IsMaskedShuffle = true;
37782     }
37783   }
37784 
37785   // If we are shuffling a splat (and not introducing zeros) then we can just
37786   // use it directly. This works for smaller elements as well as they already
37787   // repeat across each mask element.
37788   if (UnaryShuffle && !isAnyZero(BaseMask) &&
37789       V1.getValueSizeInBits() >= RootSizeInBits &&
37790       (BaseMaskEltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
37791       DAG.isSplatValue(V1, /*AllowUndefs*/ false)) {
37792     return CanonicalizeShuffleInput(RootVT, V1);
37793   }
37794 
37795   SmallVector<int, 64> Mask(BaseMask);
37796 
37797   // See if the shuffle is a hidden identity shuffle - repeated args in HOPs
37798   // etc. can be simplified.
37799   if (VT1 == VT2 && VT1.getSizeInBits() == RootSizeInBits && VT1.isVector()) {
37800     SmallVector<int> ScaledMask, IdentityMask;
37801     unsigned NumElts = VT1.getVectorNumElements();
37802     if (Mask.size() <= NumElts &&
37803         scaleShuffleElements(Mask, NumElts, ScaledMask)) {
37804       for (unsigned i = 0; i != NumElts; ++i)
37805         IdentityMask.push_back(i);
37806       if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, DAG, V1,
37807                                     V2))
37808         return CanonicalizeShuffleInput(RootVT, V1);
37809     }
37810   }
37811 
37812   // Handle 128/256-bit lane shuffles of 512-bit vectors.
37813   if (RootVT.is512BitVector() &&
37814       (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
37815     // If the upper subvectors are zeroable, then an extract+insert is more
37816     // optimal than using X86ISD::SHUF128. The insertion is free, even if it has
37817     // to zero the upper subvectors.
37818     if (isUndefOrZeroInRange(Mask, 1, NumBaseMaskElts - 1)) {
37819       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37820         return SDValue(); // Nothing to do!
37821       assert(isInRange(Mask[0], 0, NumBaseMaskElts) &&
37822              "Unexpected lane shuffle");
37823       Res = CanonicalizeShuffleInput(RootVT, V1);
37824       unsigned SubIdx = Mask[0] * (NumRootElts / NumBaseMaskElts);
37825       bool UseZero = isAnyZero(Mask);
37826       Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
37827       return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
37828     }
37829 
37830     // Narrow shuffle mask to v4x128.
37831     SmallVector<int, 4> ScaledMask;
37832     assert((BaseMaskEltSizeInBits % 128) == 0 && "Illegal mask size");
37833     narrowShuffleMaskElts(BaseMaskEltSizeInBits / 128, Mask, ScaledMask);
37834 
37835     // Try to lower to vshuf64x2/vshuf32x4.
37836     auto MatchSHUF128 = [&](MVT ShuffleVT, const SDLoc &DL,
37837                             ArrayRef<int> ScaledMask, SDValue V1, SDValue V2,
37838                             SelectionDAG &DAG) {
37839       int PermMask[4] = {-1, -1, -1, -1};
37840       // Ensure elements came from the same Op.
37841       SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
37842       for (int i = 0; i < 4; ++i) {
37843         assert(ScaledMask[i] >= -1 && "Illegal shuffle sentinel value");
37844         if (ScaledMask[i] < 0)
37845           continue;
37846 
37847         SDValue Op = ScaledMask[i] >= 4 ? V2 : V1;
37848         unsigned OpIndex = i / 2;
37849         if (Ops[OpIndex].isUndef())
37850           Ops[OpIndex] = Op;
37851         else if (Ops[OpIndex] != Op)
37852           return SDValue();
37853 
37854         PermMask[i] = ScaledMask[i] % 4;
37855       }
37856 
37857       return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
37858                          CanonicalizeShuffleInput(ShuffleVT, Ops[0]),
37859                          CanonicalizeShuffleInput(ShuffleVT, Ops[1]),
37860                          getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
37861     };
37862 
37863     // FIXME: Is there a better way to do this? is256BitLaneRepeatedShuffleMask
37864     // doesn't work because our mask is for 128 bits and we don't have an MVT
37865     // to match that.
37866     bool PreferPERMQ = UnaryShuffle && isUndefOrInRange(ScaledMask[0], 0, 2) &&
37867                        isUndefOrInRange(ScaledMask[1], 0, 2) &&
37868                        isUndefOrInRange(ScaledMask[2], 2, 4) &&
37869                        isUndefOrInRange(ScaledMask[3], 2, 4) &&
37870                        (ScaledMask[0] < 0 || ScaledMask[2] < 0 ||
37871                         ScaledMask[0] == (ScaledMask[2] % 2)) &&
37872                        (ScaledMask[1] < 0 || ScaledMask[3] < 0 ||
37873                         ScaledMask[1] == (ScaledMask[3] % 2));
37874 
37875     if (!isAnyZero(ScaledMask) && !PreferPERMQ) {
37876       if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
37877         return SDValue(); // Nothing to do!
37878       MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
37879       if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG))
37880         return DAG.getBitcast(RootVT, V);
37881     }
37882   }
37883 
37884   // Handle 128-bit lane shuffles of 256-bit vectors.
37885   if (RootVT.is256BitVector() && NumBaseMaskElts == 2) {
37886     // If the upper half is zeroable, then an extract+insert is more optimal
37887     // than using X86ISD::VPERM2X128. The insertion is free, even if it has to
37888     // zero the upper half.
37889     if (isUndefOrZero(Mask[1])) {
37890       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37891         return SDValue(); // Nothing to do!
37892       assert(isInRange(Mask[0], 0, 2) && "Unexpected lane shuffle");
37893       Res = CanonicalizeShuffleInput(RootVT, V1);
37894       Res = extract128BitVector(Res, Mask[0] * (NumRootElts / 2), DAG, DL);
37895       return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL,
37896                             256);
37897     }
37898 
37899     // If we're inserting the low subvector, an insert-subvector 'concat'
37900     // pattern is quicker than VPERM2X128.
37901     // TODO: Add AVX2 support instead of VPERMQ/VPERMPD.
37902     if (BaseMask[0] == 0 && (BaseMask[1] == 0 || BaseMask[1] == 2) &&
37903         !Subtarget.hasAVX2()) {
37904       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37905         return SDValue(); // Nothing to do!
37906       SDValue Lo = CanonicalizeShuffleInput(RootVT, V1);
37907       SDValue Hi = CanonicalizeShuffleInput(RootVT, BaseMask[1] == 0 ? V1 : V2);
37908       Hi = extractSubVector(Hi, 0, DAG, DL, 128);
37909       return insertSubVector(Lo, Hi, NumRootElts / 2, DAG, DL, 128);
37910     }
37911 
37912     if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
37913       return SDValue(); // Nothing to do!
37914 
37915     // If we have AVX2, prefer to use VPERMQ/VPERMPD for unary shuffles unless
37916     // we need to use the zeroing feature.
37917     // Prefer blends for sequential shuffles unless we are optimizing for size.
37918     if (UnaryShuffle &&
37919         !(Subtarget.hasAVX2() && isUndefOrInRange(Mask, 0, 2)) &&
37920         (OptForSize || !isSequentialOrUndefOrZeroInRange(Mask, 0, 2, 0))) {
37921       unsigned PermMask = 0;
37922       PermMask |= ((Mask[0] < 0 ? 0x8 : (Mask[0] & 1)) << 0);
37923       PermMask |= ((Mask[1] < 0 ? 0x8 : (Mask[1] & 1)) << 4);
37924       return DAG.getNode(
37925           X86ISD::VPERM2X128, DL, RootVT, CanonicalizeShuffleInput(RootVT, V1),
37926           DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));
37927     }
37928 
37929     if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
37930       return SDValue(); // Nothing to do!
37931 
37932     // TODO - handle AVX512VL cases with X86ISD::SHUF128.
37933     if (!UnaryShuffle && !IsMaskedShuffle) {
37934       assert(llvm::all_of(Mask, [](int M) { return 0 <= M && M < 4; }) &&
37935              "Unexpected shuffle sentinel value");
37936       // Prefer blends to X86ISD::VPERM2X128.
37937       if (!((Mask[0] == 0 && Mask[1] == 3) || (Mask[0] == 2 && Mask[1] == 1))) {
37938         unsigned PermMask = 0;
37939         PermMask |= ((Mask[0] & 3) << 0);
37940         PermMask |= ((Mask[1] & 3) << 4);
37941         SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
37942         SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
37943         return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
37944                           CanonicalizeShuffleInput(RootVT, LHS),
37945                           CanonicalizeShuffleInput(RootVT, RHS),
37946                           DAG.getTargetConstant(PermMask, DL, MVT::i8));
37947       }
37948     }
37949   }
37950 
37951   // For masks that have been widened to 128-bit elements or more,
37952   // narrow back down to 64-bit elements.
37953   if (BaseMaskEltSizeInBits > 64) {
37954     assert((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size");
37955     int MaskScale = BaseMaskEltSizeInBits / 64;
37956     SmallVector<int, 64> ScaledMask;
37957     narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
37958     Mask = std::move(ScaledMask);
37959   }
37960 
37961   // For masked shuffles, we're trying to match the root width for better
37962   // writemask folding, attempt to scale the mask.
37963   // TODO - variable shuffles might need this to be widened again.
37964   if (IsMaskedShuffle && NumRootElts > Mask.size()) {
37965     assert((NumRootElts % Mask.size()) == 0 && "Illegal mask size");
37966     int MaskScale = NumRootElts / Mask.size();
37967     SmallVector<int, 64> ScaledMask;
37968     narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
37969     Mask = std::move(ScaledMask);
37970   }
37971 
37972   unsigned NumMaskElts = Mask.size();
37973   unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts;
37974 
37975   // Determine the effective mask value type.
37976   FloatDomain &= (32 <= MaskEltSizeInBits);
37977   MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits)
37978                            : MVT::getIntegerVT(MaskEltSizeInBits);
37979   MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts);
37980 
37981   // Only allow legal mask types.
37982   if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
37983     return SDValue();
37984 
37985   // Attempt to match the mask against known shuffle patterns.
37986   MVT ShuffleSrcVT, ShuffleVT;
37987   unsigned Shuffle, PermuteImm;
37988 
37989   // Which shuffle domains are permitted?
37990   // Permit domain crossing at higher combine depths.
37991   // TODO: Should we indicate which domain is preferred if both are allowed?
37992   bool AllowFloatDomain = FloatDomain || (Depth >= 3);
37993   bool AllowIntDomain = (!FloatDomain || (Depth >= 3)) && Subtarget.hasSSE2() &&
37994                         (!MaskVT.is256BitVector() || Subtarget.hasAVX2());
37995 
37996   // Determine zeroable mask elements.
37997   APInt KnownUndef, KnownZero;
37998   resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
37999   APInt Zeroable = KnownUndef | KnownZero;
38000 
38001   if (UnaryShuffle) {
38002     // Attempt to match against broadcast-from-vector.
38003     // Limit AVX1 to cases where we're loading+broadcasting a scalar element.
38004     if ((Subtarget.hasAVX2() ||
38005          (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits)) &&
38006         (!IsMaskedShuffle || NumRootElts == NumMaskElts)) {
38007       if (isUndefOrEqual(Mask, 0)) {
38008         if (V1.getValueType() == MaskVT &&
38009             V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38010             X86::mayFoldLoad(V1.getOperand(0), Subtarget)) {
38011           if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38012             return SDValue(); // Nothing to do!
38013           Res = V1.getOperand(0);
38014           Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38015           return DAG.getBitcast(RootVT, Res);
38016         }
38017         if (Subtarget.hasAVX2()) {
38018           if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38019             return SDValue(); // Nothing to do!
38020           Res = CanonicalizeShuffleInput(MaskVT, V1);
38021           Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38022           return DAG.getBitcast(RootVT, Res);
38023         }
38024       }
38025     }
38026 
38027     if (matchUnaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, V1,
38028                           DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) &&
38029         (!IsMaskedShuffle ||
38030          (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38031       if (Depth == 0 && Root.getOpcode() == Shuffle)
38032         return SDValue(); // Nothing to do!
38033       Res = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38034       Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
38035       return DAG.getBitcast(RootVT, Res);
38036     }
38037 
38038     if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38039                                  AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
38040                                  PermuteImm) &&
38041         (!IsMaskedShuffle ||
38042          (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38043       if (Depth == 0 && Root.getOpcode() == Shuffle)
38044         return SDValue(); // Nothing to do!
38045       Res = CanonicalizeShuffleInput(ShuffleVT, V1);
38046       Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
38047                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38048       return DAG.getBitcast(RootVT, Res);
38049     }
38050   }
38051 
38052   // Attempt to combine to INSERTPS, but only if the inserted element has come
38053   // from a scalar.
38054   // TODO: Handle other insertions here as well?
38055   if (!UnaryShuffle && AllowFloatDomain && RootSizeInBits == 128 &&
38056       Subtarget.hasSSE41() &&
38057       !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG)) {
38058     if (MaskEltSizeInBits == 32) {
38059       SDValue SrcV1 = V1, SrcV2 = V2;
38060       if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask,
38061                                  DAG) &&
38062           SrcV2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
38063         if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38064           return SDValue(); // Nothing to do!
38065         Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38066                           CanonicalizeShuffleInput(MVT::v4f32, SrcV1),
38067                           CanonicalizeShuffleInput(MVT::v4f32, SrcV2),
38068                           DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38069         return DAG.getBitcast(RootVT, Res);
38070       }
38071     }
38072     if (MaskEltSizeInBits == 64 &&
38073         isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}, DAG) &&
38074         V2.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38075         V2.getScalarValueSizeInBits() <= 32) {
38076       if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38077         return SDValue(); // Nothing to do!
38078       PermuteImm = (/*DstIdx*/ 2 << 4) | (/*SrcIdx*/ 0 << 0);
38079       Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38080                         CanonicalizeShuffleInput(MVT::v4f32, V1),
38081                         CanonicalizeShuffleInput(MVT::v4f32, V2),
38082                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38083       return DAG.getBitcast(RootVT, Res);
38084     }
38085   }
38086 
38087   SDValue NewV1 = V1; // Save operands in case early exit happens.
38088   SDValue NewV2 = V2;
38089   if (matchBinaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
38090                          NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
38091                          ShuffleVT, UnaryShuffle) &&
38092       (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38093     if (Depth == 0 && Root.getOpcode() == Shuffle)
38094       return SDValue(); // Nothing to do!
38095     NewV1 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV1);
38096     NewV2 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV2);
38097     Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
38098     return DAG.getBitcast(RootVT, Res);
38099   }
38100 
38101   NewV1 = V1; // Save operands in case early exit happens.
38102   NewV2 = V2;
38103   if (matchBinaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38104                                 AllowIntDomain, NewV1, NewV2, DL, DAG,
38105                                 Subtarget, Shuffle, ShuffleVT, PermuteImm) &&
38106       (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38107     if (Depth == 0 && Root.getOpcode() == Shuffle)
38108       return SDValue(); // Nothing to do!
38109     NewV1 = CanonicalizeShuffleInput(ShuffleVT, NewV1);
38110     NewV2 = CanonicalizeShuffleInput(ShuffleVT, NewV2);
38111     Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
38112                       DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38113     return DAG.getBitcast(RootVT, Res);
38114   }
38115 
38116   // Typically from here on, we need an integer version of MaskVT.
38117   MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits);
38118   IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts);
38119 
38120   // Annoyingly, SSE4A instructions don't map into the above match helpers.
38121   if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) {
38122     uint64_t BitLen, BitIdx;
38123     if (matchShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
38124                             Zeroable)) {
38125       if (Depth == 0 && Root.getOpcode() == X86ISD::EXTRQI)
38126         return SDValue(); // Nothing to do!
38127       V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
38128       Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
38129                         DAG.getTargetConstant(BitLen, DL, MVT::i8),
38130                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));
38131       return DAG.getBitcast(RootVT, Res);
38132     }
38133 
38134     if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
38135       if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTQI)
38136         return SDValue(); // Nothing to do!
38137       V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
38138       V2 = CanonicalizeShuffleInput(IntMaskVT, V2);
38139       Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
38140                         DAG.getTargetConstant(BitLen, DL, MVT::i8),
38141                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));
38142       return DAG.getBitcast(RootVT, Res);
38143     }
38144   }
38145 
38146   // Match shuffle against TRUNCATE patterns.
38147   if (AllowIntDomain && MaskEltSizeInBits < 64 && Subtarget.hasAVX512()) {
38148     // Match against a VTRUNC instruction, accounting for src/dst sizes.
38149     if (matchShuffleAsVTRUNC(ShuffleSrcVT, ShuffleVT, IntMaskVT, Mask, Zeroable,
38150                              Subtarget)) {
38151       bool IsTRUNCATE = ShuffleVT.getVectorNumElements() ==
38152                         ShuffleSrcVT.getVectorNumElements();
38153       unsigned Opc =
38154           IsTRUNCATE ? (unsigned)ISD::TRUNCATE : (unsigned)X86ISD::VTRUNC;
38155       if (Depth == 0 && Root.getOpcode() == Opc)
38156         return SDValue(); // Nothing to do!
38157       V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38158       Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
38159       if (ShuffleVT.getSizeInBits() < RootSizeInBits)
38160         Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
38161       return DAG.getBitcast(RootVT, Res);
38162     }
38163 
38164     // Do we need a more general binary truncation pattern?
38165     if (RootSizeInBits < 512 &&
38166         ((RootVT.is256BitVector() && Subtarget.useAVX512Regs()) ||
38167          (RootVT.is128BitVector() && Subtarget.hasVLX())) &&
38168         (MaskEltSizeInBits > 8 || Subtarget.hasBWI()) &&
38169         isSequentialOrUndefInRange(Mask, 0, NumMaskElts, 0, 2)) {
38170       // Bail if this was already a truncation or PACK node.
38171       // We sometimes fail to match PACK if we demand known undef elements.
38172       if (Depth == 0 && (Root.getOpcode() == ISD::TRUNCATE ||
38173                          Root.getOpcode() == X86ISD::PACKSS ||
38174                          Root.getOpcode() == X86ISD::PACKUS))
38175         return SDValue(); // Nothing to do!
38176       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
38177       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2);
38178       V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38179       V2 = CanonicalizeShuffleInput(ShuffleSrcVT, V2);
38180       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
38181       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts);
38182       Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
38183       Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
38184       return DAG.getBitcast(RootVT, Res);
38185     }
38186   }
38187 
38188   // Don't try to re-form single instruction chains under any circumstances now
38189   // that we've done encoding canonicalization for them.
38190   if (Depth < 1)
38191     return SDValue();
38192 
38193   // Depth threshold above which we can efficiently use variable mask shuffles.
38194   int VariableCrossLaneShuffleDepth =
38195       Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2;
38196   int VariablePerLaneShuffleDepth =
38197       Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2;
38198   AllowVariableCrossLaneMask &=
38199       (Depth >= VariableCrossLaneShuffleDepth) || HasVariableMask;
38200   AllowVariablePerLaneMask &=
38201       (Depth >= VariablePerLaneShuffleDepth) || HasVariableMask;
38202   // VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a
38203   // higher depth before combining them.
38204   bool AllowBWIVPERMV3 =
38205       (Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
38206 
38207   bool MaskContainsZeros = isAnyZero(Mask);
38208 
38209   if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
38210     // If we have a single input lane-crossing shuffle then lower to VPERMV.
38211     if (UnaryShuffle && AllowVariableCrossLaneMask && !MaskContainsZeros) {
38212       if (Subtarget.hasAVX2() &&
38213           (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) {
38214         SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
38215         Res = CanonicalizeShuffleInput(MaskVT, V1);
38216         Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
38217         return DAG.getBitcast(RootVT, Res);
38218       }
38219       // AVX512 variants (non-VLX will pad to 512-bit shuffles).
38220       if ((Subtarget.hasAVX512() &&
38221            (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38222             MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
38223           (Subtarget.hasBWI() &&
38224            (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38225           (Subtarget.hasVBMI() &&
38226            (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8))) {
38227         V1 = CanonicalizeShuffleInput(MaskVT, V1);
38228         V2 = DAG.getUNDEF(MaskVT);
38229         Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38230         return DAG.getBitcast(RootVT, Res);
38231       }
38232     }
38233 
38234     // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero
38235     // vector as the second source (non-VLX will pad to 512-bit shuffles).
38236     if (UnaryShuffle && AllowVariableCrossLaneMask &&
38237         ((Subtarget.hasAVX512() &&
38238           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38239            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
38240            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32 ||
38241            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
38242          (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38243           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38244          (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38245           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
38246       // Adjust shuffle mask - replace SM_SentinelZero with second source index.
38247       for (unsigned i = 0; i != NumMaskElts; ++i)
38248         if (Mask[i] == SM_SentinelZero)
38249           Mask[i] = NumMaskElts + i;
38250       V1 = CanonicalizeShuffleInput(MaskVT, V1);
38251       V2 = getZeroVector(MaskVT, Subtarget, DAG, DL);
38252       Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38253       return DAG.getBitcast(RootVT, Res);
38254     }
38255 
38256     // If that failed and either input is extracted then try to combine as a
38257     // shuffle with the larger type.
38258     if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
38259             Inputs, Root, BaseMask, Depth, HasVariableMask,
38260             AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG,
38261             Subtarget))
38262       return WideShuffle;
38263 
38264     // If we have a dual input lane-crossing shuffle then lower to VPERMV3,
38265     // (non-VLX will pad to 512-bit shuffles).
38266     if (AllowVariableCrossLaneMask && !MaskContainsZeros &&
38267         ((Subtarget.hasAVX512() &&
38268           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38269            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
38270            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32 ||
38271            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
38272          (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38273           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38274          (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38275           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
38276       V1 = CanonicalizeShuffleInput(MaskVT, V1);
38277       V2 = CanonicalizeShuffleInput(MaskVT, V2);
38278       Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38279       return DAG.getBitcast(RootVT, Res);
38280     }
38281     return SDValue();
38282   }
38283 
38284   // See if we can combine a single input shuffle with zeros to a bit-mask,
38285   // which is much simpler than any shuffle.
38286   if (UnaryShuffle && MaskContainsZeros && AllowVariablePerLaneMask &&
38287       isSequentialOrUndefOrZeroInRange(Mask, 0, NumMaskElts, 0) &&
38288       DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) {
38289     APInt Zero = APInt::getZero(MaskEltSizeInBits);
38290     APInt AllOnes = APInt::getAllOnes(MaskEltSizeInBits);
38291     APInt UndefElts(NumMaskElts, 0);
38292     SmallVector<APInt, 64> EltBits(NumMaskElts, Zero);
38293     for (unsigned i = 0; i != NumMaskElts; ++i) {
38294       int M = Mask[i];
38295       if (M == SM_SentinelUndef) {
38296         UndefElts.setBit(i);
38297         continue;
38298       }
38299       if (M == SM_SentinelZero)
38300         continue;
38301       EltBits[i] = AllOnes;
38302     }
38303     SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
38304     Res = CanonicalizeShuffleInput(MaskVT, V1);
38305     unsigned AndOpcode =
38306         MaskVT.isFloatingPoint() ? unsigned(X86ISD::FAND) : unsigned(ISD::AND);
38307     Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
38308     return DAG.getBitcast(RootVT, Res);
38309   }
38310 
38311   // If we have a single input shuffle with different shuffle patterns in the
38312   // the 128-bit lanes use the variable mask to VPERMILPS.
38313   // TODO Combine other mask types at higher depths.
38314   if (UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
38315       ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) ||
38316        (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) {
38317     SmallVector<SDValue, 16> VPermIdx;
38318     for (int M : Mask) {
38319       SDValue Idx =
38320           M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
38321       VPermIdx.push_back(Idx);
38322     }
38323     SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
38324     Res = CanonicalizeShuffleInput(MaskVT, V1);
38325     Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
38326     return DAG.getBitcast(RootVT, Res);
38327   }
38328 
38329   // With XOP, binary shuffles of 128/256-bit floating point vectors can combine
38330   // to VPERMIL2PD/VPERMIL2PS.
38331   if (AllowVariablePerLaneMask && Subtarget.hasXOP() &&
38332       (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 ||
38333        MaskVT == MVT::v8f32)) {
38334     // VPERMIL2 Operation.
38335     // Bits[3] - Match Bit.
38336     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
38337     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
38338     unsigned NumLanes = MaskVT.getSizeInBits() / 128;
38339     unsigned NumEltsPerLane = NumMaskElts / NumLanes;
38340     SmallVector<int, 8> VPerm2Idx;
38341     unsigned M2ZImm = 0;
38342     for (int M : Mask) {
38343       if (M == SM_SentinelUndef) {
38344         VPerm2Idx.push_back(-1);
38345         continue;
38346       }
38347       if (M == SM_SentinelZero) {
38348         M2ZImm = 2;
38349         VPerm2Idx.push_back(8);
38350         continue;
38351       }
38352       int Index = (M % NumEltsPerLane) + ((M / NumMaskElts) * NumEltsPerLane);
38353       Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index);
38354       VPerm2Idx.push_back(Index);
38355     }
38356     V1 = CanonicalizeShuffleInput(MaskVT, V1);
38357     V2 = CanonicalizeShuffleInput(MaskVT, V2);
38358     SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
38359     Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
38360                       DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
38361     return DAG.getBitcast(RootVT, Res);
38362   }
38363 
38364   // If we have 3 or more shuffle instructions or a chain involving a variable
38365   // mask, we can replace them with a single PSHUFB instruction profitably.
38366   // Intel's manuals suggest only using PSHUFB if doing so replacing 5
38367   // instructions, but in practice PSHUFB tends to be *very* fast so we're
38368   // more aggressive.
38369   if (UnaryShuffle && AllowVariablePerLaneMask &&
38370       ((RootVT.is128BitVector() && Subtarget.hasSSSE3()) ||
38371        (RootVT.is256BitVector() && Subtarget.hasAVX2()) ||
38372        (RootVT.is512BitVector() && Subtarget.hasBWI()))) {
38373     SmallVector<SDValue, 16> PSHUFBMask;
38374     int NumBytes = RootVT.getSizeInBits() / 8;
38375     int Ratio = NumBytes / NumMaskElts;
38376     for (int i = 0; i < NumBytes; ++i) {
38377       int M = Mask[i / Ratio];
38378       if (M == SM_SentinelUndef) {
38379         PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
38380         continue;
38381       }
38382       if (M == SM_SentinelZero) {
38383         PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
38384         continue;
38385       }
38386       M = Ratio * M + i % Ratio;
38387       assert((M / 16) == (i / 16) && "Lane crossing detected");
38388       PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
38389     }
38390     MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);
38391     Res = CanonicalizeShuffleInput(ByteVT, V1);
38392     SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
38393     Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
38394     return DAG.getBitcast(RootVT, Res);
38395   }
38396 
38397   // With XOP, if we have a 128-bit binary input shuffle we can always combine
38398   // to VPPERM. We match the depth requirement of PSHUFB - VPPERM is never
38399   // slower than PSHUFB on targets that support both.
38400   if (AllowVariablePerLaneMask && RootVT.is128BitVector() &&
38401       Subtarget.hasXOP()) {
38402     // VPPERM Mask Operation
38403     // Bits[4:0] - Byte Index (0 - 31)
38404     // Bits[7:5] - Permute Operation (0 - Source byte, 4 - ZERO)
38405     SmallVector<SDValue, 16> VPPERMMask;
38406     int NumBytes = 16;
38407     int Ratio = NumBytes / NumMaskElts;
38408     for (int i = 0; i < NumBytes; ++i) {
38409       int M = Mask[i / Ratio];
38410       if (M == SM_SentinelUndef) {
38411         VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));
38412         continue;
38413       }
38414       if (M == SM_SentinelZero) {
38415         VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
38416         continue;
38417       }
38418       M = Ratio * M + i % Ratio;
38419       VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
38420     }
38421     MVT ByteVT = MVT::v16i8;
38422     V1 = CanonicalizeShuffleInput(ByteVT, V1);
38423     V2 = CanonicalizeShuffleInput(ByteVT, V2);
38424     SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
38425     Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
38426     return DAG.getBitcast(RootVT, Res);
38427   }
38428 
38429   // If that failed and either input is extracted then try to combine as a
38430   // shuffle with the larger type.
38431   if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
38432           Inputs, Root, BaseMask, Depth, HasVariableMask,
38433           AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget))
38434     return WideShuffle;
38435 
38436   // If we have a dual input shuffle then lower to VPERMV3,
38437   // (non-VLX will pad to 512-bit shuffles)
38438   if (!UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
38439       ((Subtarget.hasAVX512() &&
38440         (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v8f64 ||
38441          MaskVT == MVT::v2i64 || MaskVT == MVT::v4i64 || MaskVT == MVT::v8i64 ||
38442          MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || MaskVT == MVT::v8f32 ||
38443          MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 ||
38444          MaskVT == MVT::v16i32)) ||
38445        (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38446         (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16 ||
38447          MaskVT == MVT::v32i16)) ||
38448        (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38449         (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8 ||
38450          MaskVT == MVT::v64i8)))) {
38451     V1 = CanonicalizeShuffleInput(MaskVT, V1);
38452     V2 = CanonicalizeShuffleInput(MaskVT, V2);
38453     Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38454     return DAG.getBitcast(RootVT, Res);
38455   }
38456 
38457   // Failed to find any combines.
38458   return SDValue();
38459 }
38460 
38461 // Combine an arbitrary chain of shuffles + extract_subvectors into a single
38462 // instruction if possible.
38463 //
38464 // Wrapper for combineX86ShuffleChain that extends the shuffle mask to a larger
38465 // type size to attempt to combine:
38466 // shuffle(extract_subvector(x,c1),extract_subvector(y,c2),m1)
38467 // -->
38468 // extract_subvector(shuffle(x,y,m2),0)
38469 static SDValue combineX86ShuffleChainWithExtract(
38470     ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
38471     bool HasVariableMask, bool AllowVariableCrossLaneMask,
38472     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38473     const X86Subtarget &Subtarget) {
38474   unsigned NumMaskElts = BaseMask.size();
38475   unsigned NumInputs = Inputs.size();
38476   if (NumInputs == 0)
38477     return SDValue();
38478 
38479   EVT RootVT = Root.getValueType();
38480   unsigned RootSizeInBits = RootVT.getSizeInBits();
38481   unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
38482   assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
38483 
38484   // Peek through extract_subvector to find widest legal vector.
38485   // TODO: Handle ISD::TRUNCATE
38486   unsigned WideSizeInBits = RootSizeInBits;
38487   for (unsigned I = 0; I != NumInputs; ++I) {
38488     SDValue Input = peekThroughBitcasts(Inputs[I]);
38489     while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR)
38490       Input = peekThroughBitcasts(Input.getOperand(0));
38491     if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) &&
38492         WideSizeInBits < Input.getValueSizeInBits())
38493       WideSizeInBits = Input.getValueSizeInBits();
38494   }
38495 
38496   // Bail if we fail to find a source larger than the existing root.
38497   unsigned Scale = WideSizeInBits / RootSizeInBits;
38498   if (WideSizeInBits <= RootSizeInBits ||
38499       (WideSizeInBits % RootSizeInBits) != 0)
38500     return SDValue();
38501 
38502   // Create new mask for larger type.
38503   SmallVector<int, 64> WideMask(BaseMask);
38504   for (int &M : WideMask) {
38505     if (M < 0)
38506       continue;
38507     M = (M % NumMaskElts) + ((M / NumMaskElts) * Scale * NumMaskElts);
38508   }
38509   WideMask.append((Scale - 1) * NumMaskElts, SM_SentinelUndef);
38510 
38511   // Attempt to peek through inputs and adjust mask when we extract from an
38512   // upper subvector.
38513   int AdjustedMasks = 0;
38514   SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
38515   for (unsigned I = 0; I != NumInputs; ++I) {
38516     SDValue &Input = WideInputs[I];
38517     Input = peekThroughBitcasts(Input);
38518     while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
38519            Input.getOperand(0).getValueSizeInBits() <= WideSizeInBits) {
38520       uint64_t Idx = Input.getConstantOperandVal(1);
38521       if (Idx != 0) {
38522         ++AdjustedMasks;
38523         unsigned InputEltSizeInBits = Input.getScalarValueSizeInBits();
38524         Idx = (Idx * InputEltSizeInBits) / RootEltSizeInBits;
38525 
38526         int lo = I * WideMask.size();
38527         int hi = (I + 1) * WideMask.size();
38528         for (int &M : WideMask)
38529           if (lo <= M && M < hi)
38530             M += Idx;
38531       }
38532       Input = peekThroughBitcasts(Input.getOperand(0));
38533     }
38534   }
38535 
38536   // Remove unused/repeated shuffle source ops.
38537   resolveTargetShuffleInputsAndMask(WideInputs, WideMask);
38538   assert(!WideInputs.empty() && "Shuffle with no inputs detected");
38539 
38540   // Bail if we're always extracting from the lowest subvectors,
38541   // combineX86ShuffleChain should match this for the current width, or the
38542   // shuffle still references too many inputs.
38543   if (AdjustedMasks == 0 || WideInputs.size() > 2)
38544     return SDValue();
38545 
38546   // Minor canonicalization of the accumulated shuffle mask to make it easier
38547   // to match below. All this does is detect masks with sequential pairs of
38548   // elements, and shrink them to the half-width mask. It does this in a loop
38549   // so it will reduce the size of the mask to the minimal width mask which
38550   // performs an equivalent shuffle.
38551   while (WideMask.size() > 1) {
38552     SmallVector<int, 64> WidenedMask;
38553     if (!canWidenShuffleElements(WideMask, WidenedMask))
38554       break;
38555     WideMask = std::move(WidenedMask);
38556   }
38557 
38558   // Canonicalization of binary shuffle masks to improve pattern matching by
38559   // commuting the inputs.
38560   if (WideInputs.size() == 2 && canonicalizeShuffleMaskWithCommute(WideMask)) {
38561     ShuffleVectorSDNode::commuteMask(WideMask);
38562     std::swap(WideInputs[0], WideInputs[1]);
38563   }
38564 
38565   // Increase depth for every upper subvector we've peeked through.
38566   Depth += AdjustedMasks;
38567 
38568   // Attempt to combine wider chain.
38569   // TODO: Can we use a better Root?
38570   SDValue WideRoot = WideInputs.front().getValueSizeInBits() >
38571                              WideInputs.back().getValueSizeInBits()
38572                          ? WideInputs.front()
38573                          : WideInputs.back();
38574   assert(WideRoot.getValueSizeInBits() == WideSizeInBits &&
38575          "WideRootSize mismatch");
38576 
38577   if (SDValue WideShuffle =
38578           combineX86ShuffleChain(WideInputs, WideRoot, WideMask, Depth,
38579                                  HasVariableMask, AllowVariableCrossLaneMask,
38580                                  AllowVariablePerLaneMask, DAG, Subtarget)) {
38581     WideShuffle =
38582         extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
38583     return DAG.getBitcast(RootVT, WideShuffle);
38584   }
38585 
38586   return SDValue();
38587 }
38588 
38589 // Canonicalize the combined shuffle mask chain with horizontal ops.
38590 // NOTE: This may update the Ops and Mask.
38591 static SDValue canonicalizeShuffleMaskWithHorizOp(
38592     MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
38593     unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
38594     const X86Subtarget &Subtarget) {
38595   if (Mask.empty() || Ops.empty())
38596     return SDValue();
38597 
38598   SmallVector<SDValue> BC;
38599   for (SDValue Op : Ops)
38600     BC.push_back(peekThroughBitcasts(Op));
38601 
38602   // All ops must be the same horizop + type.
38603   SDValue BC0 = BC[0];
38604   EVT VT0 = BC0.getValueType();
38605   unsigned Opcode0 = BC0.getOpcode();
38606   if (VT0.getSizeInBits() != RootSizeInBits || llvm::any_of(BC, [&](SDValue V) {
38607         return V.getOpcode() != Opcode0 || V.getValueType() != VT0;
38608       }))
38609     return SDValue();
38610 
38611   bool isHoriz = (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD ||
38612                   Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB);
38613   bool isPack = (Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS);
38614   if (!isHoriz && !isPack)
38615     return SDValue();
38616 
38617   // Do all ops have a single use?
38618   bool OneUseOps = llvm::all_of(Ops, [](SDValue Op) {
38619     return Op.hasOneUse() &&
38620            peekThroughBitcasts(Op) == peekThroughOneUseBitcasts(Op);
38621   });
38622 
38623   int NumElts = VT0.getVectorNumElements();
38624   int NumLanes = VT0.getSizeInBits() / 128;
38625   int NumEltsPerLane = NumElts / NumLanes;
38626   int NumHalfEltsPerLane = NumEltsPerLane / 2;
38627   MVT SrcVT = BC0.getOperand(0).getSimpleValueType();
38628   unsigned EltSizeInBits = RootSizeInBits / Mask.size();
38629 
38630   if (NumEltsPerLane >= 4 &&
38631       (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) {
38632     SmallVector<int> LaneMask, ScaledMask;
38633     if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, LaneMask) &&
38634         scaleShuffleElements(LaneMask, 4, ScaledMask)) {
38635       // See if we can remove the shuffle by resorting the HOP chain so that
38636       // the HOP args are pre-shuffled.
38637       // TODO: Generalize to any sized/depth chain.
38638       // TODO: Add support for PACKSS/PACKUS.
38639       if (isHoriz) {
38640         // Attempt to find a HOP(HOP(X,Y),HOP(Z,W)) source operand.
38641         auto GetHOpSrc = [&](int M) {
38642           if (M == SM_SentinelUndef)
38643             return DAG.getUNDEF(VT0);
38644           if (M == SM_SentinelZero)
38645             return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL);
38646           SDValue Src0 = BC[M / 4];
38647           SDValue Src1 = Src0.getOperand((M % 4) >= 2);
38648           if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode()))
38649             return Src1.getOperand(M % 2);
38650           return SDValue();
38651         };
38652         SDValue M0 = GetHOpSrc(ScaledMask[0]);
38653         SDValue M1 = GetHOpSrc(ScaledMask[1]);
38654         SDValue M2 = GetHOpSrc(ScaledMask[2]);
38655         SDValue M3 = GetHOpSrc(ScaledMask[3]);
38656         if (M0 && M1 && M2 && M3) {
38657           SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1);
38658           SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3);
38659           return DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
38660         }
38661       }
38662       // shuffle(hop(x,y),hop(z,w)) -> permute(hop(x,z)) etc.
38663       if (Ops.size() >= 2) {
38664         SDValue LHS, RHS;
38665         auto GetHOpSrc = [&](int M, int &OutM) {
38666           // TODO: Support SM_SentinelZero
38667           if (M < 0)
38668             return M == SM_SentinelUndef;
38669           SDValue Src = BC[M / 4].getOperand((M % 4) >= 2);
38670           if (!LHS || LHS == Src) {
38671             LHS = Src;
38672             OutM = (M % 2);
38673             return true;
38674           }
38675           if (!RHS || RHS == Src) {
38676             RHS = Src;
38677             OutM = (M % 2) + 2;
38678             return true;
38679           }
38680           return false;
38681         };
38682         int PostMask[4] = {-1, -1, -1, -1};
38683         if (GetHOpSrc(ScaledMask[0], PostMask[0]) &&
38684             GetHOpSrc(ScaledMask[1], PostMask[1]) &&
38685             GetHOpSrc(ScaledMask[2], PostMask[2]) &&
38686             GetHOpSrc(ScaledMask[3], PostMask[3])) {
38687           LHS = DAG.getBitcast(SrcVT, LHS);
38688           RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
38689           SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
38690           // Use SHUFPS for the permute so this will work on SSE2 targets,
38691           // shuffle combining and domain handling will simplify this later on.
38692           MVT ShuffleVT = MVT::getVectorVT(MVT::f32, RootSizeInBits / 32);
38693           Res = DAG.getBitcast(ShuffleVT, Res);
38694           return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res,
38695                              getV4X86ShuffleImm8ForMask(PostMask, DL, DAG));
38696         }
38697       }
38698     }
38699   }
38700 
38701   if (2 < Ops.size())
38702     return SDValue();
38703 
38704   SDValue BC1 = BC[BC.size() - 1];
38705   if (Mask.size() == VT0.getVectorNumElements()) {
38706     // Canonicalize binary shuffles of horizontal ops that use the
38707     // same sources to an unary shuffle.
38708     // TODO: Try to perform this fold even if the shuffle remains.
38709     if (Ops.size() == 2) {
38710       auto ContainsOps = [](SDValue HOp, SDValue Op) {
38711         return Op == HOp.getOperand(0) || Op == HOp.getOperand(1);
38712       };
38713       // Commute if all BC0's ops are contained in BC1.
38714       if (ContainsOps(BC1, BC0.getOperand(0)) &&
38715           ContainsOps(BC1, BC0.getOperand(1))) {
38716         ShuffleVectorSDNode::commuteMask(Mask);
38717         std::swap(Ops[0], Ops[1]);
38718         std::swap(BC0, BC1);
38719       }
38720 
38721       // If BC1 can be represented by BC0, then convert to unary shuffle.
38722       if (ContainsOps(BC0, BC1.getOperand(0)) &&
38723           ContainsOps(BC0, BC1.getOperand(1))) {
38724         for (int &M : Mask) {
38725           if (M < NumElts) // BC0 element or UNDEF/Zero sentinel.
38726             continue;
38727           int SubLane = ((M % NumEltsPerLane) >= NumHalfEltsPerLane) ? 1 : 0;
38728           M -= NumElts + (SubLane * NumHalfEltsPerLane);
38729           if (BC1.getOperand(SubLane) != BC0.getOperand(0))
38730             M += NumHalfEltsPerLane;
38731         }
38732       }
38733     }
38734 
38735     // Canonicalize unary horizontal ops to only refer to lower halves.
38736     for (int i = 0; i != NumElts; ++i) {
38737       int &M = Mask[i];
38738       if (isUndefOrZero(M))
38739         continue;
38740       if (M < NumElts && BC0.getOperand(0) == BC0.getOperand(1) &&
38741           (M % NumEltsPerLane) >= NumHalfEltsPerLane)
38742         M -= NumHalfEltsPerLane;
38743       if (NumElts <= M && BC1.getOperand(0) == BC1.getOperand(1) &&
38744           (M % NumEltsPerLane) >= NumHalfEltsPerLane)
38745         M -= NumHalfEltsPerLane;
38746     }
38747   }
38748 
38749   // Combine binary shuffle of 2 similar 'Horizontal' instructions into a
38750   // single instruction. Attempt to match a v2X64 repeating shuffle pattern that
38751   // represents the LHS/RHS inputs for the lower/upper halves.
38752   SmallVector<int, 16> TargetMask128, WideMask128;
38753   if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, TargetMask128) &&
38754       scaleShuffleElements(TargetMask128, 2, WideMask128)) {
38755     assert(isUndefOrZeroOrInRange(WideMask128, 0, 4) && "Illegal shuffle");
38756     bool SingleOp = (Ops.size() == 1);
38757     if (isPack || OneUseOps ||
38758         shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
38759       SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1;
38760       SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1;
38761       Lo = Lo.getOperand(WideMask128[0] & 1);
38762       Hi = Hi.getOperand(WideMask128[1] & 1);
38763       if (SingleOp) {
38764         SDValue Undef = DAG.getUNDEF(SrcVT);
38765         SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
38766         Lo = (WideMask128[0] == SM_SentinelZero ? Zero : Lo);
38767         Hi = (WideMask128[1] == SM_SentinelZero ? Zero : Hi);
38768         Lo = (WideMask128[0] == SM_SentinelUndef ? Undef : Lo);
38769         Hi = (WideMask128[1] == SM_SentinelUndef ? Undef : Hi);
38770       }
38771       return DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
38772     }
38773   }
38774 
38775   // If we are post-shuffling a 256-bit hop and not requiring the upper
38776   // elements, then try to narrow to a 128-bit hop directly.
38777   SmallVector<int, 16> WideMask64;
38778   if (Ops.size() == 1 && NumLanes == 2 &&
38779       scaleShuffleElements(Mask, 4, WideMask64) &&
38780       isUndefInRange(WideMask64, 2, 2)) {
38781     int M0 = WideMask64[0];
38782     int M1 = WideMask64[1];
38783     if (isInRange(M0, 0, 4) && isInRange(M1, 0, 4)) {
38784       MVT HalfVT = VT0.getSimpleVT().getHalfNumVectorElementsVT();
38785       unsigned Idx0 = (M0 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
38786       unsigned Idx1 = (M1 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
38787       SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL);
38788       SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL);
38789       SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1);
38790       return widenSubVector(Res, false, Subtarget, DAG, DL, 256);
38791     }
38792   }
38793 
38794   return SDValue();
38795 }
38796 
38797 // Attempt to constant fold all of the constant source ops.
38798 // Returns true if the entire shuffle is folded to a constant.
38799 // TODO: Extend this to merge multiple constant Ops and update the mask.
38800 static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
38801                                            ArrayRef<int> Mask, SDValue Root,
38802                                            bool HasVariableMask,
38803                                            SelectionDAG &DAG,
38804                                            const X86Subtarget &Subtarget) {
38805   MVT VT = Root.getSimpleValueType();
38806 
38807   unsigned SizeInBits = VT.getSizeInBits();
38808   unsigned NumMaskElts = Mask.size();
38809   unsigned MaskSizeInBits = SizeInBits / NumMaskElts;
38810   unsigned NumOps = Ops.size();
38811 
38812   // Extract constant bits from each source op.
38813   SmallVector<APInt, 16> UndefEltsOps(NumOps);
38814   SmallVector<SmallVector<APInt, 16>, 16> RawBitsOps(NumOps);
38815   for (unsigned I = 0; I != NumOps; ++I)
38816     if (!getTargetConstantBitsFromNode(Ops[I], MaskSizeInBits, UndefEltsOps[I],
38817                                        RawBitsOps[I]))
38818       return SDValue();
38819 
38820   // If we're optimizing for size, only fold if at least one of the constants is
38821   // only used once or the combined shuffle has included a variable mask
38822   // shuffle, this is to avoid constant pool bloat.
38823   bool IsOptimizingSize = DAG.shouldOptForSize();
38824   if (IsOptimizingSize && !HasVariableMask &&
38825       llvm::none_of(Ops, [](SDValue SrcOp) { return SrcOp->hasOneUse(); }))
38826     return SDValue();
38827 
38828   // Shuffle the constant bits according to the mask.
38829   SDLoc DL(Root);
38830   APInt UndefElts(NumMaskElts, 0);
38831   APInt ZeroElts(NumMaskElts, 0);
38832   APInt ConstantElts(NumMaskElts, 0);
38833   SmallVector<APInt, 8> ConstantBitData(NumMaskElts,
38834                                         APInt::getZero(MaskSizeInBits));
38835   for (unsigned i = 0; i != NumMaskElts; ++i) {
38836     int M = Mask[i];
38837     if (M == SM_SentinelUndef) {
38838       UndefElts.setBit(i);
38839       continue;
38840     } else if (M == SM_SentinelZero) {
38841       ZeroElts.setBit(i);
38842       continue;
38843     }
38844     assert(0 <= M && M < (int)(NumMaskElts * NumOps));
38845 
38846     unsigned SrcOpIdx = (unsigned)M / NumMaskElts;
38847     unsigned SrcMaskIdx = (unsigned)M % NumMaskElts;
38848 
38849     auto &SrcUndefElts = UndefEltsOps[SrcOpIdx];
38850     if (SrcUndefElts[SrcMaskIdx]) {
38851       UndefElts.setBit(i);
38852       continue;
38853     }
38854 
38855     auto &SrcEltBits = RawBitsOps[SrcOpIdx];
38856     APInt &Bits = SrcEltBits[SrcMaskIdx];
38857     if (!Bits) {
38858       ZeroElts.setBit(i);
38859       continue;
38860     }
38861 
38862     ConstantElts.setBit(i);
38863     ConstantBitData[i] = Bits;
38864   }
38865   assert((UndefElts | ZeroElts | ConstantElts).isAllOnes());
38866 
38867   // Attempt to create a zero vector.
38868   if ((UndefElts | ZeroElts).isAllOnes())
38869     return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL);
38870 
38871   // Create the constant data.
38872   MVT MaskSVT;
38873   if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))
38874     MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);
38875   else
38876     MaskSVT = MVT::getIntegerVT(MaskSizeInBits);
38877 
38878   MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts);
38879   if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
38880     return SDValue();
38881 
38882   SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
38883   return DAG.getBitcast(VT, CstOp);
38884 }
38885 
38886 namespace llvm {
38887   namespace X86 {
38888     enum {
38889       MaxShuffleCombineDepth = 8
38890     };
38891   } // namespace X86
38892 } // namespace llvm
38893 
38894 /// Fully generic combining of x86 shuffle instructions.
38895 ///
38896 /// This should be the last combine run over the x86 shuffle instructions. Once
38897 /// they have been fully optimized, this will recursively consider all chains
38898 /// of single-use shuffle instructions, build a generic model of the cumulative
38899 /// shuffle operation, and check for simpler instructions which implement this
38900 /// operation. We use this primarily for two purposes:
38901 ///
38902 /// 1) Collapse generic shuffles to specialized single instructions when
38903 ///    equivalent. In most cases, this is just an encoding size win, but
38904 ///    sometimes we will collapse multiple generic shuffles into a single
38905 ///    special-purpose shuffle.
38906 /// 2) Look for sequences of shuffle instructions with 3 or more total
38907 ///    instructions, and replace them with the slightly more expensive SSSE3
38908 ///    PSHUFB instruction if available. We do this as the last combining step
38909 ///    to ensure we avoid using PSHUFB if we can implement the shuffle with
38910 ///    a suitable short sequence of other instructions. The PSHUFB will either
38911 ///    use a register or have to read from memory and so is slightly (but only
38912 ///    slightly) more expensive than the other shuffle instructions.
38913 ///
38914 /// Because this is inherently a quadratic operation (for each shuffle in
38915 /// a chain, we recurse up the chain), the depth is limited to 8 instructions.
38916 /// This should never be an issue in practice as the shuffle lowering doesn't
38917 /// produce sequences of more than 8 instructions.
38918 ///
38919 /// FIXME: We will currently miss some cases where the redundant shuffling
38920 /// would simplify under the threshold for PSHUFB formation because of
38921 /// combine-ordering. To fix this, we should do the redundant instruction
38922 /// combining in this recursive walk.
38923 static SDValue combineX86ShufflesRecursively(
38924     ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
38925     ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
38926     unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask,
38927     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38928     const X86Subtarget &Subtarget) {
38929   assert(!RootMask.empty() &&
38930          (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
38931          "Illegal shuffle root mask");
38932   MVT RootVT = Root.getSimpleValueType();
38933   assert(RootVT.isVector() && "Shuffles operate on vector types!");
38934   unsigned RootSizeInBits = RootVT.getSizeInBits();
38935 
38936   // Bound the depth of our recursive combine because this is ultimately
38937   // quadratic in nature.
38938   if (Depth >= MaxDepth)
38939     return SDValue();
38940 
38941   // Directly rip through bitcasts to find the underlying operand.
38942   SDValue Op = SrcOps[SrcOpIndex];
38943   Op = peekThroughOneUseBitcasts(Op);
38944 
38945   EVT VT = Op.getValueType();
38946   if (!VT.isVector() || !VT.isSimple())
38947     return SDValue(); // Bail if we hit a non-simple non-vector.
38948 
38949   // FIXME: Just bail on f16 for now.
38950   if (VT.getVectorElementType() == MVT::f16)
38951     return SDValue();
38952 
38953   assert((RootSizeInBits % VT.getSizeInBits()) == 0 &&
38954          "Can only combine shuffles upto size of the root op.");
38955 
38956   // Create a demanded elts mask from the referenced elements of Op.
38957   APInt OpDemandedElts = APInt::getZero(RootMask.size());
38958   for (int M : RootMask) {
38959     int BaseIdx = RootMask.size() * SrcOpIndex;
38960     if (isInRange(M, BaseIdx, BaseIdx + RootMask.size()))
38961       OpDemandedElts.setBit(M - BaseIdx);
38962   }
38963   if (RootSizeInBits != VT.getSizeInBits()) {
38964     // Op is smaller than Root - extract the demanded elts for the subvector.
38965     unsigned Scale = RootSizeInBits / VT.getSizeInBits();
38966     unsigned NumOpMaskElts = RootMask.size() / Scale;
38967     assert((RootMask.size() % Scale) == 0 && "Root mask size mismatch");
38968     assert(OpDemandedElts
38969                .extractBits(RootMask.size() - NumOpMaskElts, NumOpMaskElts)
38970                .isZero() &&
38971            "Out of range elements referenced in root mask");
38972     OpDemandedElts = OpDemandedElts.extractBits(NumOpMaskElts, 0);
38973   }
38974   OpDemandedElts =
38975       APIntOps::ScaleBitMask(OpDemandedElts, VT.getVectorNumElements());
38976 
38977   // Extract target shuffle mask and resolve sentinels and inputs.
38978   SmallVector<int, 64> OpMask;
38979   SmallVector<SDValue, 2> OpInputs;
38980   APInt OpUndef, OpZero;
38981   bool IsOpVariableMask = isTargetShuffleVariableMask(Op.getOpcode());
38982   if (getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef,
38983                              OpZero, DAG, Depth, false)) {
38984     // Shuffle inputs must not be larger than the shuffle result.
38985     // TODO: Relax this for single input faux shuffles (e.g. trunc).
38986     if (llvm::any_of(OpInputs, [VT](SDValue OpInput) {
38987           return OpInput.getValueSizeInBits() > VT.getSizeInBits();
38988         }))
38989       return SDValue();
38990   } else if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
38991              (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
38992              !isNullConstant(Op.getOperand(1))) {
38993     SDValue SrcVec = Op.getOperand(0);
38994     int ExtractIdx = Op.getConstantOperandVal(1);
38995     unsigned NumElts = VT.getVectorNumElements();
38996     OpInputs.assign({SrcVec});
38997     OpMask.assign(NumElts, SM_SentinelUndef);
38998     std::iota(OpMask.begin(), OpMask.end(), ExtractIdx);
38999     OpZero = OpUndef = APInt::getZero(NumElts);
39000   } else {
39001     return SDValue();
39002   }
39003 
39004   // If the shuffle result was smaller than the root, we need to adjust the
39005   // mask indices and pad the mask with undefs.
39006   if (RootSizeInBits > VT.getSizeInBits()) {
39007     unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits();
39008     unsigned OpMaskSize = OpMask.size();
39009     if (OpInputs.size() > 1) {
39010       unsigned PaddedMaskSize = NumSubVecs * OpMaskSize;
39011       for (int &M : OpMask) {
39012         if (M < 0)
39013           continue;
39014         int EltIdx = M % OpMaskSize;
39015         int OpIdx = M / OpMaskSize;
39016         M = (PaddedMaskSize * OpIdx) + EltIdx;
39017       }
39018     }
39019     OpZero = OpZero.zext(NumSubVecs * OpMaskSize);
39020     OpUndef = OpUndef.zext(NumSubVecs * OpMaskSize);
39021     OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef);
39022   }
39023 
39024   SmallVector<int, 64> Mask;
39025   SmallVector<SDValue, 16> Ops;
39026 
39027   // We don't need to merge masks if the root is empty.
39028   bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
39029   if (EmptyRoot) {
39030     // Only resolve zeros if it will remove an input, otherwise we might end
39031     // up in an infinite loop.
39032     bool ResolveKnownZeros = true;
39033     if (!OpZero.isZero()) {
39034       APInt UsedInputs = APInt::getZero(OpInputs.size());
39035       for (int i = 0, e = OpMask.size(); i != e; ++i) {
39036         int M = OpMask[i];
39037         if (OpUndef[i] || OpZero[i] || isUndefOrZero(M))
39038           continue;
39039         UsedInputs.setBit(M / OpMask.size());
39040         if (UsedInputs.isAllOnes()) {
39041           ResolveKnownZeros = false;
39042           break;
39043         }
39044       }
39045     }
39046     resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero,
39047                                       ResolveKnownZeros);
39048 
39049     Mask = OpMask;
39050     Ops.append(OpInputs.begin(), OpInputs.end());
39051   } else {
39052     resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero);
39053 
39054     // Add the inputs to the Ops list, avoiding duplicates.
39055     Ops.append(SrcOps.begin(), SrcOps.end());
39056 
39057     auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
39058       // Attempt to find an existing match.
39059       SDValue InputBC = peekThroughBitcasts(Input);
39060       for (int i = 0, e = Ops.size(); i < e; ++i)
39061         if (InputBC == peekThroughBitcasts(Ops[i]))
39062           return i;
39063       // Match failed - should we replace an existing Op?
39064       if (InsertionPoint >= 0) {
39065         Ops[InsertionPoint] = Input;
39066         return InsertionPoint;
39067       }
39068       // Add to the end of the Ops list.
39069       Ops.push_back(Input);
39070       return Ops.size() - 1;
39071     };
39072 
39073     SmallVector<int, 2> OpInputIdx;
39074     for (SDValue OpInput : OpInputs)
39075       OpInputIdx.push_back(
39076           AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
39077 
39078     assert(((RootMask.size() > OpMask.size() &&
39079              RootMask.size() % OpMask.size() == 0) ||
39080             (OpMask.size() > RootMask.size() &&
39081              OpMask.size() % RootMask.size() == 0) ||
39082             OpMask.size() == RootMask.size()) &&
39083            "The smaller number of elements must divide the larger.");
39084 
39085     // This function can be performance-critical, so we rely on the power-of-2
39086     // knowledge that we have about the mask sizes to replace div/rem ops with
39087     // bit-masks and shifts.
39088     assert(llvm::has_single_bit<uint32_t>(RootMask.size()) &&
39089            "Non-power-of-2 shuffle mask sizes");
39090     assert(llvm::has_single_bit<uint32_t>(OpMask.size()) &&
39091            "Non-power-of-2 shuffle mask sizes");
39092     unsigned RootMaskSizeLog2 = llvm::countr_zero(RootMask.size());
39093     unsigned OpMaskSizeLog2 = llvm::countr_zero(OpMask.size());
39094 
39095     unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
39096     unsigned RootRatio =
39097         std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
39098     unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
39099     assert((RootRatio == 1 || OpRatio == 1) &&
39100            "Must not have a ratio for both incoming and op masks!");
39101 
39102     assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes");
39103     assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes");
39104     assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes");
39105     unsigned RootRatioLog2 = llvm::countr_zero(RootRatio);
39106     unsigned OpRatioLog2 = llvm::countr_zero(OpRatio);
39107 
39108     Mask.resize(MaskWidth, SM_SentinelUndef);
39109 
39110     // Merge this shuffle operation's mask into our accumulated mask. Note that
39111     // this shuffle's mask will be the first applied to the input, followed by
39112     // the root mask to get us all the way to the root value arrangement. The
39113     // reason for this order is that we are recursing up the operation chain.
39114     for (unsigned i = 0; i < MaskWidth; ++i) {
39115       unsigned RootIdx = i >> RootRatioLog2;
39116       if (RootMask[RootIdx] < 0) {
39117         // This is a zero or undef lane, we're done.
39118         Mask[i] = RootMask[RootIdx];
39119         continue;
39120       }
39121 
39122       unsigned RootMaskedIdx =
39123           RootRatio == 1
39124               ? RootMask[RootIdx]
39125               : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
39126 
39127       // Just insert the scaled root mask value if it references an input other
39128       // than the SrcOp we're currently inserting.
39129       if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
39130           (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
39131         Mask[i] = RootMaskedIdx;
39132         continue;
39133       }
39134 
39135       RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
39136       unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
39137       if (OpMask[OpIdx] < 0) {
39138         // The incoming lanes are zero or undef, it doesn't matter which ones we
39139         // are using.
39140         Mask[i] = OpMask[OpIdx];
39141         continue;
39142       }
39143 
39144       // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
39145       unsigned OpMaskedIdx = OpRatio == 1 ? OpMask[OpIdx]
39146                                           : (OpMask[OpIdx] << OpRatioLog2) +
39147                                                 (RootMaskedIdx & (OpRatio - 1));
39148 
39149       OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
39150       int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
39151       assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input");
39152       OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
39153 
39154       Mask[i] = OpMaskedIdx;
39155     }
39156   }
39157 
39158   // Peek through vector widenings and set out of bounds mask indices to undef.
39159   // TODO: Can resolveTargetShuffleInputsAndMask do some of this?
39160   for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
39161     SDValue &Op = Ops[I];
39162     if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(0).isUndef() &&
39163         isNullConstant(Op.getOperand(2))) {
39164       Op = Op.getOperand(1);
39165       unsigned Scale = RootSizeInBits / Op.getValueSizeInBits();
39166       int Lo = I * Mask.size();
39167       int Hi = (I + 1) * Mask.size();
39168       int NewHi = Lo + (Mask.size() / Scale);
39169       for (int &M : Mask) {
39170         if (Lo <= M && NewHi <= M && M < Hi)
39171           M = SM_SentinelUndef;
39172       }
39173     }
39174   }
39175 
39176   // Peek through any free extract_subvector nodes back to root size.
39177   for (SDValue &Op : Ops)
39178     while (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
39179            (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
39180            isNullConstant(Op.getOperand(1)))
39181       Op = Op.getOperand(0);
39182 
39183   // Remove unused/repeated shuffle source ops.
39184   resolveTargetShuffleInputsAndMask(Ops, Mask);
39185 
39186   // Handle the all undef/zero/ones cases early.
39187   if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; }))
39188     return DAG.getUNDEF(RootVT);
39189   if (all_of(Mask, [](int Idx) { return Idx < 0; }))
39190     return getZeroVector(RootVT, Subtarget, DAG, SDLoc(Root));
39191   if (Ops.size() == 1 && ISD::isBuildVectorAllOnes(Ops[0].getNode()) &&
39192       !llvm::is_contained(Mask, SM_SentinelZero))
39193     return getOnesVector(RootVT, DAG, SDLoc(Root));
39194 
39195   assert(!Ops.empty() && "Shuffle with no inputs detected");
39196   HasVariableMask |= IsOpVariableMask;
39197 
39198   // Update the list of shuffle nodes that have been combined so far.
39199   SmallVector<const SDNode *, 16> CombinedNodes(SrcNodes.begin(),
39200                                                 SrcNodes.end());
39201   CombinedNodes.push_back(Op.getNode());
39202 
39203   // See if we can recurse into each shuffle source op (if it's a target
39204   // shuffle). The source op should only be generally combined if it either has
39205   // a single use (i.e. current Op) or all its users have already been combined,
39206   // if not then we can still combine but should prevent generation of variable
39207   // shuffles to avoid constant pool bloat.
39208   // Don't recurse if we already have more source ops than we can combine in
39209   // the remaining recursion depth.
39210   if (Ops.size() < (MaxDepth - Depth)) {
39211     for (int i = 0, e = Ops.size(); i < e; ++i) {
39212       // For empty roots, we need to resolve zeroable elements before combining
39213       // them with other shuffles.
39214       SmallVector<int, 64> ResolvedMask = Mask;
39215       if (EmptyRoot)
39216         resolveTargetShuffleFromZeroables(ResolvedMask, OpUndef, OpZero);
39217       bool AllowCrossLaneVar = false;
39218       bool AllowPerLaneVar = false;
39219       if (Ops[i].getNode()->hasOneUse() ||
39220           SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode())) {
39221         AllowCrossLaneVar = AllowVariableCrossLaneMask;
39222         AllowPerLaneVar = AllowVariablePerLaneMask;
39223       }
39224       if (SDValue Res = combineX86ShufflesRecursively(
39225               Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
39226               HasVariableMask, AllowCrossLaneVar, AllowPerLaneVar, DAG,
39227               Subtarget))
39228         return Res;
39229     }
39230   }
39231 
39232   // Attempt to constant fold all of the constant source ops.
39233   if (SDValue Cst = combineX86ShufflesConstants(
39234           Ops, Mask, Root, HasVariableMask, DAG, Subtarget))
39235     return Cst;
39236 
39237   // If constant fold failed and we only have constants - then we have
39238   // multiple uses by a single non-variable shuffle - just bail.
39239   if (Depth == 0 && llvm::all_of(Ops, [&](SDValue Op) {
39240         APInt UndefElts;
39241         SmallVector<APInt> RawBits;
39242         unsigned EltSizeInBits = RootSizeInBits / Mask.size();
39243         return getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
39244                                              RawBits);
39245       })) {
39246     return SDValue();
39247   }
39248 
39249   // Canonicalize the combined shuffle mask chain with horizontal ops.
39250   // NOTE: This will update the Ops and Mask.
39251   if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
39252           Ops, Mask, RootSizeInBits, SDLoc(Root), DAG, Subtarget))
39253     return DAG.getBitcast(RootVT, HOp);
39254 
39255   // Try to refine our inputs given our knowledge of target shuffle mask.
39256   for (auto I : enumerate(Ops)) {
39257     int OpIdx = I.index();
39258     SDValue &Op = I.value();
39259 
39260     // What range of shuffle mask element values results in picking from Op?
39261     int Lo = OpIdx * Mask.size();
39262     int Hi = Lo + Mask.size();
39263 
39264     // Which elements of Op do we demand, given the mask's granularity?
39265     APInt OpDemandedElts(Mask.size(), 0);
39266     for (int MaskElt : Mask) {
39267       if (isInRange(MaskElt, Lo, Hi)) { // Picks from Op?
39268         int OpEltIdx = MaskElt - Lo;
39269         OpDemandedElts.setBit(OpEltIdx);
39270       }
39271     }
39272 
39273     // Is the shuffle result smaller than the root?
39274     if (Op.getValueSizeInBits() < RootSizeInBits) {
39275       // We padded the mask with undefs. But we now need to undo that.
39276       unsigned NumExpectedVectorElts = Mask.size();
39277       unsigned EltSizeInBits = RootSizeInBits / NumExpectedVectorElts;
39278       unsigned NumOpVectorElts = Op.getValueSizeInBits() / EltSizeInBits;
39279       assert(!OpDemandedElts.extractBits(
39280                  NumExpectedVectorElts - NumOpVectorElts, NumOpVectorElts) &&
39281              "Demanding the virtual undef widening padding?");
39282       OpDemandedElts = OpDemandedElts.trunc(NumOpVectorElts); // NUW
39283     }
39284 
39285     // The Op itself may be of different VT, so we need to scale the mask.
39286     unsigned NumOpElts = Op.getValueType().getVectorNumElements();
39287     APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
39288 
39289     // Can this operand be simplified any further, given it's demanded elements?
39290     if (SDValue NewOp =
39291             DAG.getTargetLoweringInfo().SimplifyMultipleUseDemandedVectorElts(
39292                 Op, OpScaledDemandedElts, DAG))
39293       Op = NewOp;
39294   }
39295   // FIXME: should we rerun resolveTargetShuffleInputsAndMask() now?
39296 
39297   // Widen any subvector shuffle inputs we've collected.
39298   // TODO: Remove this to avoid generating temporary nodes, we should only
39299   // widen once combineX86ShuffleChain has found a match.
39300   if (any_of(Ops, [RootSizeInBits](SDValue Op) {
39301         return Op.getValueSizeInBits() < RootSizeInBits;
39302       })) {
39303     for (SDValue &Op : Ops)
39304       if (Op.getValueSizeInBits() < RootSizeInBits)
39305         Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op),
39306                             RootSizeInBits);
39307     // Reresolve - we might have repeated subvector sources.
39308     resolveTargetShuffleInputsAndMask(Ops, Mask);
39309   }
39310 
39311   // We can only combine unary and binary shuffle mask cases.
39312   if (Ops.size() <= 2) {
39313     // Minor canonicalization of the accumulated shuffle mask to make it easier
39314     // to match below. All this does is detect masks with sequential pairs of
39315     // elements, and shrink them to the half-width mask. It does this in a loop
39316     // so it will reduce the size of the mask to the minimal width mask which
39317     // performs an equivalent shuffle.
39318     while (Mask.size() > 1) {
39319       SmallVector<int, 64> WidenedMask;
39320       if (!canWidenShuffleElements(Mask, WidenedMask))
39321         break;
39322       Mask = std::move(WidenedMask);
39323     }
39324 
39325     // Canonicalization of binary shuffle masks to improve pattern matching by
39326     // commuting the inputs.
39327     if (Ops.size() == 2 && canonicalizeShuffleMaskWithCommute(Mask)) {
39328       ShuffleVectorSDNode::commuteMask(Mask);
39329       std::swap(Ops[0], Ops[1]);
39330     }
39331 
39332     // Try to combine into a single shuffle instruction.
39333     if (SDValue Shuffle = combineX86ShuffleChain(
39334             Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
39335             AllowVariablePerLaneMask, DAG, Subtarget))
39336       return Shuffle;
39337 
39338     // If all the operands come from the same larger vector, fallthrough and try
39339     // to use combineX86ShuffleChainWithExtract.
39340     SDValue LHS = peekThroughBitcasts(Ops.front());
39341     SDValue RHS = peekThroughBitcasts(Ops.back());
39342     if (Ops.size() != 2 || !Subtarget.hasAVX2() || RootSizeInBits != 128 ||
39343         (RootSizeInBits / Mask.size()) != 64 ||
39344         LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
39345         RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
39346         LHS.getOperand(0) != RHS.getOperand(0))
39347       return SDValue();
39348   }
39349 
39350   // If that failed and any input is extracted then try to combine as a
39351   // shuffle with the larger type.
39352   return combineX86ShuffleChainWithExtract(
39353       Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
39354       AllowVariablePerLaneMask, DAG, Subtarget);
39355 }
39356 
39357 /// Helper entry wrapper to combineX86ShufflesRecursively.
39358 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
39359                                              const X86Subtarget &Subtarget) {
39360   return combineX86ShufflesRecursively(
39361       {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
39362       /*HasVarMask*/ false,
39363       /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
39364       Subtarget);
39365 }
39366 
39367 /// Get the PSHUF-style mask from PSHUF node.
39368 ///
39369 /// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
39370 /// PSHUF-style masks that can be reused with such instructions.
39371 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
39372   MVT VT = N.getSimpleValueType();
39373   SmallVector<int, 4> Mask;
39374   SmallVector<SDValue, 2> Ops;
39375   bool HaveMask =
39376       getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask);
39377   (void)HaveMask;
39378   assert(HaveMask);
39379 
39380   // If we have more than 128-bits, only the low 128-bits of shuffle mask
39381   // matter. Check that the upper masks are repeats and remove them.
39382   if (VT.getSizeInBits() > 128) {
39383     int LaneElts = 128 / VT.getScalarSizeInBits();
39384 #ifndef NDEBUG
39385     for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)
39386       for (int j = 0; j < LaneElts; ++j)
39387         assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
39388                "Mask doesn't repeat in high 128-bit lanes!");
39389 #endif
39390     Mask.resize(LaneElts);
39391   }
39392 
39393   switch (N.getOpcode()) {
39394   case X86ISD::PSHUFD:
39395     return Mask;
39396   case X86ISD::PSHUFLW:
39397     Mask.resize(4);
39398     return Mask;
39399   case X86ISD::PSHUFHW:
39400     Mask.erase(Mask.begin(), Mask.begin() + 4);
39401     for (int &M : Mask)
39402       M -= 4;
39403     return Mask;
39404   default:
39405     llvm_unreachable("No valid shuffle instruction found!");
39406   }
39407 }
39408 
39409 /// Search for a combinable shuffle across a chain ending in pshufd.
39410 ///
39411 /// We walk up the chain and look for a combinable shuffle, skipping over
39412 /// shuffles that we could hoist this shuffle's transformation past without
39413 /// altering anything.
39414 static SDValue
39415 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
39416                              SelectionDAG &DAG) {
39417   assert(N.getOpcode() == X86ISD::PSHUFD &&
39418          "Called with something other than an x86 128-bit half shuffle!");
39419   SDLoc DL(N);
39420 
39421   // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
39422   // of the shuffles in the chain so that we can form a fresh chain to replace
39423   // this one.
39424   SmallVector<SDValue, 8> Chain;
39425   SDValue V = N.getOperand(0);
39426   for (; V.hasOneUse(); V = V.getOperand(0)) {
39427     switch (V.getOpcode()) {
39428     default:
39429       return SDValue(); // Nothing combined!
39430 
39431     case ISD::BITCAST:
39432       // Skip bitcasts as we always know the type for the target specific
39433       // instructions.
39434       continue;
39435 
39436     case X86ISD::PSHUFD:
39437       // Found another dword shuffle.
39438       break;
39439 
39440     case X86ISD::PSHUFLW:
39441       // Check that the low words (being shuffled) are the identity in the
39442       // dword shuffle, and the high words are self-contained.
39443       if (Mask[0] != 0 || Mask[1] != 1 ||
39444           !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
39445         return SDValue();
39446 
39447       Chain.push_back(V);
39448       continue;
39449 
39450     case X86ISD::PSHUFHW:
39451       // Check that the high words (being shuffled) are the identity in the
39452       // dword shuffle, and the low words are self-contained.
39453       if (Mask[2] != 2 || Mask[3] != 3 ||
39454           !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
39455         return SDValue();
39456 
39457       Chain.push_back(V);
39458       continue;
39459 
39460     case X86ISD::UNPCKL:
39461     case X86ISD::UNPCKH:
39462       // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
39463       // shuffle into a preceding word shuffle.
39464       if (V.getSimpleValueType().getVectorElementType() != MVT::i8 &&
39465           V.getSimpleValueType().getVectorElementType() != MVT::i16)
39466         return SDValue();
39467 
39468       // Search for a half-shuffle which we can combine with.
39469       unsigned CombineOp =
39470           V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
39471       if (V.getOperand(0) != V.getOperand(1) ||
39472           !V->isOnlyUserOf(V.getOperand(0).getNode()))
39473         return SDValue();
39474       Chain.push_back(V);
39475       V = V.getOperand(0);
39476       do {
39477         switch (V.getOpcode()) {
39478         default:
39479           return SDValue(); // Nothing to combine.
39480 
39481         case X86ISD::PSHUFLW:
39482         case X86ISD::PSHUFHW:
39483           if (V.getOpcode() == CombineOp)
39484             break;
39485 
39486           Chain.push_back(V);
39487 
39488           [[fallthrough]];
39489         case ISD::BITCAST:
39490           V = V.getOperand(0);
39491           continue;
39492         }
39493         break;
39494       } while (V.hasOneUse());
39495       break;
39496     }
39497     // Break out of the loop if we break out of the switch.
39498     break;
39499   }
39500 
39501   if (!V.hasOneUse())
39502     // We fell out of the loop without finding a viable combining instruction.
39503     return SDValue();
39504 
39505   // Merge this node's mask and our incoming mask.
39506   SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
39507   for (int &M : Mask)
39508     M = VMask[M];
39509   V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
39510                   getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
39511 
39512   // Rebuild the chain around this new shuffle.
39513   while (!Chain.empty()) {
39514     SDValue W = Chain.pop_back_val();
39515 
39516     if (V.getValueType() != W.getOperand(0).getValueType())
39517       V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
39518 
39519     switch (W.getOpcode()) {
39520     default:
39521       llvm_unreachable("Only PSHUF and UNPCK instructions get here!");
39522 
39523     case X86ISD::UNPCKL:
39524     case X86ISD::UNPCKH:
39525       V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
39526       break;
39527 
39528     case X86ISD::PSHUFD:
39529     case X86ISD::PSHUFLW:
39530     case X86ISD::PSHUFHW:
39531       V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
39532       break;
39533     }
39534   }
39535   if (V.getValueType() != N.getValueType())
39536     V = DAG.getBitcast(N.getValueType(), V);
39537 
39538   // Return the new chain to replace N.
39539   return V;
39540 }
39541 
39542 // Attempt to commute shufps LHS loads:
39543 // permilps(shufps(load(),x)) --> permilps(shufps(x,load()))
39544 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
39545                                       SelectionDAG &DAG) {
39546   // TODO: Add vXf64 support.
39547   if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32)
39548     return SDValue();
39549 
39550   // SHUFP(LHS, RHS) -> SHUFP(RHS, LHS) iff LHS is foldable + RHS is not.
39551   auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
39552     if (V.getOpcode() != X86ISD::SHUFP || !Parent->isOnlyUserOf(V.getNode()))
39553       return SDValue();
39554     SDValue N0 = V.getOperand(0);
39555     SDValue N1 = V.getOperand(1);
39556     unsigned Imm = V.getConstantOperandVal(2);
39557     const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
39558     if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
39559         X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
39560       return SDValue();
39561     Imm = ((Imm & 0x0F) << 4) | ((Imm & 0xF0) >> 4);
39562     return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
39563                        DAG.getTargetConstant(Imm, DL, MVT::i8));
39564   };
39565 
39566   switch (N.getOpcode()) {
39567   case X86ISD::VPERMILPI:
39568     if (SDValue NewSHUFP = commuteSHUFP(N, N.getOperand(0))) {
39569       unsigned Imm = N.getConstantOperandVal(1);
39570       return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP,
39571                          DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
39572     }
39573     break;
39574   case X86ISD::SHUFP: {
39575     SDValue N0 = N.getOperand(0);
39576     SDValue N1 = N.getOperand(1);
39577     unsigned Imm = N.getConstantOperandVal(2);
39578     if (N0 == N1) {
39579       if (SDValue NewSHUFP = commuteSHUFP(N, N0))
39580         return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP,
39581                            DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
39582     } else if (SDValue NewSHUFP = commuteSHUFP(N, N0)) {
39583       return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1,
39584                          DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));
39585     } else if (SDValue NewSHUFP = commuteSHUFP(N, N1)) {
39586       return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP,
39587                          DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));
39588     }
39589     break;
39590   }
39591   }
39592 
39593   return SDValue();
39594 }
39595 
39596 // TODO - move this to TLI like isBinOp?
39597 static bool isUnaryOp(unsigned Opcode) {
39598   switch (Opcode) {
39599   case ISD::CTLZ:
39600   case ISD::CTTZ:
39601   case ISD::CTPOP:
39602     return true;
39603   }
39604   return false;
39605 }
39606 
39607 // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
39608 // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
39609 static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
39610                                          const SDLoc &DL) {
39611   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39612   EVT ShuffleVT = N.getValueType();
39613 
39614   auto IsMergeableWithShuffle = [&DAG](SDValue Op, bool FoldLoad = false) {
39615     // AllZeros/AllOnes constants are freely shuffled and will peek through
39616     // bitcasts. Other constant build vectors do not peek through bitcasts. Only
39617     // merge with target shuffles if it has one use so shuffle combining is
39618     // likely to kick in. Shuffles of splats are expected to be removed.
39619     return ISD::isBuildVectorAllOnes(Op.getNode()) ||
39620            ISD::isBuildVectorAllZeros(Op.getNode()) ||
39621            ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
39622            ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()) ||
39623            (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op->hasOneUse()) ||
39624            (isTargetShuffle(Op.getOpcode()) && Op->hasOneUse()) ||
39625            (FoldLoad && isShuffleFoldableLoad(Op)) ||
39626            DAG.isSplatValue(Op, /*AllowUndefs*/ false);
39627   };
39628   auto IsSafeToMoveShuffle = [ShuffleVT](SDValue Op, unsigned BinOp) {
39629     // Ensure we only shuffle whole vector src elements, unless its a logical
39630     // binops where we can more aggressively move shuffles from dst to src.
39631     return BinOp == ISD::AND || BinOp == ISD::OR || BinOp == ISD::XOR ||
39632            BinOp == X86ISD::ANDNP ||
39633            (Op.getScalarValueSizeInBits() <= ShuffleVT.getScalarSizeInBits());
39634   };
39635 
39636   unsigned Opc = N.getOpcode();
39637   switch (Opc) {
39638   // Unary and Unary+Permute Shuffles.
39639   case X86ISD::PSHUFB: {
39640     // Don't merge PSHUFB if it contains zero'd elements.
39641     SmallVector<int> Mask;
39642     SmallVector<SDValue> Ops;
39643     if (!getTargetShuffleMask(N.getNode(), ShuffleVT.getSimpleVT(), false, Ops,
39644                               Mask))
39645       break;
39646     [[fallthrough]];
39647   }
39648   case X86ISD::VBROADCAST:
39649   case X86ISD::MOVDDUP:
39650   case X86ISD::PSHUFD:
39651   case X86ISD::PSHUFHW:
39652   case X86ISD::PSHUFLW:
39653   case X86ISD::VPERMI:
39654   case X86ISD::VPERMILPI: {
39655     if (N.getOperand(0).getValueType() == ShuffleVT &&
39656         N->isOnlyUserOf(N.getOperand(0).getNode())) {
39657       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
39658       unsigned SrcOpcode = N0.getOpcode();
39659       if (TLI.isBinOp(SrcOpcode) && IsSafeToMoveShuffle(N0, SrcOpcode)) {
39660         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39661         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
39662         if (IsMergeableWithShuffle(Op00, Opc != X86ISD::PSHUFB) ||
39663             IsMergeableWithShuffle(Op01, Opc != X86ISD::PSHUFB)) {
39664           SDValue LHS, RHS;
39665           Op00 = DAG.getBitcast(ShuffleVT, Op00);
39666           Op01 = DAG.getBitcast(ShuffleVT, Op01);
39667           if (N.getNumOperands() == 2) {
39668             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1));
39669             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1));
39670           } else {
39671             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00);
39672             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01);
39673           }
39674           EVT OpVT = N0.getValueType();
39675           return DAG.getBitcast(ShuffleVT,
39676                                 DAG.getNode(SrcOpcode, DL, OpVT,
39677                                             DAG.getBitcast(OpVT, LHS),
39678                                             DAG.getBitcast(OpVT, RHS)));
39679         }
39680       }
39681     }
39682     break;
39683   }
39684   // Binary and Binary+Permute Shuffles.
39685   case X86ISD::INSERTPS: {
39686     // Don't merge INSERTPS if it contains zero'd elements.
39687     unsigned InsertPSMask = N.getConstantOperandVal(2);
39688     unsigned ZeroMask = InsertPSMask & 0xF;
39689     if (ZeroMask != 0)
39690       break;
39691     [[fallthrough]];
39692   }
39693   case X86ISD::MOVSD:
39694   case X86ISD::MOVSS:
39695   case X86ISD::BLENDI:
39696   case X86ISD::SHUFP:
39697   case X86ISD::UNPCKH:
39698   case X86ISD::UNPCKL: {
39699     if (N->isOnlyUserOf(N.getOperand(0).getNode()) &&
39700         N->isOnlyUserOf(N.getOperand(1).getNode())) {
39701       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
39702       SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
39703       unsigned SrcOpcode = N0.getOpcode();
39704       if (TLI.isBinOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
39705           N0.getValueType() == N1.getValueType() &&
39706           IsSafeToMoveShuffle(N0, SrcOpcode) &&
39707           IsSafeToMoveShuffle(N1, SrcOpcode)) {
39708         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39709         SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
39710         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
39711         SDValue Op11 = peekThroughOneUseBitcasts(N1.getOperand(1));
39712         // Ensure the total number of shuffles doesn't increase by folding this
39713         // shuffle through to the source ops.
39714         if (((IsMergeableWithShuffle(Op00) && IsMergeableWithShuffle(Op10)) ||
39715              (IsMergeableWithShuffle(Op01) && IsMergeableWithShuffle(Op11))) ||
39716             ((IsMergeableWithShuffle(Op00) || IsMergeableWithShuffle(Op10)) &&
39717              (IsMergeableWithShuffle(Op01) || IsMergeableWithShuffle(Op11)))) {
39718           SDValue LHS, RHS;
39719           Op00 = DAG.getBitcast(ShuffleVT, Op00);
39720           Op10 = DAG.getBitcast(ShuffleVT, Op10);
39721           Op01 = DAG.getBitcast(ShuffleVT, Op01);
39722           Op11 = DAG.getBitcast(ShuffleVT, Op11);
39723           if (N.getNumOperands() == 3) {
39724             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
39725             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2));
39726           } else {
39727             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
39728             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11);
39729           }
39730           EVT OpVT = N0.getValueType();
39731           return DAG.getBitcast(ShuffleVT,
39732                                 DAG.getNode(SrcOpcode, DL, OpVT,
39733                                             DAG.getBitcast(OpVT, LHS),
39734                                             DAG.getBitcast(OpVT, RHS)));
39735         }
39736       }
39737       if (isUnaryOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
39738           N0.getValueType() == N1.getValueType() &&
39739           IsSafeToMoveShuffle(N0, SrcOpcode) &&
39740           IsSafeToMoveShuffle(N1, SrcOpcode)) {
39741         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39742         SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
39743         SDValue Res;
39744         Op00 = DAG.getBitcast(ShuffleVT, Op00);
39745         Op10 = DAG.getBitcast(ShuffleVT, Op10);
39746         if (N.getNumOperands() == 3) {
39747           Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
39748         } else {
39749           Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
39750         }
39751         EVT OpVT = N0.getValueType();
39752         return DAG.getBitcast(
39753             ShuffleVT,
39754             DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res)));
39755       }
39756     }
39757     break;
39758   }
39759   }
39760   return SDValue();
39761 }
39762 
39763 /// Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
39764 static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
39765                                                       SelectionDAG &DAG,
39766                                                       const SDLoc &DL) {
39767   assert(V.getOpcode() == X86ISD::VPERM2X128 && "Unknown lane shuffle");
39768 
39769   MVT VT = V.getSimpleValueType();
39770   SDValue Src0 = peekThroughBitcasts(V.getOperand(0));
39771   SDValue Src1 = peekThroughBitcasts(V.getOperand(1));
39772   unsigned SrcOpc0 = Src0.getOpcode();
39773   unsigned SrcOpc1 = Src1.getOpcode();
39774   EVT SrcVT0 = Src0.getValueType();
39775   EVT SrcVT1 = Src1.getValueType();
39776 
39777   if (!Src1.isUndef() && (SrcVT0 != SrcVT1 || SrcOpc0 != SrcOpc1))
39778     return SDValue();
39779 
39780   switch (SrcOpc0) {
39781   case X86ISD::MOVDDUP: {
39782     SDValue LHS = Src0.getOperand(0);
39783     SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
39784     SDValue Res =
39785         DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2));
39786     Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res);
39787     return DAG.getBitcast(VT, Res);
39788   }
39789   case X86ISD::VPERMILPI:
39790     // TODO: Handle v4f64 permutes with different low/high lane masks.
39791     if (SrcVT0 == MVT::v4f64) {
39792       uint64_t Mask = Src0.getConstantOperandVal(1);
39793       if ((Mask & 0x3) != ((Mask >> 2) & 0x3))
39794         break;
39795     }
39796     [[fallthrough]];
39797   case X86ISD::VSHLI:
39798   case X86ISD::VSRLI:
39799   case X86ISD::VSRAI:
39800   case X86ISD::PSHUFD:
39801     if (Src1.isUndef() || Src0.getOperand(1) == Src1.getOperand(1)) {
39802       SDValue LHS = Src0.getOperand(0);
39803       SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
39804       SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS,
39805                                 V.getOperand(2));
39806       Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1));
39807       return DAG.getBitcast(VT, Res);
39808     }
39809     break;
39810   }
39811 
39812   return SDValue();
39813 }
39814 
39815 /// Try to combine x86 target specific shuffles.
39816 static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
39817                                     TargetLowering::DAGCombinerInfo &DCI,
39818                                     const X86Subtarget &Subtarget) {
39819   SDLoc DL(N);
39820   MVT VT = N.getSimpleValueType();
39821   SmallVector<int, 4> Mask;
39822   unsigned Opcode = N.getOpcode();
39823 
39824   if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
39825     return R;
39826 
39827   // Handle specific target shuffles.
39828   switch (Opcode) {
39829   case X86ISD::MOVDDUP: {
39830     SDValue Src = N.getOperand(0);
39831     // Turn a 128-bit MOVDDUP of a full vector load into movddup+vzload.
39832     if (VT == MVT::v2f64 && Src.hasOneUse() &&
39833         ISD::isNormalLoad(Src.getNode())) {
39834       LoadSDNode *LN = cast<LoadSDNode>(Src);
39835       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
39836         SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
39837         DCI.CombineTo(N.getNode(), Movddup);
39838         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
39839         DCI.recursivelyDeleteUnusedNodes(LN);
39840         return N; // Return N so it doesn't get rechecked!
39841       }
39842     }
39843 
39844     return SDValue();
39845   }
39846   case X86ISD::VBROADCAST: {
39847     SDValue Src = N.getOperand(0);
39848     SDValue BC = peekThroughBitcasts(Src);
39849     EVT SrcVT = Src.getValueType();
39850     EVT BCVT = BC.getValueType();
39851 
39852     // If broadcasting from another shuffle, attempt to simplify it.
39853     // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
39854     if (isTargetShuffle(BC.getOpcode()) &&
39855         VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {
39856       unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();
39857       SmallVector<int, 16> DemandedMask(BCVT.getVectorNumElements(),
39858                                         SM_SentinelUndef);
39859       for (unsigned i = 0; i != Scale; ++i)
39860         DemandedMask[i] = i;
39861       if (SDValue Res = combineX86ShufflesRecursively(
39862               {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
39863               X86::MaxShuffleCombineDepth,
39864               /*HasVarMask*/ false, /*AllowCrossLaneVarMask*/ true,
39865               /*AllowPerLaneVarMask*/ true, DAG, Subtarget))
39866         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
39867                            DAG.getBitcast(SrcVT, Res));
39868     }
39869 
39870     // broadcast(bitcast(src)) -> bitcast(broadcast(src))
39871     // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward.
39872     if (Src.getOpcode() == ISD::BITCAST &&
39873         SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits() &&
39874         DAG.getTargetLoweringInfo().isTypeLegal(BCVT) &&
39875         FixedVectorType::isValidElementType(
39876             BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) {
39877       EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
39878                                    VT.getVectorNumElements());
39879       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
39880     }
39881 
39882     // vbroadcast(bitcast(vbroadcast(src))) -> bitcast(vbroadcast(src))
39883     // If we're re-broadcasting a smaller type then broadcast with that type and
39884     // bitcast.
39885     // TODO: Do this for any splat?
39886     if (Src.getOpcode() == ISD::BITCAST &&
39887         (BC.getOpcode() == X86ISD::VBROADCAST ||
39888          BC.getOpcode() == X86ISD::VBROADCAST_LOAD) &&
39889         (VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits()) == 0 &&
39890         (VT.getSizeInBits() % BCVT.getSizeInBits()) == 0) {
39891       MVT NewVT =
39892           MVT::getVectorVT(BCVT.getSimpleVT().getScalarType(),
39893                            VT.getSizeInBits() / BCVT.getScalarSizeInBits());
39894       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
39895     }
39896 
39897     // Reduce broadcast source vector to lowest 128-bits.
39898     if (SrcVT.getSizeInBits() > 128)
39899       return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
39900                          extract128BitVector(Src, 0, DAG, DL));
39901 
39902     // broadcast(scalar_to_vector(x)) -> broadcast(x).
39903     if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR &&
39904         Src.getValueType().getScalarType() == Src.getOperand(0).getValueType())
39905       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
39906 
39907     // broadcast(extract_vector_elt(x, 0)) -> broadcast(x).
39908     if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
39909         isNullConstant(Src.getOperand(1)) &&
39910         Src.getValueType() ==
39911             Src.getOperand(0).getValueType().getScalarType() &&
39912         DAG.getTargetLoweringInfo().isTypeLegal(
39913             Src.getOperand(0).getValueType()))
39914       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
39915 
39916     // Share broadcast with the longest vector and extract low subvector (free).
39917     // Ensure the same SDValue from the SDNode use is being used.
39918     for (SDNode *User : Src->uses())
39919       if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
39920           Src == User->getOperand(0) &&
39921           User->getValueSizeInBits(0).getFixedValue() >
39922               VT.getFixedSizeInBits()) {
39923         return extractSubVector(SDValue(User, 0), 0, DAG, DL,
39924                                 VT.getSizeInBits());
39925       }
39926 
39927     // vbroadcast(scalarload X) -> vbroadcast_load X
39928     // For float loads, extract other uses of the scalar from the broadcast.
39929     if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) &&
39930         ISD::isNormalLoad(Src.getNode())) {
39931       LoadSDNode *LN = cast<LoadSDNode>(Src);
39932       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
39933       SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
39934       SDValue BcastLd =
39935           DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
39936                                   LN->getMemoryVT(), LN->getMemOperand());
39937       // If the load value is used only by N, replace it via CombineTo N.
39938       bool NoReplaceExtract = Src.hasOneUse();
39939       DCI.CombineTo(N.getNode(), BcastLd);
39940       if (NoReplaceExtract) {
39941         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
39942         DCI.recursivelyDeleteUnusedNodes(LN);
39943       } else {
39944         SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
39945                                   DAG.getIntPtrConstant(0, DL));
39946         DCI.CombineTo(LN, Scl, BcastLd.getValue(1));
39947       }
39948       return N; // Return N so it doesn't get rechecked!
39949     }
39950 
39951     // Due to isTypeDesirableForOp, we won't always shrink a load truncated to
39952     // i16. So shrink it ourselves if we can make a broadcast_load.
39953     if (SrcVT == MVT::i16 && Src.getOpcode() == ISD::TRUNCATE &&
39954         Src.hasOneUse() && Src.getOperand(0).hasOneUse()) {
39955       assert(Subtarget.hasAVX2() && "Expected AVX2");
39956       SDValue TruncIn = Src.getOperand(0);
39957 
39958       // If this is a truncate of a non extending load we can just narrow it to
39959       // use a broadcast_load.
39960       if (ISD::isNormalLoad(TruncIn.getNode())) {
39961         LoadSDNode *LN = cast<LoadSDNode>(TruncIn);
39962         // Unless its volatile or atomic.
39963         if (LN->isSimple()) {
39964           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
39965           SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
39966           SDValue BcastLd = DAG.getMemIntrinsicNode(
39967               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
39968               LN->getPointerInfo(), LN->getOriginalAlign(),
39969               LN->getMemOperand()->getFlags());
39970           DCI.CombineTo(N.getNode(), BcastLd);
39971           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
39972           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
39973           return N; // Return N so it doesn't get rechecked!
39974         }
39975       }
39976 
39977       // If this is a truncate of an i16 extload, we can directly replace it.
39978       if (ISD::isUNINDEXEDLoad(Src.getOperand(0).getNode()) &&
39979           ISD::isEXTLoad(Src.getOperand(0).getNode())) {
39980         LoadSDNode *LN = cast<LoadSDNode>(Src.getOperand(0));
39981         if (LN->getMemoryVT().getSizeInBits() == 16) {
39982           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
39983           SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
39984           SDValue BcastLd =
39985               DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
39986                                       LN->getMemoryVT(), LN->getMemOperand());
39987           DCI.CombineTo(N.getNode(), BcastLd);
39988           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
39989           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
39990           return N; // Return N so it doesn't get rechecked!
39991         }
39992       }
39993 
39994       // If this is a truncate of load that has been shifted right, we can
39995       // offset the pointer and use a narrower load.
39996       if (TruncIn.getOpcode() == ISD::SRL &&
39997           TruncIn.getOperand(0).hasOneUse() &&
39998           isa<ConstantSDNode>(TruncIn.getOperand(1)) &&
39999           ISD::isNormalLoad(TruncIn.getOperand(0).getNode())) {
40000         LoadSDNode *LN = cast<LoadSDNode>(TruncIn.getOperand(0));
40001         unsigned ShiftAmt = TruncIn.getConstantOperandVal(1);
40002         // Make sure the shift amount and the load size are divisible by 16.
40003         // Don't do this if the load is volatile or atomic.
40004         if (ShiftAmt % 16 == 0 && TruncIn.getValueSizeInBits() % 16 == 0 &&
40005             LN->isSimple()) {
40006           unsigned Offset = ShiftAmt / 8;
40007           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40008           SDValue Ptr = DAG.getMemBasePlusOffset(
40009               LN->getBasePtr(), TypeSize::getFixed(Offset), DL);
40010           SDValue Ops[] = { LN->getChain(), Ptr };
40011           SDValue BcastLd = DAG.getMemIntrinsicNode(
40012               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
40013               LN->getPointerInfo().getWithOffset(Offset),
40014               LN->getOriginalAlign(),
40015               LN->getMemOperand()->getFlags());
40016           DCI.CombineTo(N.getNode(), BcastLd);
40017           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40018           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40019           return N; // Return N so it doesn't get rechecked!
40020         }
40021       }
40022     }
40023 
40024     // vbroadcast(vzload X) -> vbroadcast_load X
40025     if (Src.getOpcode() == X86ISD::VZEXT_LOAD && Src.hasOneUse()) {
40026       MemSDNode *LN = cast<MemIntrinsicSDNode>(Src);
40027       if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) {
40028         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40029         SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40030         SDValue BcastLd =
40031             DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40032                                     LN->getMemoryVT(), LN->getMemOperand());
40033         DCI.CombineTo(N.getNode(), BcastLd);
40034         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40035         DCI.recursivelyDeleteUnusedNodes(LN);
40036         return N; // Return N so it doesn't get rechecked!
40037       }
40038     }
40039 
40040     // vbroadcast(vector load X) -> vbroadcast_load
40041     if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 ||
40042          SrcVT == MVT::v4i32) &&
40043         Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) {
40044       LoadSDNode *LN = cast<LoadSDNode>(Src);
40045       // Unless the load is volatile or atomic.
40046       if (LN->isSimple()) {
40047         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40048         SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
40049         SDValue BcastLd = DAG.getMemIntrinsicNode(
40050             X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(),
40051             LN->getPointerInfo(), LN->getOriginalAlign(),
40052             LN->getMemOperand()->getFlags());
40053         DCI.CombineTo(N.getNode(), BcastLd);
40054         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40055         DCI.recursivelyDeleteUnusedNodes(LN);
40056         return N; // Return N so it doesn't get rechecked!
40057       }
40058     }
40059 
40060     return SDValue();
40061   }
40062   case X86ISD::VZEXT_MOVL: {
40063     SDValue N0 = N.getOperand(0);
40064 
40065     // If this a vzmovl of a full vector load, replace it with a vzload, unless
40066     // the load is volatile.
40067     if (N0.hasOneUse() && ISD::isNormalLoad(N0.getNode())) {
40068       auto *LN = cast<LoadSDNode>(N0);
40069       if (SDValue VZLoad =
40070               narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) {
40071         DCI.CombineTo(N.getNode(), VZLoad);
40072         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
40073         DCI.recursivelyDeleteUnusedNodes(LN);
40074         return N;
40075       }
40076     }
40077 
40078     // If this a VZEXT_MOVL of a VBROADCAST_LOAD, we don't need the broadcast
40079     // and can just use a VZEXT_LOAD.
40080     // FIXME: Is there some way to do this with SimplifyDemandedVectorElts?
40081     if (N0.hasOneUse() && N0.getOpcode() == X86ISD::VBROADCAST_LOAD) {
40082       auto *LN = cast<MemSDNode>(N0);
40083       if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) {
40084         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40085         SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
40086         SDValue VZLoad =
40087             DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops,
40088                                     LN->getMemoryVT(), LN->getMemOperand());
40089         DCI.CombineTo(N.getNode(), VZLoad);
40090         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
40091         DCI.recursivelyDeleteUnusedNodes(LN);
40092         return N;
40093       }
40094     }
40095 
40096     // Turn (v2i64 (vzext_movl (scalar_to_vector (i64 X)))) into
40097     // (v2i64 (bitcast (v4i32 (vzext_movl (scalar_to_vector (i32 (trunc X)))))))
40098     // if the upper bits of the i64 are zero.
40099     if (N0.hasOneUse() && N0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
40100         N0.getOperand(0).hasOneUse() &&
40101         N0.getOperand(0).getValueType() == MVT::i64) {
40102       SDValue In = N0.getOperand(0);
40103       APInt Mask = APInt::getHighBitsSet(64, 32);
40104       if (DAG.MaskedValueIsZero(In, Mask)) {
40105         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
40106         MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
40107         SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
40108         SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
40109         return DAG.getBitcast(VT, Movl);
40110       }
40111     }
40112 
40113     // Load a scalar integer constant directly to XMM instead of transferring an
40114     // immediate value from GPR.
40115     // vzext_movl (scalar_to_vector C) --> load [C,0...]
40116     if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR) {
40117       if (auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
40118         // Create a vector constant - scalar constant followed by zeros.
40119         EVT ScalarVT = N0.getOperand(0).getValueType();
40120         Type *ScalarTy = ScalarVT.getTypeForEVT(*DAG.getContext());
40121         unsigned NumElts = VT.getVectorNumElements();
40122         Constant *Zero = ConstantInt::getNullValue(ScalarTy);
40123         SmallVector<Constant *, 32> ConstantVec(NumElts, Zero);
40124         ConstantVec[0] = const_cast<ConstantInt *>(C->getConstantIntValue());
40125 
40126         // Load the vector constant from constant pool.
40127         MVT PVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
40128         SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
40129         MachinePointerInfo MPI =
40130             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
40131         Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
40132         return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment,
40133                            MachineMemOperand::MOLoad);
40134       }
40135     }
40136 
40137     // Pull subvector inserts into undef through VZEXT_MOVL by making it an
40138     // insert into a zero vector. This helps get VZEXT_MOVL closer to
40139     // scalar_to_vectors where 256/512 are canonicalized to an insert and a
40140     // 128-bit scalar_to_vector. This reduces the number of isel patterns.
40141     if (!DCI.isBeforeLegalizeOps() && N0.hasOneUse()) {
40142       SDValue V = peekThroughOneUseBitcasts(N0);
40143 
40144       if (V.getOpcode() == ISD::INSERT_SUBVECTOR && V.getOperand(0).isUndef() &&
40145           isNullConstant(V.getOperand(2))) {
40146         SDValue In = V.getOperand(1);
40147         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
40148                                      In.getValueSizeInBits() /
40149                                          VT.getScalarSizeInBits());
40150         In = DAG.getBitcast(SubVT, In);
40151         SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In);
40152         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
40153                            getZeroVector(VT, Subtarget, DAG, DL), Movl,
40154                            V.getOperand(2));
40155       }
40156     }
40157 
40158     return SDValue();
40159   }
40160   case X86ISD::BLENDI: {
40161     SDValue N0 = N.getOperand(0);
40162     SDValue N1 = N.getOperand(1);
40163 
40164     // blend(bitcast(x),bitcast(y)) -> bitcast(blend(x,y)) to narrower types.
40165     // TODO: Handle MVT::v16i16 repeated blend mask.
40166     if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST &&
40167         N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) {
40168       MVT SrcVT = N0.getOperand(0).getSimpleValueType();
40169       if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
40170           SrcVT.getScalarSizeInBits() >= 32) {
40171         unsigned BlendMask = N.getConstantOperandVal(2);
40172         unsigned Size = VT.getVectorNumElements();
40173         unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
40174         BlendMask = scaleVectorShuffleBlendMask(BlendMask, Size, Scale);
40175         return DAG.getBitcast(
40176             VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
40177                             N1.getOperand(0),
40178                             DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
40179       }
40180     }
40181     return SDValue();
40182   }
40183   case X86ISD::SHUFP: {
40184     // Fold shufps(shuffle(x),shuffle(y)) -> shufps(x,y).
40185     // This is a more relaxed shuffle combiner that can ignore oneuse limits.
40186     // TODO: Support types other than v4f32.
40187     if (VT == MVT::v4f32) {
40188       bool Updated = false;
40189       SmallVector<int> Mask;
40190       SmallVector<SDValue> Ops;
40191       if (getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask) &&
40192           Ops.size() == 2) {
40193         for (int i = 0; i != 2; ++i) {
40194           SmallVector<SDValue> SubOps;
40195           SmallVector<int> SubMask, SubScaledMask;
40196           SDValue Sub = peekThroughBitcasts(Ops[i]);
40197           // TODO: Scaling might be easier if we specify the demanded elts.
40198           if (getTargetShuffleInputs(Sub, SubOps, SubMask, DAG, 0, false) &&
40199               scaleShuffleElements(SubMask, 4, SubScaledMask) &&
40200               SubOps.size() == 1 && isUndefOrInRange(SubScaledMask, 0, 4)) {
40201             int Ofs = i * 2;
40202             Mask[Ofs + 0] = SubScaledMask[Mask[Ofs + 0] % 4] + (i * 4);
40203             Mask[Ofs + 1] = SubScaledMask[Mask[Ofs + 1] % 4] + (i * 4);
40204             Ops[i] = DAG.getBitcast(VT, SubOps[0]);
40205             Updated = true;
40206           }
40207         }
40208       }
40209       if (Updated) {
40210         for (int &M : Mask)
40211           M %= 4;
40212         Ops.push_back(getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
40213         return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops);
40214       }
40215     }
40216     return SDValue();
40217   }
40218   case X86ISD::VPERMI: {
40219     // vpermi(bitcast(x)) -> bitcast(vpermi(x)) for same number of elements.
40220     // TODO: Remove when we have preferred domains in combineX86ShuffleChain.
40221     SDValue N0 = N.getOperand(0);
40222     SDValue N1 = N.getOperand(1);
40223     unsigned EltSizeInBits = VT.getScalarSizeInBits();
40224     if (N0.getOpcode() == ISD::BITCAST &&
40225         N0.getOperand(0).getScalarValueSizeInBits() == EltSizeInBits) {
40226       SDValue Src = N0.getOperand(0);
40227       EVT SrcVT = Src.getValueType();
40228       SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
40229       return DAG.getBitcast(VT, Res);
40230     }
40231     return SDValue();
40232   }
40233   case X86ISD::SHUF128: {
40234     // If we're permuting the upper 256-bits subvectors of a concatenation, then
40235     // see if we can peek through and access the subvector directly.
40236     if (VT.is512BitVector()) {
40237       // 512-bit mask uses 4 x i2 indices - if the msb is always set then only the
40238       // upper subvector is used.
40239       SDValue LHS = N->getOperand(0);
40240       SDValue RHS = N->getOperand(1);
40241       uint64_t Mask = N->getConstantOperandVal(2);
40242       SmallVector<SDValue> LHSOps, RHSOps;
40243       SDValue NewLHS, NewRHS;
40244       if ((Mask & 0x0A) == 0x0A &&
40245           collectConcatOps(LHS.getNode(), LHSOps, DAG) && LHSOps.size() == 2) {
40246         NewLHS = widenSubVector(LHSOps[1], false, Subtarget, DAG, DL, 512);
40247         Mask &= ~0x0A;
40248       }
40249       if ((Mask & 0xA0) == 0xA0 &&
40250           collectConcatOps(RHS.getNode(), RHSOps, DAG) && RHSOps.size() == 2) {
40251         NewRHS = widenSubVector(RHSOps[1], false, Subtarget, DAG, DL, 512);
40252         Mask &= ~0xA0;
40253       }
40254       if (NewLHS || NewRHS)
40255         return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS,
40256                            NewRHS ? NewRHS : RHS,
40257                            DAG.getTargetConstant(Mask, DL, MVT::i8));
40258     }
40259     return SDValue();
40260   }
40261   case X86ISD::VPERM2X128: {
40262     // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
40263     SDValue LHS = N->getOperand(0);
40264     SDValue RHS = N->getOperand(1);
40265     if (LHS.getOpcode() == ISD::BITCAST &&
40266         (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) {
40267       EVT SrcVT = LHS.getOperand(0).getValueType();
40268       if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) {
40269         return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT,
40270                                               DAG.getBitcast(SrcVT, LHS),
40271                                               DAG.getBitcast(SrcVT, RHS),
40272                                               N->getOperand(2)));
40273       }
40274     }
40275 
40276     // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()).
40277     if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
40278       return Res;
40279 
40280     // Fold vperm2x128 subvector shuffle with an inner concat pattern.
40281     // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc.
40282     auto FindSubVector128 = [&](unsigned Idx) {
40283       if (Idx > 3)
40284         return SDValue();
40285       SDValue Src = peekThroughBitcasts(N.getOperand(Idx < 2 ? 0 : 1));
40286       SmallVector<SDValue> SubOps;
40287       if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2)
40288         return SubOps[Idx & 1];
40289       unsigned NumElts = Src.getValueType().getVectorNumElements();
40290       if ((Idx & 1) == 1 && Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
40291           Src.getOperand(1).getValueSizeInBits() == 128 &&
40292           Src.getConstantOperandAPInt(2) == (NumElts / 2)) {
40293         return Src.getOperand(1);
40294       }
40295       return SDValue();
40296     };
40297     unsigned Imm = N.getConstantOperandVal(2);
40298     if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) {
40299       if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) {
40300         MVT SubVT = VT.getHalfNumVectorElementsVT();
40301         SubLo = DAG.getBitcast(SubVT, SubLo);
40302         SubHi = DAG.getBitcast(SubVT, SubHi);
40303         return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi);
40304       }
40305     }
40306     return SDValue();
40307   }
40308   case X86ISD::PSHUFD:
40309   case X86ISD::PSHUFLW:
40310   case X86ISD::PSHUFHW: {
40311     SDValue N0 = N.getOperand(0);
40312     SDValue N1 = N.getOperand(1);
40313     if (N0->hasOneUse()) {
40314       SDValue V = peekThroughOneUseBitcasts(N0);
40315       switch (V.getOpcode()) {
40316       case X86ISD::VSHL:
40317       case X86ISD::VSRL:
40318       case X86ISD::VSRA:
40319       case X86ISD::VSHLI:
40320       case X86ISD::VSRLI:
40321       case X86ISD::VSRAI:
40322       case X86ISD::VROTLI:
40323       case X86ISD::VROTRI: {
40324         MVT InnerVT = V.getSimpleValueType();
40325         if (InnerVT.getScalarSizeInBits() <= VT.getScalarSizeInBits()) {
40326           SDValue Res = DAG.getNode(Opcode, DL, VT,
40327                                     DAG.getBitcast(VT, V.getOperand(0)), N1);
40328           Res = DAG.getBitcast(InnerVT, Res);
40329           Res = DAG.getNode(V.getOpcode(), DL, InnerVT, Res, V.getOperand(1));
40330           return DAG.getBitcast(VT, Res);
40331         }
40332         break;
40333       }
40334       }
40335     }
40336 
40337     Mask = getPSHUFShuffleMask(N);
40338     assert(Mask.size() == 4);
40339     break;
40340   }
40341   case X86ISD::MOVSD:
40342   case X86ISD::MOVSH:
40343   case X86ISD::MOVSS: {
40344     SDValue N0 = N.getOperand(0);
40345     SDValue N1 = N.getOperand(1);
40346 
40347     // Canonicalize scalar FPOps:
40348     // MOVS*(N0, OP(N0, N1)) --> MOVS*(N0, SCALAR_TO_VECTOR(OP(N0[0], N1[0])))
40349     // If commutable, allow OP(N1[0], N0[0]).
40350     unsigned Opcode1 = N1.getOpcode();
40351     if (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL || Opcode1 == ISD::FSUB ||
40352         Opcode1 == ISD::FDIV) {
40353       SDValue N10 = N1.getOperand(0);
40354       SDValue N11 = N1.getOperand(1);
40355       if (N10 == N0 ||
40356           (N11 == N0 && (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL))) {
40357         if (N10 != N0)
40358           std::swap(N10, N11);
40359         MVT SVT = VT.getVectorElementType();
40360         SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
40361         N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
40362         N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
40363         SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
40364         SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
40365         return DAG.getNode(Opcode, DL, VT, N0, SclVec);
40366       }
40367     }
40368 
40369     return SDValue();
40370   }
40371   case X86ISD::INSERTPS: {
40372     assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32");
40373     SDValue Op0 = N.getOperand(0);
40374     SDValue Op1 = N.getOperand(1);
40375     unsigned InsertPSMask = N.getConstantOperandVal(2);
40376     unsigned SrcIdx = (InsertPSMask >> 6) & 0x3;
40377     unsigned DstIdx = (InsertPSMask >> 4) & 0x3;
40378     unsigned ZeroMask = InsertPSMask & 0xF;
40379 
40380     // If we zero out all elements from Op0 then we don't need to reference it.
40381     if (((ZeroMask | (1u << DstIdx)) == 0xF) && !Op0.isUndef())
40382       return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
40383                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40384 
40385     // If we zero out the element from Op1 then we don't need to reference it.
40386     if ((ZeroMask & (1u << DstIdx)) && !Op1.isUndef())
40387       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
40388                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40389 
40390     // Attempt to merge insertps Op1 with an inner target shuffle node.
40391     SmallVector<int, 8> TargetMask1;
40392     SmallVector<SDValue, 2> Ops1;
40393     APInt KnownUndef1, KnownZero1;
40394     if (getTargetShuffleAndZeroables(Op1, TargetMask1, Ops1, KnownUndef1,
40395                                      KnownZero1)) {
40396       if (KnownUndef1[SrcIdx] || KnownZero1[SrcIdx]) {
40397         // Zero/UNDEF insertion - zero out element and remove dependency.
40398         InsertPSMask |= (1u << DstIdx);
40399         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
40400                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40401       }
40402       // Update insertps mask srcidx and reference the source input directly.
40403       int M = TargetMask1[SrcIdx];
40404       assert(0 <= M && M < 8 && "Shuffle index out of range");
40405       InsertPSMask = (InsertPSMask & 0x3f) | ((M & 0x3) << 6);
40406       Op1 = Ops1[M < 4 ? 0 : 1];
40407       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
40408                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40409     }
40410 
40411     // Attempt to merge insertps Op0 with an inner target shuffle node.
40412     SmallVector<int, 8> TargetMask0;
40413     SmallVector<SDValue, 2> Ops0;
40414     APInt KnownUndef0, KnownZero0;
40415     if (getTargetShuffleAndZeroables(Op0, TargetMask0, Ops0, KnownUndef0,
40416                                      KnownZero0)) {
40417       bool Updated = false;
40418       bool UseInput00 = false;
40419       bool UseInput01 = false;
40420       for (int i = 0; i != 4; ++i) {
40421         if ((InsertPSMask & (1u << i)) || (i == (int)DstIdx)) {
40422           // No change if element is already zero or the inserted element.
40423           continue;
40424         }
40425 
40426         if (KnownUndef0[i] || KnownZero0[i]) {
40427           // If the target mask is undef/zero then we must zero the element.
40428           InsertPSMask |= (1u << i);
40429           Updated = true;
40430           continue;
40431         }
40432 
40433         // The input vector element must be inline.
40434         int M = TargetMask0[i];
40435         if (M != i && M != (i + 4))
40436           return SDValue();
40437 
40438         // Determine which inputs of the target shuffle we're using.
40439         UseInput00 |= (0 <= M && M < 4);
40440         UseInput01 |= (4 <= M);
40441       }
40442 
40443       // If we're not using both inputs of the target shuffle then use the
40444       // referenced input directly.
40445       if (UseInput00 && !UseInput01) {
40446         Updated = true;
40447         Op0 = Ops0[0];
40448       } else if (!UseInput00 && UseInput01) {
40449         Updated = true;
40450         Op0 = Ops0[1];
40451       }
40452 
40453       if (Updated)
40454         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
40455                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40456     }
40457 
40458     // If we're inserting an element from a vbroadcast load, fold the
40459     // load into the X86insertps instruction. We need to convert the scalar
40460     // load to a vector and clear the source lane of the INSERTPS control.
40461     if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
40462       auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
40463       if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
40464         SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
40465                                    MemIntr->getBasePtr(),
40466                                    MemIntr->getMemOperand());
40467         SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
40468                            DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
40469                                        Load),
40470                            DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
40471         DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
40472         return Insert;
40473       }
40474     }
40475 
40476     return SDValue();
40477   }
40478   default:
40479     return SDValue();
40480   }
40481 
40482   // Nuke no-op shuffles that show up after combining.
40483   if (isNoopShuffleMask(Mask))
40484     return N.getOperand(0);
40485 
40486   // Look for simplifications involving one or two shuffle instructions.
40487   SDValue V = N.getOperand(0);
40488   switch (N.getOpcode()) {
40489   default:
40490     break;
40491   case X86ISD::PSHUFLW:
40492   case X86ISD::PSHUFHW:
40493     assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!");
40494 
40495     // See if this reduces to a PSHUFD which is no more expensive and can
40496     // combine with more operations. Note that it has to at least flip the
40497     // dwords as otherwise it would have been removed as a no-op.
40498     if (ArrayRef<int>(Mask).equals({2, 3, 0, 1})) {
40499       int DMask[] = {0, 1, 2, 3};
40500       int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
40501       DMask[DOffset + 0] = DOffset + 1;
40502       DMask[DOffset + 1] = DOffset + 0;
40503       MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
40504       V = DAG.getBitcast(DVT, V);
40505       V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
40506                       getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
40507       return DAG.getBitcast(VT, V);
40508     }
40509 
40510     // Look for shuffle patterns which can be implemented as a single unpack.
40511     // FIXME: This doesn't handle the location of the PSHUFD generically, and
40512     // only works when we have a PSHUFD followed by two half-shuffles.
40513     if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
40514         (V.getOpcode() == X86ISD::PSHUFLW ||
40515          V.getOpcode() == X86ISD::PSHUFHW) &&
40516         V.getOpcode() != N.getOpcode() &&
40517         V.hasOneUse() && V.getOperand(0).hasOneUse()) {
40518       SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
40519       if (D.getOpcode() == X86ISD::PSHUFD) {
40520         SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
40521         SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
40522         int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
40523         int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
40524         int WordMask[8];
40525         for (int i = 0; i < 4; ++i) {
40526           WordMask[i + NOffset] = Mask[i] + NOffset;
40527           WordMask[i + VOffset] = VMask[i] + VOffset;
40528         }
40529         // Map the word mask through the DWord mask.
40530         int MappedMask[8];
40531         for (int i = 0; i < 8; ++i)
40532           MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
40533         if (ArrayRef<int>(MappedMask).equals({0, 0, 1, 1, 2, 2, 3, 3}) ||
40534             ArrayRef<int>(MappedMask).equals({4, 4, 5, 5, 6, 6, 7, 7})) {
40535           // We can replace all three shuffles with an unpack.
40536           V = DAG.getBitcast(VT, D.getOperand(0));
40537           return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
40538                                                 : X86ISD::UNPCKH,
40539                              DL, VT, V, V);
40540         }
40541       }
40542     }
40543 
40544     break;
40545 
40546   case X86ISD::PSHUFD:
40547     if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
40548       return NewN;
40549 
40550     break;
40551   }
40552 
40553   return SDValue();
40554 }
40555 
40556 /// Checks if the shuffle mask takes subsequent elements
40557 /// alternately from two vectors.
40558 /// For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
40559 static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
40560 
40561   int ParitySrc[2] = {-1, -1};
40562   unsigned Size = Mask.size();
40563   for (unsigned i = 0; i != Size; ++i) {
40564     int M = Mask[i];
40565     if (M < 0)
40566       continue;
40567 
40568     // Make sure we are using the matching element from the input.
40569     if ((M % Size) != i)
40570       return false;
40571 
40572     // Make sure we use the same input for all elements of the same parity.
40573     int Src = M / Size;
40574     if (ParitySrc[i % 2] >= 0 && ParitySrc[i % 2] != Src)
40575       return false;
40576     ParitySrc[i % 2] = Src;
40577   }
40578 
40579   // Make sure each input is used.
40580   if (ParitySrc[0] < 0 || ParitySrc[1] < 0 || ParitySrc[0] == ParitySrc[1])
40581     return false;
40582 
40583   Op0Even = ParitySrc[0] == 0;
40584   return true;
40585 }
40586 
40587 /// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
40588 /// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
40589 /// are written to the parameters \p Opnd0 and \p Opnd1.
40590 ///
40591 /// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
40592 /// so it is easier to generically match. We also insert dummy vector shuffle
40593 /// nodes for the operands which explicitly discard the lanes which are unused
40594 /// by this operation to try to flow through the rest of the combiner
40595 /// the fact that they're unused.
40596 static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
40597                              SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
40598                              bool &IsSubAdd) {
40599 
40600   EVT VT = N->getValueType(0);
40601   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40602   if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) ||
40603       !VT.getSimpleVT().isFloatingPoint())
40604     return false;
40605 
40606   // We only handle target-independent shuffles.
40607   // FIXME: It would be easy and harmless to use the target shuffle mask
40608   // extraction tool to support more.
40609   if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
40610     return false;
40611 
40612   SDValue V1 = N->getOperand(0);
40613   SDValue V2 = N->getOperand(1);
40614 
40615   // Make sure we have an FADD and an FSUB.
40616   if ((V1.getOpcode() != ISD::FADD && V1.getOpcode() != ISD::FSUB) ||
40617       (V2.getOpcode() != ISD::FADD && V2.getOpcode() != ISD::FSUB) ||
40618       V1.getOpcode() == V2.getOpcode())
40619     return false;
40620 
40621   // If there are other uses of these operations we can't fold them.
40622   if (!V1->hasOneUse() || !V2->hasOneUse())
40623     return false;
40624 
40625   // Ensure that both operations have the same operands. Note that we can
40626   // commute the FADD operands.
40627   SDValue LHS, RHS;
40628   if (V1.getOpcode() == ISD::FSUB) {
40629     LHS = V1->getOperand(0); RHS = V1->getOperand(1);
40630     if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
40631         (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
40632       return false;
40633   } else {
40634     assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
40635     LHS = V2->getOperand(0); RHS = V2->getOperand(1);
40636     if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
40637         (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
40638       return false;
40639   }
40640 
40641   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
40642   bool Op0Even;
40643   if (!isAddSubOrSubAddMask(Mask, Op0Even))
40644     return false;
40645 
40646   // It's a subadd if the vector in the even parity is an FADD.
40647   IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
40648                      : V2->getOpcode() == ISD::FADD;
40649 
40650   Opnd0 = LHS;
40651   Opnd1 = RHS;
40652   return true;
40653 }
40654 
40655 /// Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
40656 static SDValue combineShuffleToFMAddSub(SDNode *N,
40657                                         const X86Subtarget &Subtarget,
40658                                         SelectionDAG &DAG) {
40659   // We only handle target-independent shuffles.
40660   // FIXME: It would be easy and harmless to use the target shuffle mask
40661   // extraction tool to support more.
40662   if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
40663     return SDValue();
40664 
40665   MVT VT = N->getSimpleValueType(0);
40666   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40667   if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT))
40668     return SDValue();
40669 
40670   // We're trying to match (shuffle fma(a, b, c), X86Fmsub(a, b, c).
40671   SDValue Op0 = N->getOperand(0);
40672   SDValue Op1 = N->getOperand(1);
40673   SDValue FMAdd = Op0, FMSub = Op1;
40674   if (FMSub.getOpcode() != X86ISD::FMSUB)
40675     std::swap(FMAdd, FMSub);
40676 
40677   if (FMAdd.getOpcode() != ISD::FMA || FMSub.getOpcode() != X86ISD::FMSUB ||
40678       FMAdd.getOperand(0) != FMSub.getOperand(0) || !FMAdd.hasOneUse() ||
40679       FMAdd.getOperand(1) != FMSub.getOperand(1) || !FMSub.hasOneUse() ||
40680       FMAdd.getOperand(2) != FMSub.getOperand(2))
40681     return SDValue();
40682 
40683   // Check for correct shuffle mask.
40684   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
40685   bool Op0Even;
40686   if (!isAddSubOrSubAddMask(Mask, Op0Even))
40687     return SDValue();
40688 
40689   // FMAddSub takes zeroth operand from FMSub node.
40690   SDLoc DL(N);
40691   bool IsSubAdd = Op0Even ? Op0 == FMAdd : Op1 == FMAdd;
40692   unsigned Opcode = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
40693   return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
40694                      FMAdd.getOperand(2));
40695 }
40696 
40697 /// Try to combine a shuffle into a target-specific add-sub or
40698 /// mul-add-sub node.
40699 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
40700                                                 const X86Subtarget &Subtarget,
40701                                                 SelectionDAG &DAG) {
40702   if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG))
40703     return V;
40704 
40705   SDValue Opnd0, Opnd1;
40706   bool IsSubAdd;
40707   if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
40708     return SDValue();
40709 
40710   MVT VT = N->getSimpleValueType(0);
40711   SDLoc DL(N);
40712 
40713   // Try to generate X86ISD::FMADDSUB node here.
40714   SDValue Opnd2;
40715   if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
40716     unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
40717     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
40718   }
40719 
40720   if (IsSubAdd)
40721     return SDValue();
40722 
40723   // Do not generate X86ISD::ADDSUB node for 512-bit types even though
40724   // the ADDSUB idiom has been successfully recognized. There are no known
40725   // X86 targets with 512-bit ADDSUB instructions!
40726   if (VT.is512BitVector())
40727     return SDValue();
40728 
40729   // Do not generate X86ISD::ADDSUB node for FP16's vector types even though
40730   // the ADDSUB idiom has been successfully recognized. There are no known
40731   // X86 targets with FP16 ADDSUB instructions!
40732   if (VT.getVectorElementType() == MVT::f16)
40733     return SDValue();
40734 
40735   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
40736 }
40737 
40738 // We are looking for a shuffle where both sources are concatenated with undef
40739 // and have a width that is half of the output's width. AVX2 has VPERMD/Q, so
40740 // if we can express this as a single-source shuffle, that's preferable.
40741 static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
40742                                            const X86Subtarget &Subtarget) {
40743   if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N))
40744     return SDValue();
40745 
40746   EVT VT = N->getValueType(0);
40747 
40748   // We only care about shuffles of 128/256-bit vectors of 32/64-bit values.
40749   if (!VT.is128BitVector() && !VT.is256BitVector())
40750     return SDValue();
40751 
40752   if (VT.getVectorElementType() != MVT::i32 &&
40753       VT.getVectorElementType() != MVT::i64 &&
40754       VT.getVectorElementType() != MVT::f32 &&
40755       VT.getVectorElementType() != MVT::f64)
40756     return SDValue();
40757 
40758   SDValue N0 = N->getOperand(0);
40759   SDValue N1 = N->getOperand(1);
40760 
40761   // Check that both sources are concats with undef.
40762   if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
40763       N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
40764       N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
40765       !N1.getOperand(1).isUndef())
40766     return SDValue();
40767 
40768   // Construct the new shuffle mask. Elements from the first source retain their
40769   // index, but elements from the second source no longer need to skip an undef.
40770   SmallVector<int, 8> Mask;
40771   int NumElts = VT.getVectorNumElements();
40772 
40773   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
40774   for (int Elt : SVOp->getMask())
40775     Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2));
40776 
40777   SDLoc DL(N);
40778   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
40779                                N1.getOperand(0));
40780   return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
40781 }
40782 
40783 /// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
40784 /// low half of each source vector and does not set any high half elements in
40785 /// the destination vector, narrow the shuffle to half its original size.
40786 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
40787   EVT VT = Shuf->getValueType(0);
40788   if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0)))
40789     return SDValue();
40790   if (!VT.is256BitVector() && !VT.is512BitVector())
40791     return SDValue();
40792 
40793   // See if we can ignore all of the high elements of the shuffle.
40794   ArrayRef<int> Mask = Shuf->getMask();
40795   if (!isUndefUpperHalf(Mask))
40796     return SDValue();
40797 
40798   // Check if the shuffle mask accesses only the low half of each input vector
40799   // (half-index output is 0 or 2).
40800   int HalfIdx1, HalfIdx2;
40801   SmallVector<int, 8> HalfMask(Mask.size() / 2);
40802   if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2) ||
40803       (HalfIdx1 % 2 == 1) || (HalfIdx2 % 2 == 1))
40804     return SDValue();
40805 
40806   // Create a half-width shuffle to replace the unnecessarily wide shuffle.
40807   // The trick is knowing that all of the insert/extract are actually free
40808   // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle
40809   // of narrow inputs into a narrow output, and that is always cheaper than
40810   // the wide shuffle that we started with.
40811   return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0),
40812                                Shuf->getOperand(1), HalfMask, HalfIdx1,
40813                                HalfIdx2, false, DAG, /*UseConcat*/ true);
40814 }
40815 
40816 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
40817                               TargetLowering::DAGCombinerInfo &DCI,
40818                               const X86Subtarget &Subtarget) {
40819   if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N))
40820     if (SDValue V = narrowShuffle(Shuf, DAG))
40821       return V;
40822 
40823   // If we have legalized the vector types, look for blends of FADD and FSUB
40824   // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
40825   SDLoc dl(N);
40826   EVT VT = N->getValueType(0);
40827   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40828   if (TLI.isTypeLegal(VT) && !isSoftF16(VT, Subtarget))
40829     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
40830       return AddSub;
40831 
40832   // Attempt to combine into a vector load/broadcast.
40833   if (SDValue LD = combineToConsecutiveLoads(
40834           VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
40835     return LD;
40836 
40837   // For AVX2, we sometimes want to combine
40838   // (vector_shuffle <mask> (concat_vectors t1, undef)
40839   //                        (concat_vectors t2, undef))
40840   // Into:
40841   // (vector_shuffle <mask> (concat_vectors t1, t2), undef)
40842   // Since the latter can be efficiently lowered with VPERMD/VPERMQ
40843   if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget))
40844     return ShufConcat;
40845 
40846   if (isTargetShuffle(N->getOpcode())) {
40847     SDValue Op(N, 0);
40848     if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget))
40849       return Shuffle;
40850 
40851     // Try recursively combining arbitrary sequences of x86 shuffle
40852     // instructions into higher-order shuffles. We do this after combining
40853     // specific PSHUF instruction sequences into their minimal form so that we
40854     // can evaluate how many specialized shuffle instructions are involved in
40855     // a particular chain.
40856     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
40857       return Res;
40858 
40859     // Simplify source operands based on shuffle mask.
40860     // TODO - merge this into combineX86ShufflesRecursively.
40861     APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
40862     if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, DCI))
40863       return SDValue(N, 0);
40864 
40865     // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
40866     // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
40867     // Perform this after other shuffle combines to allow inner shuffles to be
40868     // combined away first.
40869     if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl))
40870       return BinOp;
40871   }
40872 
40873   return SDValue();
40874 }
40875 
40876 // Simplify variable target shuffle masks based on the demanded elements.
40877 // TODO: Handle DemandedBits in mask indices as well?
40878 bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle(
40879     SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
40880     TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const {
40881   // If we're demanding all elements don't bother trying to simplify the mask.
40882   unsigned NumElts = DemandedElts.getBitWidth();
40883   if (DemandedElts.isAllOnes())
40884     return false;
40885 
40886   SDValue Mask = Op.getOperand(MaskIndex);
40887   if (!Mask.hasOneUse())
40888     return false;
40889 
40890   // Attempt to generically simplify the variable shuffle mask.
40891   APInt MaskUndef, MaskZero;
40892   if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
40893                                  Depth + 1))
40894     return true;
40895 
40896   // Attempt to extract+simplify a (constant pool load) shuffle mask.
40897   // TODO: Support other types from getTargetShuffleMaskIndices?
40898   SDValue BC = peekThroughOneUseBitcasts(Mask);
40899   EVT BCVT = BC.getValueType();
40900   auto *Load = dyn_cast<LoadSDNode>(BC);
40901   if (!Load || !Load->getBasePtr().hasOneUse())
40902     return false;
40903 
40904   const Constant *C = getTargetConstantFromNode(Load);
40905   if (!C)
40906     return false;
40907 
40908   Type *CTy = C->getType();
40909   if (!CTy->isVectorTy() ||
40910       CTy->getPrimitiveSizeInBits() != Mask.getValueSizeInBits())
40911     return false;
40912 
40913   // Handle scaling for i64 elements on 32-bit targets.
40914   unsigned NumCstElts = cast<FixedVectorType>(CTy)->getNumElements();
40915   if (NumCstElts != NumElts && NumCstElts != (NumElts * 2))
40916     return false;
40917   unsigned Scale = NumCstElts / NumElts;
40918 
40919   // Simplify mask if we have an undemanded element that is not undef.
40920   bool Simplified = false;
40921   SmallVector<Constant *, 32> ConstVecOps;
40922   for (unsigned i = 0; i != NumCstElts; ++i) {
40923     Constant *Elt = C->getAggregateElement(i);
40924     if (!DemandedElts[i / Scale] && !isa<UndefValue>(Elt)) {
40925       ConstVecOps.push_back(UndefValue::get(Elt->getType()));
40926       Simplified = true;
40927       continue;
40928     }
40929     ConstVecOps.push_back(Elt);
40930   }
40931   if (!Simplified)
40932     return false;
40933 
40934   // Generate new constant pool entry + legalize immediately for the load.
40935   SDLoc DL(Op);
40936   SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
40937   SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
40938   SDValue NewMask = TLO.DAG.getLoad(
40939       BCVT, DL, TLO.DAG.getEntryNode(), LegalCV,
40940       MachinePointerInfo::getConstantPool(TLO.DAG.getMachineFunction()),
40941       Load->getAlign());
40942   return TLO.CombineTo(Mask, TLO.DAG.getBitcast(Mask.getValueType(), NewMask));
40943 }
40944 
40945 bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
40946     SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
40947     TargetLoweringOpt &TLO, unsigned Depth) const {
40948   int NumElts = DemandedElts.getBitWidth();
40949   unsigned Opc = Op.getOpcode();
40950   EVT VT = Op.getValueType();
40951 
40952   // Handle special case opcodes.
40953   switch (Opc) {
40954   case X86ISD::PMULDQ:
40955   case X86ISD::PMULUDQ: {
40956     APInt LHSUndef, LHSZero;
40957     APInt RHSUndef, RHSZero;
40958     SDValue LHS = Op.getOperand(0);
40959     SDValue RHS = Op.getOperand(1);
40960     if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
40961                                    Depth + 1))
40962       return true;
40963     if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
40964                                    Depth + 1))
40965       return true;
40966     // Multiply by zero.
40967     KnownZero = LHSZero | RHSZero;
40968     break;
40969   }
40970   case X86ISD::VPMADDWD: {
40971     APInt LHSUndef, LHSZero;
40972     APInt RHSUndef, RHSZero;
40973     SDValue LHS = Op.getOperand(0);
40974     SDValue RHS = Op.getOperand(1);
40975     APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, 2 * NumElts);
40976 
40977     if (SimplifyDemandedVectorElts(LHS, DemandedSrcElts, LHSUndef, LHSZero, TLO,
40978                                    Depth + 1))
40979       return true;
40980     if (SimplifyDemandedVectorElts(RHS, DemandedSrcElts, RHSUndef, RHSZero, TLO,
40981                                    Depth + 1))
40982       return true;
40983 
40984     // TODO: Multiply by zero.
40985 
40986     // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent.
40987     APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero;
40988     if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO,
40989                                    Depth + 1))
40990       return true;
40991     APInt DemandedRHSElts = DemandedSrcElts & ~LHSZero;
40992     if (SimplifyDemandedVectorElts(RHS, DemandedRHSElts, RHSUndef, RHSZero, TLO,
40993                                    Depth + 1))
40994       return true;
40995     break;
40996   }
40997   case X86ISD::PSADBW: {
40998     SDValue LHS = Op.getOperand(0);
40999     SDValue RHS = Op.getOperand(1);
41000     assert(VT.getScalarType() == MVT::i64 &&
41001            LHS.getValueType() == RHS.getValueType() &&
41002            LHS.getValueType().getScalarType() == MVT::i8 &&
41003            "Unexpected PSADBW types");
41004 
41005     // Aggressively peek through ops to get at the demanded elts.
41006     if (!DemandedElts.isAllOnes()) {
41007       unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
41008       APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
41009       SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts(
41010           LHS, DemandedSrcElts, TLO.DAG, Depth + 1);
41011       SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts(
41012           RHS, DemandedSrcElts, TLO.DAG, Depth + 1);
41013       if (NewLHS || NewRHS) {
41014         NewLHS = NewLHS ? NewLHS : LHS;
41015         NewRHS = NewRHS ? NewRHS : RHS;
41016         return TLO.CombineTo(
41017             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
41018       }
41019     }
41020     break;
41021   }
41022   case X86ISD::VSHL:
41023   case X86ISD::VSRL:
41024   case X86ISD::VSRA: {
41025     // We only need the bottom 64-bits of the (128-bit) shift amount.
41026     SDValue Amt = Op.getOperand(1);
41027     MVT AmtVT = Amt.getSimpleValueType();
41028     assert(AmtVT.is128BitVector() && "Unexpected value type");
41029 
41030     // If we reuse the shift amount just for sse shift amounts then we know that
41031     // only the bottom 64-bits are only ever used.
41032     bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) {
41033       unsigned UseOpc = Use->getOpcode();
41034       return (UseOpc == X86ISD::VSHL || UseOpc == X86ISD::VSRL ||
41035               UseOpc == X86ISD::VSRA) &&
41036              Use->getOperand(0) != Amt;
41037     });
41038 
41039     APInt AmtUndef, AmtZero;
41040     unsigned NumAmtElts = AmtVT.getVectorNumElements();
41041     APInt AmtElts = APInt::getLowBitsSet(NumAmtElts, NumAmtElts / 2);
41042     if (SimplifyDemandedVectorElts(Amt, AmtElts, AmtUndef, AmtZero, TLO,
41043                                    Depth + 1, AssumeSingleUse))
41044       return true;
41045     [[fallthrough]];
41046   }
41047   case X86ISD::VSHLI:
41048   case X86ISD::VSRLI:
41049   case X86ISD::VSRAI: {
41050     SDValue Src = Op.getOperand(0);
41051     APInt SrcUndef;
41052     if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
41053                                    Depth + 1))
41054       return true;
41055 
41056     // Fold shift(0,x) -> 0
41057     if (DemandedElts.isSubsetOf(KnownZero))
41058       return TLO.CombineTo(
41059           Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41060 
41061     // Aggressively peek through ops to get at the demanded elts.
41062     if (!DemandedElts.isAllOnes())
41063       if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
41064               Src, DemandedElts, TLO.DAG, Depth + 1))
41065         return TLO.CombineTo(
41066             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
41067     break;
41068   }
41069   case X86ISD::VPSHA:
41070   case X86ISD::VPSHL:
41071   case X86ISD::VSHLV:
41072   case X86ISD::VSRLV:
41073   case X86ISD::VSRAV: {
41074     APInt LHSUndef, LHSZero;
41075     APInt RHSUndef, RHSZero;
41076     SDValue LHS = Op.getOperand(0);
41077     SDValue RHS = Op.getOperand(1);
41078     if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
41079                                    Depth + 1))
41080       return true;
41081 
41082     // Fold shift(0,x) -> 0
41083     if (DemandedElts.isSubsetOf(LHSZero))
41084       return TLO.CombineTo(
41085           Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41086 
41087     if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
41088                                    Depth + 1))
41089       return true;
41090 
41091     KnownZero = LHSZero;
41092     break;
41093   }
41094   case X86ISD::KSHIFTL: {
41095     SDValue Src = Op.getOperand(0);
41096     auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
41097     assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
41098     unsigned ShiftAmt = Amt->getZExtValue();
41099 
41100     if (ShiftAmt == 0)
41101       return TLO.CombineTo(Op, Src);
41102 
41103     // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
41104     // single shift.  We can do this if the bottom bits (which are shifted
41105     // out) are never demanded.
41106     if (Src.getOpcode() == X86ISD::KSHIFTR) {
41107       if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) {
41108         unsigned C1 = Src.getConstantOperandVal(1);
41109         unsigned NewOpc = X86ISD::KSHIFTL;
41110         int Diff = ShiftAmt - C1;
41111         if (Diff < 0) {
41112           Diff = -Diff;
41113           NewOpc = X86ISD::KSHIFTR;
41114         }
41115 
41116         SDLoc dl(Op);
41117         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
41118         return TLO.CombineTo(
41119             Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
41120       }
41121     }
41122 
41123     APInt DemandedSrc = DemandedElts.lshr(ShiftAmt);
41124     if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
41125                                    Depth + 1))
41126       return true;
41127 
41128     KnownUndef <<= ShiftAmt;
41129     KnownZero <<= ShiftAmt;
41130     KnownZero.setLowBits(ShiftAmt);
41131     break;
41132   }
41133   case X86ISD::KSHIFTR: {
41134     SDValue Src = Op.getOperand(0);
41135     auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
41136     assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
41137     unsigned ShiftAmt = Amt->getZExtValue();
41138 
41139     if (ShiftAmt == 0)
41140       return TLO.CombineTo(Op, Src);
41141 
41142     // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
41143     // single shift.  We can do this if the top bits (which are shifted
41144     // out) are never demanded.
41145     if (Src.getOpcode() == X86ISD::KSHIFTL) {
41146       if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) {
41147         unsigned C1 = Src.getConstantOperandVal(1);
41148         unsigned NewOpc = X86ISD::KSHIFTR;
41149         int Diff = ShiftAmt - C1;
41150         if (Diff < 0) {
41151           Diff = -Diff;
41152           NewOpc = X86ISD::KSHIFTL;
41153         }
41154 
41155         SDLoc dl(Op);
41156         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
41157         return TLO.CombineTo(
41158             Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
41159       }
41160     }
41161 
41162     APInt DemandedSrc = DemandedElts.shl(ShiftAmt);
41163     if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
41164                                    Depth + 1))
41165       return true;
41166 
41167     KnownUndef.lshrInPlace(ShiftAmt);
41168     KnownZero.lshrInPlace(ShiftAmt);
41169     KnownZero.setHighBits(ShiftAmt);
41170     break;
41171   }
41172   case X86ISD::ANDNP: {
41173     // ANDNP = (~LHS & RHS);
41174     SDValue LHS = Op.getOperand(0);
41175     SDValue RHS = Op.getOperand(1);
41176 
41177     auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
41178       APInt UndefElts;
41179       SmallVector<APInt> EltBits;
41180       int NumElts = VT.getVectorNumElements();
41181       int EltSizeInBits = VT.getScalarSizeInBits();
41182       APInt OpBits = APInt::getAllOnes(EltSizeInBits);
41183       APInt OpElts = DemandedElts;
41184       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
41185                                         EltBits)) {
41186         OpBits.clearAllBits();
41187         OpElts.clearAllBits();
41188         for (int I = 0; I != NumElts; ++I) {
41189           if (!DemandedElts[I])
41190             continue;
41191           if (UndefElts[I]) {
41192             // We can't assume an undef src element gives an undef dst - the
41193             // other src might be zero.
41194             OpBits.setAllBits();
41195             OpElts.setBit(I);
41196           } else if ((Invert && !EltBits[I].isAllOnes()) ||
41197                      (!Invert && !EltBits[I].isZero())) {
41198             OpBits |= Invert ? ~EltBits[I] : EltBits[I];
41199             OpElts.setBit(I);
41200           }
41201         }
41202       }
41203       return std::make_pair(OpBits, OpElts);
41204     };
41205     APInt BitsLHS, EltsLHS;
41206     APInt BitsRHS, EltsRHS;
41207     std::tie(BitsLHS, EltsLHS) = GetDemandedMasks(RHS);
41208     std::tie(BitsRHS, EltsRHS) = GetDemandedMasks(LHS, true);
41209 
41210     APInt LHSUndef, LHSZero;
41211     APInt RHSUndef, RHSZero;
41212     if (SimplifyDemandedVectorElts(LHS, EltsLHS, LHSUndef, LHSZero, TLO,
41213                                    Depth + 1))
41214       return true;
41215     if (SimplifyDemandedVectorElts(RHS, EltsRHS, RHSUndef, RHSZero, TLO,
41216                                    Depth + 1))
41217       return true;
41218 
41219     if (!DemandedElts.isAllOnes()) {
41220       SDValue NewLHS = SimplifyMultipleUseDemandedBits(LHS, BitsLHS, EltsLHS,
41221                                                        TLO.DAG, Depth + 1);
41222       SDValue NewRHS = SimplifyMultipleUseDemandedBits(RHS, BitsRHS, EltsRHS,
41223                                                        TLO.DAG, Depth + 1);
41224       if (NewLHS || NewRHS) {
41225         NewLHS = NewLHS ? NewLHS : LHS;
41226         NewRHS = NewRHS ? NewRHS : RHS;
41227         return TLO.CombineTo(
41228             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
41229       }
41230     }
41231     break;
41232   }
41233   case X86ISD::CVTSI2P:
41234   case X86ISD::CVTUI2P: {
41235     SDValue Src = Op.getOperand(0);
41236     MVT SrcVT = Src.getSimpleValueType();
41237     APInt SrcUndef, SrcZero;
41238     APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
41239     if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
41240                                    Depth + 1))
41241       return true;
41242     break;
41243   }
41244   case X86ISD::PACKSS:
41245   case X86ISD::PACKUS: {
41246     SDValue N0 = Op.getOperand(0);
41247     SDValue N1 = Op.getOperand(1);
41248 
41249     APInt DemandedLHS, DemandedRHS;
41250     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
41251 
41252     APInt LHSUndef, LHSZero;
41253     if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
41254                                    Depth + 1))
41255       return true;
41256     APInt RHSUndef, RHSZero;
41257     if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
41258                                    Depth + 1))
41259       return true;
41260 
41261     // TODO - pass on known zero/undef.
41262 
41263     // Aggressively peek through ops to get at the demanded elts.
41264     // TODO - we should do this for all target/faux shuffles ops.
41265     if (!DemandedElts.isAllOnes()) {
41266       SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
41267                                                             TLO.DAG, Depth + 1);
41268       SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
41269                                                             TLO.DAG, Depth + 1);
41270       if (NewN0 || NewN1) {
41271         NewN0 = NewN0 ? NewN0 : N0;
41272         NewN1 = NewN1 ? NewN1 : N1;
41273         return TLO.CombineTo(Op,
41274                              TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
41275       }
41276     }
41277     break;
41278   }
41279   case X86ISD::HADD:
41280   case X86ISD::HSUB:
41281   case X86ISD::FHADD:
41282   case X86ISD::FHSUB: {
41283     SDValue N0 = Op.getOperand(0);
41284     SDValue N1 = Op.getOperand(1);
41285 
41286     APInt DemandedLHS, DemandedRHS;
41287     getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
41288 
41289     APInt LHSUndef, LHSZero;
41290     if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
41291                                    Depth + 1))
41292       return true;
41293     APInt RHSUndef, RHSZero;
41294     if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
41295                                    Depth + 1))
41296       return true;
41297 
41298     // TODO - pass on known zero/undef.
41299 
41300     // Aggressively peek through ops to get at the demanded elts.
41301     // TODO: Handle repeated operands.
41302     if (N0 != N1 && !DemandedElts.isAllOnes()) {
41303       SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
41304                                                             TLO.DAG, Depth + 1);
41305       SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
41306                                                             TLO.DAG, Depth + 1);
41307       if (NewN0 || NewN1) {
41308         NewN0 = NewN0 ? NewN0 : N0;
41309         NewN1 = NewN1 ? NewN1 : N1;
41310         return TLO.CombineTo(Op,
41311                              TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
41312       }
41313     }
41314     break;
41315   }
41316   case X86ISD::VTRUNC:
41317   case X86ISD::VTRUNCS:
41318   case X86ISD::VTRUNCUS: {
41319     SDValue Src = Op.getOperand(0);
41320     MVT SrcVT = Src.getSimpleValueType();
41321     APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
41322     APInt SrcUndef, SrcZero;
41323     if (SimplifyDemandedVectorElts(Src, DemandedSrc, SrcUndef, SrcZero, TLO,
41324                                    Depth + 1))
41325       return true;
41326     KnownZero = SrcZero.zextOrTrunc(NumElts);
41327     KnownUndef = SrcUndef.zextOrTrunc(NumElts);
41328     break;
41329   }
41330   case X86ISD::BLENDV: {
41331     APInt SelUndef, SelZero;
41332     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef,
41333                                    SelZero, TLO, Depth + 1))
41334       return true;
41335 
41336     // TODO: Use SelZero to adjust LHS/RHS DemandedElts.
41337     APInt LHSUndef, LHSZero;
41338     if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef,
41339                                    LHSZero, TLO, Depth + 1))
41340       return true;
41341 
41342     APInt RHSUndef, RHSZero;
41343     if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef,
41344                                    RHSZero, TLO, Depth + 1))
41345       return true;
41346 
41347     KnownZero = LHSZero & RHSZero;
41348     KnownUndef = LHSUndef & RHSUndef;
41349     break;
41350   }
41351   case X86ISD::VZEXT_MOVL: {
41352     // If upper demanded elements are already zero then we have nothing to do.
41353     SDValue Src = Op.getOperand(0);
41354     APInt DemandedUpperElts = DemandedElts;
41355     DemandedUpperElts.clearLowBits(1);
41356     if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1))
41357       return TLO.CombineTo(Op, Src);
41358     break;
41359   }
41360   case X86ISD::VZEXT_LOAD: {
41361     // If upper demanded elements are not demanded then simplify to a
41362     // scalar_to_vector(load()).
41363     MVT SVT = VT.getSimpleVT().getVectorElementType();
41364     if (DemandedElts == 1 && Op.getValue(1).use_empty() && isTypeLegal(SVT)) {
41365       SDLoc DL(Op);
41366       auto *Mem = cast<MemSDNode>(Op);
41367       SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(),
41368                                     Mem->getMemOperand());
41369       SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt);
41370       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec));
41371     }
41372     break;
41373   }
41374   case X86ISD::VBROADCAST: {
41375     SDValue Src = Op.getOperand(0);
41376     MVT SrcVT = Src.getSimpleValueType();
41377     if (!SrcVT.isVector())
41378       break;
41379     // Don't bother broadcasting if we just need the 0'th element.
41380     if (DemandedElts == 1) {
41381       if (Src.getValueType() != VT)
41382         Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
41383                              SDLoc(Op));
41384       return TLO.CombineTo(Op, Src);
41385     }
41386     APInt SrcUndef, SrcZero;
41387     APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0);
41388     if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
41389                                    Depth + 1))
41390       return true;
41391     // Aggressively peek through src to get at the demanded elt.
41392     // TODO - we should do this for all target/faux shuffles ops.
41393     if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
41394             Src, SrcElts, TLO.DAG, Depth + 1))
41395       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
41396     break;
41397   }
41398   case X86ISD::VPERMV:
41399     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 0, TLO,
41400                                                    Depth))
41401       return true;
41402     break;
41403   case X86ISD::PSHUFB:
41404   case X86ISD::VPERMV3:
41405   case X86ISD::VPERMILPV:
41406     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 1, TLO,
41407                                                    Depth))
41408       return true;
41409     break;
41410   case X86ISD::VPPERM:
41411   case X86ISD::VPERMIL2:
41412     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 2, TLO,
41413                                                    Depth))
41414       return true;
41415     break;
41416   }
41417 
41418   // For 256/512-bit ops that are 128/256-bit ops glued together, if we do not
41419   // demand any of the high elements, then narrow the op to 128/256-bits: e.g.
41420   // (op ymm0, ymm1) --> insert undef, (op xmm0, xmm1), 0
41421   if ((VT.is256BitVector() || VT.is512BitVector()) &&
41422       DemandedElts.lshr(NumElts / 2) == 0) {
41423     unsigned SizeInBits = VT.getSizeInBits();
41424     unsigned ExtSizeInBits = SizeInBits / 2;
41425 
41426     // See if 512-bit ops only use the bottom 128-bits.
41427     if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0)
41428       ExtSizeInBits = SizeInBits / 4;
41429 
41430     switch (Opc) {
41431       // Scalar broadcast.
41432     case X86ISD::VBROADCAST: {
41433       SDLoc DL(Op);
41434       SDValue Src = Op.getOperand(0);
41435       if (Src.getValueSizeInBits() > ExtSizeInBits)
41436         Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
41437       EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41438                                     ExtSizeInBits / VT.getScalarSizeInBits());
41439       SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src);
41440       return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
41441                                                TLO.DAG, DL, ExtSizeInBits));
41442     }
41443     case X86ISD::VBROADCAST_LOAD: {
41444       SDLoc DL(Op);
41445       auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
41446       EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41447                                     ExtSizeInBits / VT.getScalarSizeInBits());
41448       SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other);
41449       SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)};
41450       SDValue Bcst = TLO.DAG.getMemIntrinsicNode(
41451           X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(),
41452           MemIntr->getMemOperand());
41453       TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
41454                                            Bcst.getValue(1));
41455       return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
41456                                                TLO.DAG, DL, ExtSizeInBits));
41457     }
41458       // Subvector broadcast.
41459     case X86ISD::SUBV_BROADCAST_LOAD: {
41460       auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
41461       EVT MemVT = MemIntr->getMemoryVT();
41462       if (ExtSizeInBits == MemVT.getStoreSizeInBits()) {
41463         SDLoc DL(Op);
41464         SDValue Ld =
41465             TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(),
41466                             MemIntr->getBasePtr(), MemIntr->getMemOperand());
41467         TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
41468                                              Ld.getValue(1));
41469         return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0,
41470                                                  TLO.DAG, DL, ExtSizeInBits));
41471       } else if ((ExtSizeInBits % MemVT.getStoreSizeInBits()) == 0) {
41472         SDLoc DL(Op);
41473         EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41474                                       ExtSizeInBits / VT.getScalarSizeInBits());
41475         if (SDValue BcstLd =
41476                 getBROADCAST_LOAD(Opc, DL, BcstVT, MemVT, MemIntr, 0, TLO.DAG))
41477           return TLO.CombineTo(Op,
41478                                insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0,
41479                                                TLO.DAG, DL, ExtSizeInBits));
41480       }
41481       break;
41482     }
41483       // Byte shifts by immediate.
41484     case X86ISD::VSHLDQ:
41485     case X86ISD::VSRLDQ:
41486       // Shift by uniform.
41487     case X86ISD::VSHL:
41488     case X86ISD::VSRL:
41489     case X86ISD::VSRA:
41490       // Shift by immediate.
41491     case X86ISD::VSHLI:
41492     case X86ISD::VSRLI:
41493     case X86ISD::VSRAI: {
41494       SDLoc DL(Op);
41495       SDValue Ext0 =
41496           extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
41497       SDValue ExtOp =
41498           TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
41499       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41500       SDValue Insert =
41501           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41502       return TLO.CombineTo(Op, Insert);
41503     }
41504     case X86ISD::VPERMI: {
41505       // Simplify PERMPD/PERMQ to extract_subvector.
41506       // TODO: This should be done in shuffle combining.
41507       if (VT == MVT::v4f64 || VT == MVT::v4i64) {
41508         SmallVector<int, 4> Mask;
41509         DecodeVPERMMask(NumElts, Op.getConstantOperandVal(1), Mask);
41510         if (isUndefOrEqual(Mask[0], 2) && isUndefOrEqual(Mask[1], 3)) {
41511           SDLoc DL(Op);
41512           SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
41513           SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41514           SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
41515           return TLO.CombineTo(Op, Insert);
41516         }
41517       }
41518       break;
41519     }
41520     case X86ISD::VPERM2X128: {
41521       // Simplify VPERM2F128/VPERM2I128 to extract_subvector.
41522       SDLoc DL(Op);
41523       unsigned LoMask = Op.getConstantOperandVal(2) & 0xF;
41524       if (LoMask & 0x8)
41525         return TLO.CombineTo(
41526             Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL));
41527       unsigned EltIdx = (LoMask & 0x1) * (NumElts / 2);
41528       unsigned SrcIdx = (LoMask & 0x2) >> 1;
41529       SDValue ExtOp =
41530           extractSubVector(Op.getOperand(SrcIdx), EltIdx, TLO.DAG, DL, 128);
41531       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41532       SDValue Insert =
41533           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41534       return TLO.CombineTo(Op, Insert);
41535     }
41536       // Zero upper elements.
41537     case X86ISD::VZEXT_MOVL:
41538       // Target unary shuffles by immediate:
41539     case X86ISD::PSHUFD:
41540     case X86ISD::PSHUFLW:
41541     case X86ISD::PSHUFHW:
41542     case X86ISD::VPERMILPI:
41543       // (Non-Lane Crossing) Target Shuffles.
41544     case X86ISD::VPERMILPV:
41545     case X86ISD::VPERMIL2:
41546     case X86ISD::PSHUFB:
41547     case X86ISD::UNPCKL:
41548     case X86ISD::UNPCKH:
41549     case X86ISD::BLENDI:
41550       // Integer ops.
41551     case X86ISD::PACKSS:
41552     case X86ISD::PACKUS:
41553     case X86ISD::PCMPEQ:
41554     case X86ISD::PCMPGT:
41555     case X86ISD::PMULUDQ:
41556     case X86ISD::PMULDQ:
41557     case X86ISD::VSHLV:
41558     case X86ISD::VSRLV:
41559     case X86ISD::VSRAV:
41560       // Float ops.
41561     case X86ISD::FMAX:
41562     case X86ISD::FMIN:
41563     case X86ISD::FMAXC:
41564     case X86ISD::FMINC:
41565       // Horizontal Ops.
41566     case X86ISD::HADD:
41567     case X86ISD::HSUB:
41568     case X86ISD::FHADD:
41569     case X86ISD::FHSUB: {
41570       SDLoc DL(Op);
41571       SmallVector<SDValue, 4> Ops;
41572       for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
41573         SDValue SrcOp = Op.getOperand(i);
41574         EVT SrcVT = SrcOp.getValueType();
41575         assert((!SrcVT.isVector() || SrcVT.getSizeInBits() == SizeInBits) &&
41576                "Unsupported vector size");
41577         Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
41578                                                           ExtSizeInBits)
41579                                        : SrcOp);
41580       }
41581       MVT ExtVT = VT.getSimpleVT();
41582       ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
41583                                ExtSizeInBits / ExtVT.getScalarSizeInBits());
41584       SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
41585       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41586       SDValue Insert =
41587           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41588       return TLO.CombineTo(Op, Insert);
41589     }
41590     }
41591   }
41592 
41593   // For splats, unless we *only* demand the 0'th element,
41594   // stop attempts at simplification here, we aren't going to improve things,
41595   // this is better than any potential shuffle.
41596   if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
41597     return false;
41598 
41599   // Get target/faux shuffle mask.
41600   APInt OpUndef, OpZero;
41601   SmallVector<int, 64> OpMask;
41602   SmallVector<SDValue, 2> OpInputs;
41603   if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef,
41604                               OpZero, TLO.DAG, Depth, false))
41605     return false;
41606 
41607   // Shuffle inputs must be the same size as the result.
41608   if (OpMask.size() != (unsigned)NumElts ||
41609       llvm::any_of(OpInputs, [VT](SDValue V) {
41610         return VT.getSizeInBits() != V.getValueSizeInBits() ||
41611                !V.getValueType().isVector();
41612       }))
41613     return false;
41614 
41615   KnownZero = OpZero;
41616   KnownUndef = OpUndef;
41617 
41618   // Check if shuffle mask can be simplified to undef/zero/identity.
41619   int NumSrcs = OpInputs.size();
41620   for (int i = 0; i != NumElts; ++i)
41621     if (!DemandedElts[i])
41622       OpMask[i] = SM_SentinelUndef;
41623 
41624   if (isUndefInRange(OpMask, 0, NumElts)) {
41625     KnownUndef.setAllBits();
41626     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
41627   }
41628   if (isUndefOrZeroInRange(OpMask, 0, NumElts)) {
41629     KnownZero.setAllBits();
41630     return TLO.CombineTo(
41631         Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41632   }
41633   for (int Src = 0; Src != NumSrcs; ++Src)
41634     if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts))
41635       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src]));
41636 
41637   // Attempt to simplify inputs.
41638   for (int Src = 0; Src != NumSrcs; ++Src) {
41639     // TODO: Support inputs of different types.
41640     if (OpInputs[Src].getValueType() != VT)
41641       continue;
41642 
41643     int Lo = Src * NumElts;
41644     APInt SrcElts = APInt::getZero(NumElts);
41645     for (int i = 0; i != NumElts; ++i)
41646       if (DemandedElts[i]) {
41647         int M = OpMask[i] - Lo;
41648         if (0 <= M && M < NumElts)
41649           SrcElts.setBit(M);
41650       }
41651 
41652     // TODO - Propagate input undef/zero elts.
41653     APInt SrcUndef, SrcZero;
41654     if (SimplifyDemandedVectorElts(OpInputs[Src], SrcElts, SrcUndef, SrcZero,
41655                                    TLO, Depth + 1))
41656       return true;
41657   }
41658 
41659   // If we don't demand all elements, then attempt to combine to a simpler
41660   // shuffle.
41661   // We need to convert the depth to something combineX86ShufflesRecursively
41662   // can handle - so pretend its Depth == 0 again, and reduce the max depth
41663   // to match. This prevents combineX86ShuffleChain from returning a
41664   // combined shuffle that's the same as the original root, causing an
41665   // infinite loop.
41666   if (!DemandedElts.isAllOnes()) {
41667     assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range");
41668 
41669     SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef);
41670     for (int i = 0; i != NumElts; ++i)
41671       if (DemandedElts[i])
41672         DemandedMask[i] = i;
41673 
41674     SDValue NewShuffle = combineX86ShufflesRecursively(
41675         {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
41676         /*HasVarMask*/ false,
41677         /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
41678         Subtarget);
41679     if (NewShuffle)
41680       return TLO.CombineTo(Op, NewShuffle);
41681   }
41682 
41683   return false;
41684 }
41685 
41686 bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
41687     SDValue Op, const APInt &OriginalDemandedBits,
41688     const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
41689     unsigned Depth) const {
41690   EVT VT = Op.getValueType();
41691   unsigned BitWidth = OriginalDemandedBits.getBitWidth();
41692   unsigned Opc = Op.getOpcode();
41693   switch(Opc) {
41694   case X86ISD::VTRUNC: {
41695     KnownBits KnownOp;
41696     SDValue Src = Op.getOperand(0);
41697     MVT SrcVT = Src.getSimpleValueType();
41698 
41699     // Simplify the input, using demanded bit information.
41700     APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits());
41701     APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
41702     if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO, Depth + 1))
41703       return true;
41704     break;
41705   }
41706   case X86ISD::PMULDQ:
41707   case X86ISD::PMULUDQ: {
41708     // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
41709     KnownBits KnownLHS, KnownRHS;
41710     SDValue LHS = Op.getOperand(0);
41711     SDValue RHS = Op.getOperand(1);
41712 
41713     // Don't mask bits on 32-bit AVX512 targets which might lose a broadcast.
41714     // FIXME: Can we bound this better?
41715     APInt DemandedMask = APInt::getLowBitsSet(64, 32);
41716     APInt DemandedMaskLHS = APInt::getAllOnes(64);
41717     APInt DemandedMaskRHS = APInt::getAllOnes(64);
41718 
41719     bool Is32BitAVX512 = !Subtarget.is64Bit() && Subtarget.hasAVX512();
41720     if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(LHS))
41721       DemandedMaskLHS = DemandedMask;
41722     if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(RHS))
41723       DemandedMaskRHS = DemandedMask;
41724 
41725     if (SimplifyDemandedBits(LHS, DemandedMaskLHS, OriginalDemandedElts,
41726                              KnownLHS, TLO, Depth + 1))
41727       return true;
41728     if (SimplifyDemandedBits(RHS, DemandedMaskRHS, OriginalDemandedElts,
41729                              KnownRHS, TLO, Depth + 1))
41730       return true;
41731 
41732     // PMULUDQ(X,1) -> AND(X,(1<<32)-1) 'getZeroExtendInReg'.
41733     KnownRHS = KnownRHS.trunc(32);
41734     if (Opc == X86ISD::PMULUDQ && KnownRHS.isConstant() &&
41735         KnownRHS.getConstant().isOne()) {
41736       SDLoc DL(Op);
41737       SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT);
41738       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask));
41739     }
41740 
41741     // Aggressively peek through ops to get at the demanded low bits.
41742     SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
41743         LHS, DemandedMaskLHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
41744     SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
41745         RHS, DemandedMaskRHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
41746     if (DemandedLHS || DemandedRHS) {
41747       DemandedLHS = DemandedLHS ? DemandedLHS : LHS;
41748       DemandedRHS = DemandedRHS ? DemandedRHS : RHS;
41749       return TLO.CombineTo(
41750           Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS));
41751     }
41752     break;
41753   }
41754   case X86ISD::ANDNP: {
41755     KnownBits Known2;
41756     SDValue Op0 = Op.getOperand(0);
41757     SDValue Op1 = Op.getOperand(1);
41758 
41759     if (SimplifyDemandedBits(Op1, OriginalDemandedBits, OriginalDemandedElts,
41760                              Known, TLO, Depth + 1))
41761       return true;
41762     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41763 
41764     if (SimplifyDemandedBits(Op0, ~Known.Zero & OriginalDemandedBits,
41765                              OriginalDemandedElts, Known2, TLO, Depth + 1))
41766       return true;
41767     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
41768 
41769     // If the RHS is a constant, see if we can simplify it.
41770     if (ShrinkDemandedConstant(Op, ~Known2.One & OriginalDemandedBits,
41771                                OriginalDemandedElts, TLO))
41772       return true;
41773 
41774     // ANDNP = (~Op0 & Op1);
41775     Known.One &= Known2.Zero;
41776     Known.Zero |= Known2.One;
41777     break;
41778   }
41779   case X86ISD::VSHLI: {
41780     SDValue Op0 = Op.getOperand(0);
41781 
41782     unsigned ShAmt = Op.getConstantOperandVal(1);
41783     if (ShAmt >= BitWidth)
41784       break;
41785 
41786     APInt DemandedMask = OriginalDemandedBits.lshr(ShAmt);
41787 
41788     // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
41789     // single shift.  We can do this if the bottom bits (which are shifted
41790     // out) are never demanded.
41791     if (Op0.getOpcode() == X86ISD::VSRLI &&
41792         OriginalDemandedBits.countr_zero() >= ShAmt) {
41793       unsigned Shift2Amt = Op0.getConstantOperandVal(1);
41794       if (Shift2Amt < BitWidth) {
41795         int Diff = ShAmt - Shift2Amt;
41796         if (Diff == 0)
41797           return TLO.CombineTo(Op, Op0.getOperand(0));
41798 
41799         unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
41800         SDValue NewShift = TLO.DAG.getNode(
41801             NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
41802             TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
41803         return TLO.CombineTo(Op, NewShift);
41804       }
41805     }
41806 
41807     // If we are only demanding sign bits then we can use the shift source directly.
41808     unsigned NumSignBits =
41809         TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
41810     unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero();
41811     if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
41812       return TLO.CombineTo(Op, Op0);
41813 
41814     if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
41815                              TLO, Depth + 1))
41816       return true;
41817 
41818     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41819     Known.Zero <<= ShAmt;
41820     Known.One <<= ShAmt;
41821 
41822     // Low bits known zero.
41823     Known.Zero.setLowBits(ShAmt);
41824     return false;
41825   }
41826   case X86ISD::VSRLI: {
41827     unsigned ShAmt = Op.getConstantOperandVal(1);
41828     if (ShAmt >= BitWidth)
41829       break;
41830 
41831     APInt DemandedMask = OriginalDemandedBits << ShAmt;
41832 
41833     if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
41834                              OriginalDemandedElts, Known, TLO, Depth + 1))
41835       return true;
41836 
41837     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41838     Known.Zero.lshrInPlace(ShAmt);
41839     Known.One.lshrInPlace(ShAmt);
41840 
41841     // High bits known zero.
41842     Known.Zero.setHighBits(ShAmt);
41843     return false;
41844   }
41845   case X86ISD::VSRAI: {
41846     SDValue Op0 = Op.getOperand(0);
41847     SDValue Op1 = Op.getOperand(1);
41848 
41849     unsigned ShAmt = Op1->getAsZExtVal();
41850     if (ShAmt >= BitWidth)
41851       break;
41852 
41853     APInt DemandedMask = OriginalDemandedBits << ShAmt;
41854 
41855     // If we just want the sign bit then we don't need to shift it.
41856     if (OriginalDemandedBits.isSignMask())
41857       return TLO.CombineTo(Op, Op0);
41858 
41859     // fold (VSRAI (VSHLI X, C1), C1) --> X iff NumSignBits(X) > C1
41860     if (Op0.getOpcode() == X86ISD::VSHLI &&
41861         Op.getOperand(1) == Op0.getOperand(1)) {
41862       SDValue Op00 = Op0.getOperand(0);
41863       unsigned NumSignBits =
41864           TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts);
41865       if (ShAmt < NumSignBits)
41866         return TLO.CombineTo(Op, Op00);
41867     }
41868 
41869     // If any of the demanded bits are produced by the sign extension, we also
41870     // demand the input sign bit.
41871     if (OriginalDemandedBits.countl_zero() < ShAmt)
41872       DemandedMask.setSignBit();
41873 
41874     if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
41875                              TLO, Depth + 1))
41876       return true;
41877 
41878     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41879     Known.Zero.lshrInPlace(ShAmt);
41880     Known.One.lshrInPlace(ShAmt);
41881 
41882     // If the input sign bit is known to be zero, or if none of the top bits
41883     // are demanded, turn this into an unsigned shift right.
41884     if (Known.Zero[BitWidth - ShAmt - 1] ||
41885         OriginalDemandedBits.countl_zero() >= ShAmt)
41886       return TLO.CombineTo(
41887           Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));
41888 
41889     // High bits are known one.
41890     if (Known.One[BitWidth - ShAmt - 1])
41891       Known.One.setHighBits(ShAmt);
41892     return false;
41893   }
41894   case X86ISD::BLENDV: {
41895     SDValue Sel = Op.getOperand(0);
41896     SDValue LHS = Op.getOperand(1);
41897     SDValue RHS = Op.getOperand(2);
41898 
41899     APInt SignMask = APInt::getSignMask(BitWidth);
41900     SDValue NewSel = SimplifyMultipleUseDemandedBits(
41901         Sel, SignMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
41902     SDValue NewLHS = SimplifyMultipleUseDemandedBits(
41903         LHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
41904     SDValue NewRHS = SimplifyMultipleUseDemandedBits(
41905         RHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
41906 
41907     if (NewSel || NewLHS || NewRHS) {
41908       NewSel = NewSel ? NewSel : Sel;
41909       NewLHS = NewLHS ? NewLHS : LHS;
41910       NewRHS = NewRHS ? NewRHS : RHS;
41911       return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT,
41912                                                NewSel, NewLHS, NewRHS));
41913     }
41914     break;
41915   }
41916   case X86ISD::PEXTRB:
41917   case X86ISD::PEXTRW: {
41918     SDValue Vec = Op.getOperand(0);
41919     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
41920     MVT VecVT = Vec.getSimpleValueType();
41921     unsigned NumVecElts = VecVT.getVectorNumElements();
41922 
41923     if (CIdx && CIdx->getAPIntValue().ult(NumVecElts)) {
41924       unsigned Idx = CIdx->getZExtValue();
41925       unsigned VecBitWidth = VecVT.getScalarSizeInBits();
41926 
41927       // If we demand no bits from the vector then we must have demanded
41928       // bits from the implict zext - simplify to zero.
41929       APInt DemandedVecBits = OriginalDemandedBits.trunc(VecBitWidth);
41930       if (DemandedVecBits == 0)
41931         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
41932 
41933       APInt KnownUndef, KnownZero;
41934       APInt DemandedVecElts = APInt::getOneBitSet(NumVecElts, Idx);
41935       if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
41936                                      KnownZero, TLO, Depth + 1))
41937         return true;
41938 
41939       KnownBits KnownVec;
41940       if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
41941                                KnownVec, TLO, Depth + 1))
41942         return true;
41943 
41944       if (SDValue V = SimplifyMultipleUseDemandedBits(
41945               Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1))
41946         return TLO.CombineTo(
41947             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1)));
41948 
41949       Known = KnownVec.zext(BitWidth);
41950       return false;
41951     }
41952     break;
41953   }
41954   case X86ISD::PINSRB:
41955   case X86ISD::PINSRW: {
41956     SDValue Vec = Op.getOperand(0);
41957     SDValue Scl = Op.getOperand(1);
41958     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
41959     MVT VecVT = Vec.getSimpleValueType();
41960 
41961     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
41962       unsigned Idx = CIdx->getZExtValue();
41963       if (!OriginalDemandedElts[Idx])
41964         return TLO.CombineTo(Op, Vec);
41965 
41966       KnownBits KnownVec;
41967       APInt DemandedVecElts(OriginalDemandedElts);
41968       DemandedVecElts.clearBit(Idx);
41969       if (SimplifyDemandedBits(Vec, OriginalDemandedBits, DemandedVecElts,
41970                                KnownVec, TLO, Depth + 1))
41971         return true;
41972 
41973       KnownBits KnownScl;
41974       unsigned NumSclBits = Scl.getScalarValueSizeInBits();
41975       APInt DemandedSclBits = OriginalDemandedBits.zext(NumSclBits);
41976       if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
41977         return true;
41978 
41979       KnownScl = KnownScl.trunc(VecVT.getScalarSizeInBits());
41980       Known = KnownVec.intersectWith(KnownScl);
41981       return false;
41982     }
41983     break;
41984   }
41985   case X86ISD::PACKSS:
41986     // PACKSS saturates to MIN/MAX integer values. So if we just want the
41987     // sign bit then we can just ask for the source operands sign bit.
41988     // TODO - add known bits handling.
41989     if (OriginalDemandedBits.isSignMask()) {
41990       APInt DemandedLHS, DemandedRHS;
41991       getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS);
41992 
41993       KnownBits KnownLHS, KnownRHS;
41994       APInt SignMask = APInt::getSignMask(BitWidth * 2);
41995       if (SimplifyDemandedBits(Op.getOperand(0), SignMask, DemandedLHS,
41996                                KnownLHS, TLO, Depth + 1))
41997         return true;
41998       if (SimplifyDemandedBits(Op.getOperand(1), SignMask, DemandedRHS,
41999                                KnownRHS, TLO, Depth + 1))
42000         return true;
42001 
42002       // Attempt to avoid multi-use ops if we don't need anything from them.
42003       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
42004           Op.getOperand(0), SignMask, DemandedLHS, TLO.DAG, Depth + 1);
42005       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
42006           Op.getOperand(1), SignMask, DemandedRHS, TLO.DAG, Depth + 1);
42007       if (DemandedOp0 || DemandedOp1) {
42008         SDValue Op0 = DemandedOp0 ? DemandedOp0 : Op.getOperand(0);
42009         SDValue Op1 = DemandedOp1 ? DemandedOp1 : Op.getOperand(1);
42010         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1));
42011       }
42012     }
42013     // TODO - add general PACKSS/PACKUS SimplifyDemandedBits support.
42014     break;
42015   case X86ISD::VBROADCAST: {
42016     SDValue Src = Op.getOperand(0);
42017     MVT SrcVT = Src.getSimpleValueType();
42018     APInt DemandedElts = APInt::getOneBitSet(
42019         SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1, 0);
42020     if (SimplifyDemandedBits(Src, OriginalDemandedBits, DemandedElts, Known,
42021                              TLO, Depth + 1))
42022       return true;
42023     // If we don't need the upper bits, attempt to narrow the broadcast source.
42024     // Don't attempt this on AVX512 as it might affect broadcast folding.
42025     // TODO: Should we attempt this for i32/i16 splats? They tend to be slower.
42026     if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() &&
42027         OriginalDemandedBits.countl_zero() >= (BitWidth / 2) &&
42028         Src->hasOneUse()) {
42029       MVT NewSrcVT = MVT::getIntegerVT(BitWidth / 2);
42030       SDValue NewSrc =
42031           TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src);
42032       MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2);
42033       SDValue NewBcst =
42034           TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc);
42035       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst));
42036     }
42037     break;
42038   }
42039   case X86ISD::PCMPGT:
42040     // icmp sgt(0, R) == ashr(R, BitWidth-1).
42041     // iff we only need the sign bit then we can use R directly.
42042     if (OriginalDemandedBits.isSignMask() &&
42043         ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
42044       return TLO.CombineTo(Op, Op.getOperand(1));
42045     break;
42046   case X86ISD::MOVMSK: {
42047     SDValue Src = Op.getOperand(0);
42048     MVT SrcVT = Src.getSimpleValueType();
42049     unsigned SrcBits = SrcVT.getScalarSizeInBits();
42050     unsigned NumElts = SrcVT.getVectorNumElements();
42051 
42052     // If we don't need the sign bits at all just return zero.
42053     if (OriginalDemandedBits.countr_zero() >= NumElts)
42054       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
42055 
42056     // See if we only demand bits from the lower 128-bit vector.
42057     if (SrcVT.is256BitVector() &&
42058         OriginalDemandedBits.getActiveBits() <= (NumElts / 2)) {
42059       SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src));
42060       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
42061     }
42062 
42063     // Only demand the vector elements of the sign bits we need.
42064     APInt KnownUndef, KnownZero;
42065     APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
42066     if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
42067                                    TLO, Depth + 1))
42068       return true;
42069 
42070     Known.Zero = KnownZero.zext(BitWidth);
42071     Known.Zero.setHighBits(BitWidth - NumElts);
42072 
42073     // MOVMSK only uses the MSB from each vector element.
42074     KnownBits KnownSrc;
42075     APInt DemandedSrcBits = APInt::getSignMask(SrcBits);
42076     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, KnownSrc, TLO,
42077                              Depth + 1))
42078       return true;
42079 
42080     if (KnownSrc.One[SrcBits - 1])
42081       Known.One.setLowBits(NumElts);
42082     else if (KnownSrc.Zero[SrcBits - 1])
42083       Known.Zero.setLowBits(NumElts);
42084 
42085     // Attempt to avoid multi-use os if we don't need anything from it.
42086     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
42087             Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
42088       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
42089     return false;
42090   }
42091   case X86ISD::TESTP: {
42092     SDValue Op0 = Op.getOperand(0);
42093     SDValue Op1 = Op.getOperand(1);
42094     MVT OpVT = Op0.getSimpleValueType();
42095     assert((OpVT.getVectorElementType() == MVT::f32 ||
42096             OpVT.getVectorElementType() == MVT::f64) &&
42097            "Illegal vector type for X86ISD::TESTP");
42098 
42099     // TESTPS/TESTPD only demands the sign bits of ALL the elements.
42100     KnownBits KnownSrc;
42101     APInt SignMask = APInt::getSignMask(OpVT.getScalarSizeInBits());
42102     bool AssumeSingleUse = (Op0 == Op1) && Op->isOnlyUserOf(Op0.getNode());
42103     return SimplifyDemandedBits(Op0, SignMask, KnownSrc, TLO, Depth + 1,
42104                                 AssumeSingleUse) ||
42105            SimplifyDemandedBits(Op1, SignMask, KnownSrc, TLO, Depth + 1,
42106                                 AssumeSingleUse);
42107   }
42108   case X86ISD::BEXTR:
42109   case X86ISD::BEXTRI: {
42110     SDValue Op0 = Op.getOperand(0);
42111     SDValue Op1 = Op.getOperand(1);
42112 
42113     // Only bottom 16-bits of the control bits are required.
42114     if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
42115       // NOTE: SimplifyDemandedBits won't do this for constants.
42116       uint64_t Val1 = Cst1->getZExtValue();
42117       uint64_t MaskedVal1 = Val1 & 0xFFFF;
42118       if (Opc == X86ISD::BEXTR && MaskedVal1 != Val1) {
42119         SDLoc DL(Op);
42120         return TLO.CombineTo(
42121             Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0,
42122                                 TLO.DAG.getConstant(MaskedVal1, DL, VT)));
42123       }
42124 
42125       unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
42126       unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
42127 
42128       // If the length is 0, the result is 0.
42129       if (Length == 0) {
42130         Known.setAllZero();
42131         return false;
42132       }
42133 
42134       if ((Shift + Length) <= BitWidth) {
42135         APInt DemandedMask = APInt::getBitsSet(BitWidth, Shift, Shift + Length);
42136         if (SimplifyDemandedBits(Op0, DemandedMask, Known, TLO, Depth + 1))
42137           return true;
42138 
42139         Known = Known.extractBits(Length, Shift);
42140         Known = Known.zextOrTrunc(BitWidth);
42141         return false;
42142       }
42143     } else {
42144       assert(Opc == X86ISD::BEXTR && "Unexpected opcode!");
42145       KnownBits Known1;
42146       APInt DemandedMask(APInt::getLowBitsSet(BitWidth, 16));
42147       if (SimplifyDemandedBits(Op1, DemandedMask, Known1, TLO, Depth + 1))
42148         return true;
42149 
42150       // If the length is 0, replace with 0.
42151       KnownBits LengthBits = Known1.extractBits(8, 8);
42152       if (LengthBits.isZero())
42153         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
42154     }
42155 
42156     break;
42157   }
42158   case X86ISD::PDEP: {
42159     SDValue Op0 = Op.getOperand(0);
42160     SDValue Op1 = Op.getOperand(1);
42161 
42162     unsigned DemandedBitsLZ = OriginalDemandedBits.countl_zero();
42163     APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
42164 
42165     // If the demanded bits has leading zeroes, we don't demand those from the
42166     // mask.
42167     if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1))
42168       return true;
42169 
42170     // The number of possible 1s in the mask determines the number of LSBs of
42171     // operand 0 used. Undemanded bits from the mask don't matter so filter
42172     // them before counting.
42173     KnownBits Known2;
42174     uint64_t Count = (~Known.Zero & LoMask).popcount();
42175     APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count));
42176     if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1))
42177       return true;
42178 
42179     // Zeroes are retained from the mask, but not ones.
42180     Known.One.clearAllBits();
42181     // The result will have at least as many trailing zeros as the non-mask
42182     // operand since bits can only map to the same or higher bit position.
42183     Known.Zero.setLowBits(Known2.countMinTrailingZeros());
42184     return false;
42185   }
42186   }
42187 
42188   return TargetLowering::SimplifyDemandedBitsForTargetNode(
42189       Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
42190 }
42191 
42192 SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
42193     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
42194     SelectionDAG &DAG, unsigned Depth) const {
42195   int NumElts = DemandedElts.getBitWidth();
42196   unsigned Opc = Op.getOpcode();
42197   EVT VT = Op.getValueType();
42198 
42199   switch (Opc) {
42200   case X86ISD::PINSRB:
42201   case X86ISD::PINSRW: {
42202     // If we don't demand the inserted element, return the base vector.
42203     SDValue Vec = Op.getOperand(0);
42204     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
42205     MVT VecVT = Vec.getSimpleValueType();
42206     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
42207         !DemandedElts[CIdx->getZExtValue()])
42208       return Vec;
42209     break;
42210   }
42211   case X86ISD::VSHLI: {
42212     // If we are only demanding sign bits then we can use the shift source
42213     // directly.
42214     SDValue Op0 = Op.getOperand(0);
42215     unsigned ShAmt = Op.getConstantOperandVal(1);
42216     unsigned BitWidth = DemandedBits.getBitWidth();
42217     unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
42218     unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
42219     if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
42220       return Op0;
42221     break;
42222   }
42223   case X86ISD::VSRAI:
42224     // iff we only need the sign bit then we can use the source directly.
42225     // TODO: generalize where we only demand extended signbits.
42226     if (DemandedBits.isSignMask())
42227       return Op.getOperand(0);
42228     break;
42229   case X86ISD::PCMPGT:
42230     // icmp sgt(0, R) == ashr(R, BitWidth-1).
42231     // iff we only need the sign bit then we can use R directly.
42232     if (DemandedBits.isSignMask() &&
42233         ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
42234       return Op.getOperand(1);
42235     break;
42236   case X86ISD::BLENDV: {
42237     // BLENDV: Cond (MSB) ? LHS : RHS
42238     SDValue Cond = Op.getOperand(0);
42239     SDValue LHS = Op.getOperand(1);
42240     SDValue RHS = Op.getOperand(2);
42241 
42242     KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1);
42243     if (CondKnown.isNegative())
42244       return LHS;
42245     if (CondKnown.isNonNegative())
42246       return RHS;
42247     break;
42248   }
42249   case X86ISD::ANDNP: {
42250     // ANDNP = (~LHS & RHS);
42251     SDValue LHS = Op.getOperand(0);
42252     SDValue RHS = Op.getOperand(1);
42253 
42254     KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1);
42255     KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1);
42256 
42257     // If all of the demanded bits are known 0 on LHS and known 0 on RHS, then
42258     // the (inverted) LHS bits cannot contribute to the result of the 'andn' in
42259     // this context, so return RHS.
42260     if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.Zero))
42261       return RHS;
42262     break;
42263   }
42264   }
42265 
42266   APInt ShuffleUndef, ShuffleZero;
42267   SmallVector<int, 16> ShuffleMask;
42268   SmallVector<SDValue, 2> ShuffleOps;
42269   if (getTargetShuffleInputs(Op, DemandedElts, ShuffleOps, ShuffleMask,
42270                              ShuffleUndef, ShuffleZero, DAG, Depth, false)) {
42271     // If all the demanded elts are from one operand and are inline,
42272     // then we can use the operand directly.
42273     int NumOps = ShuffleOps.size();
42274     if (ShuffleMask.size() == (unsigned)NumElts &&
42275         llvm::all_of(ShuffleOps, [VT](SDValue V) {
42276           return VT.getSizeInBits() == V.getValueSizeInBits();
42277         })) {
42278 
42279       if (DemandedElts.isSubsetOf(ShuffleUndef))
42280         return DAG.getUNDEF(VT);
42281       if (DemandedElts.isSubsetOf(ShuffleUndef | ShuffleZero))
42282         return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op));
42283 
42284       // Bitmask that indicates which ops have only been accessed 'inline'.
42285       APInt IdentityOp = APInt::getAllOnes(NumOps);
42286       for (int i = 0; i != NumElts; ++i) {
42287         int M = ShuffleMask[i];
42288         if (!DemandedElts[i] || ShuffleUndef[i])
42289           continue;
42290         int OpIdx = M / NumElts;
42291         int EltIdx = M % NumElts;
42292         if (M < 0 || EltIdx != i) {
42293           IdentityOp.clearAllBits();
42294           break;
42295         }
42296         IdentityOp &= APInt::getOneBitSet(NumOps, OpIdx);
42297         if (IdentityOp == 0)
42298           break;
42299       }
42300       assert((IdentityOp == 0 || IdentityOp.popcount() == 1) &&
42301              "Multiple identity shuffles detected");
42302 
42303       if (IdentityOp != 0)
42304         return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]);
42305     }
42306   }
42307 
42308   return TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
42309       Op, DemandedBits, DemandedElts, DAG, Depth);
42310 }
42311 
42312 bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
42313     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
42314     bool PoisonOnly, unsigned Depth) const {
42315   unsigned EltsBits = Op.getScalarValueSizeInBits();
42316   unsigned NumElts = DemandedElts.getBitWidth();
42317 
42318   // TODO: Add more target shuffles.
42319   switch (Op.getOpcode()) {
42320   case X86ISD::PSHUFD:
42321   case X86ISD::VPERMILPI: {
42322     SmallVector<int, 8> Mask;
42323     DecodePSHUFMask(NumElts, EltsBits, Op.getConstantOperandVal(1), Mask);
42324 
42325     APInt DemandedSrcElts = APInt::getZero(NumElts);
42326     for (unsigned I = 0; I != NumElts; ++I)
42327       if (DemandedElts[I])
42328         DemandedSrcElts.setBit(Mask[I]);
42329 
42330     return DAG.isGuaranteedNotToBeUndefOrPoison(
42331         Op.getOperand(0), DemandedSrcElts, PoisonOnly, Depth + 1);
42332   }
42333   }
42334   return TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
42335       Op, DemandedElts, DAG, PoisonOnly, Depth);
42336 }
42337 
42338 bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
42339     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
42340     bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
42341 
42342   // TODO: Add more target shuffles.
42343   switch (Op.getOpcode()) {
42344   case X86ISD::PSHUFD:
42345   case X86ISD::VPERMILPI:
42346     return false;
42347   }
42348   return TargetLowering::canCreateUndefOrPoisonForTargetNode(
42349       Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
42350 }
42351 
42352 bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
42353                                                   const APInt &DemandedElts,
42354                                                   APInt &UndefElts,
42355                                                   const SelectionDAG &DAG,
42356                                                   unsigned Depth) const {
42357   unsigned NumElts = DemandedElts.getBitWidth();
42358   unsigned Opc = Op.getOpcode();
42359 
42360   switch (Opc) {
42361   case X86ISD::VBROADCAST:
42362   case X86ISD::VBROADCAST_LOAD:
42363     UndefElts = APInt::getZero(NumElts);
42364     return true;
42365   }
42366 
42367   return TargetLowering::isSplatValueForTargetNode(Op, DemandedElts, UndefElts,
42368                                                    DAG, Depth);
42369 }
42370 
42371 // Helper to peek through bitops/trunc/setcc to determine size of source vector.
42372 // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
42373 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
42374                                       bool AllowTruncate) {
42375   switch (Src.getOpcode()) {
42376   case ISD::TRUNCATE:
42377     if (!AllowTruncate)
42378       return false;
42379     [[fallthrough]];
42380   case ISD::SETCC:
42381     return Src.getOperand(0).getValueSizeInBits() == Size;
42382   case ISD::AND:
42383   case ISD::XOR:
42384   case ISD::OR:
42385     return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate) &&
42386            checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate);
42387   case ISD::SELECT:
42388   case ISD::VSELECT:
42389     return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
42390            checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate) &&
42391            checkBitcastSrcVectorSize(Src.getOperand(2), Size, AllowTruncate);
42392   case ISD::BUILD_VECTOR:
42393     return ISD::isBuildVectorAllZeros(Src.getNode()) ||
42394            ISD::isBuildVectorAllOnes(Src.getNode());
42395   }
42396   return false;
42397 }
42398 
42399 // Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
42400 static unsigned getAltBitOpcode(unsigned Opcode) {
42401   switch(Opcode) {
42402   case ISD::AND: return X86ISD::FAND;
42403   case ISD::OR: return X86ISD::FOR;
42404   case ISD::XOR: return X86ISD::FXOR;
42405   case X86ISD::ANDNP: return X86ISD::FANDN;
42406   }
42407   llvm_unreachable("Unknown bitwise opcode");
42408 }
42409 
42410 // Helper to adjust v4i32 MOVMSK expansion to work with SSE1-only targets.
42411 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
42412                                           const SDLoc &DL) {
42413   EVT SrcVT = Src.getValueType();
42414   if (SrcVT != MVT::v4i1)
42415     return SDValue();
42416 
42417   switch (Src.getOpcode()) {
42418   case ISD::SETCC:
42419     if (Src.getOperand(0).getValueType() == MVT::v4i32 &&
42420         ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode()) &&
42421         cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT) {
42422       SDValue Op0 = Src.getOperand(0);
42423       if (ISD::isNormalLoad(Op0.getNode()))
42424         return DAG.getBitcast(MVT::v4f32, Op0);
42425       if (Op0.getOpcode() == ISD::BITCAST &&
42426           Op0.getOperand(0).getValueType() == MVT::v4f32)
42427         return Op0.getOperand(0);
42428     }
42429     break;
42430   case ISD::AND:
42431   case ISD::XOR:
42432   case ISD::OR: {
42433     SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
42434     SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
42435     if (Op0 && Op1)
42436       return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,
42437                          Op1);
42438     break;
42439   }
42440   }
42441   return SDValue();
42442 }
42443 
42444 // Helper to push sign extension of vXi1 SETCC result through bitops.
42445 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
42446                                           SDValue Src, const SDLoc &DL) {
42447   switch (Src.getOpcode()) {
42448   case ISD::SETCC:
42449   case ISD::TRUNCATE:
42450   case ISD::BUILD_VECTOR:
42451     return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
42452   case ISD::AND:
42453   case ISD::XOR:
42454   case ISD::OR:
42455     return DAG.getNode(
42456         Src.getOpcode(), DL, SExtVT,
42457         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
42458         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
42459   case ISD::SELECT:
42460   case ISD::VSELECT:
42461     return DAG.getSelect(
42462         DL, SExtVT, Src.getOperand(0),
42463         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
42464         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
42465   }
42466   llvm_unreachable("Unexpected node type for vXi1 sign extension");
42467 }
42468 
42469 // Try to match patterns such as
42470 // (i16 bitcast (v16i1 x))
42471 // ->
42472 // (i16 movmsk (16i8 sext (v16i1 x)))
42473 // before the illegal vector is scalarized on subtargets that don't have legal
42474 // vxi1 types.
42475 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
42476                                   const SDLoc &DL,
42477                                   const X86Subtarget &Subtarget) {
42478   EVT SrcVT = Src.getValueType();
42479   if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
42480     return SDValue();
42481 
42482   // Recognize the IR pattern for the movmsk intrinsic under SSE1 before type
42483   // legalization destroys the v4i32 type.
42484   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2()) {
42485     if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
42486       V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,
42487                       DAG.getBitcast(MVT::v4f32, V));
42488       return DAG.getZExtOrTrunc(V, DL, VT);
42489     }
42490   }
42491 
42492   // If the input is a truncate from v16i8 or v32i8 go ahead and use a
42493   // movmskb even with avx512. This will be better than truncating to vXi1 and
42494   // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
42495   // vpcmpeqb/vpcmpgtb.
42496   bool PreferMovMsk = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
42497                       (Src.getOperand(0).getValueType() == MVT::v16i8 ||
42498                        Src.getOperand(0).getValueType() == MVT::v32i8 ||
42499                        Src.getOperand(0).getValueType() == MVT::v64i8);
42500 
42501   // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)) which can be handled
42502   // directly with vpmovmskb/vmovmskps/vmovmskpd.
42503   if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse() &&
42504       cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT &&
42505       ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) {
42506     EVT CmpVT = Src.getOperand(0).getValueType();
42507     EVT EltVT = CmpVT.getVectorElementType();
42508     if (CmpVT.getSizeInBits() <= 256 &&
42509         (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))
42510       PreferMovMsk = true;
42511   }
42512 
42513   // With AVX512 vxi1 types are legal and we prefer using k-regs.
42514   // MOVMSK is supported in SSE2 or later.
42515   if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !PreferMovMsk))
42516     return SDValue();
42517 
42518   // If the upper ops of a concatenation are undef, then try to bitcast the
42519   // lower op and extend.
42520   SmallVector<SDValue, 4> SubSrcOps;
42521   if (collectConcatOps(Src.getNode(), SubSrcOps, DAG) &&
42522       SubSrcOps.size() >= 2) {
42523     SDValue LowerOp = SubSrcOps[0];
42524     ArrayRef<SDValue> UpperOps(std::next(SubSrcOps.begin()), SubSrcOps.end());
42525     if (LowerOp.getOpcode() == ISD::SETCC &&
42526         all_of(UpperOps, [](SDValue Op) { return Op.isUndef(); })) {
42527       EVT SubVT = VT.getIntegerVT(
42528           *DAG.getContext(), LowerOp.getValueType().getVectorMinNumElements());
42529       if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) {
42530         EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
42531         return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V));
42532       }
42533     }
42534   }
42535 
42536   // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
42537   // v8f64. So all legal 128-bit and 256-bit vectors are covered except for
42538   // v8i16 and v16i16.
42539   // For these two cases, we can shuffle the upper element bytes to a
42540   // consecutive sequence at the start of the vector and treat the results as
42541   // v16i8 or v32i8, and for v16i8 this is the preferable solution. However,
42542   // for v16i16 this is not the case, because the shuffle is expensive, so we
42543   // avoid sign-extending to this type entirely.
42544   // For example, t0 := (v8i16 sext(v8i1 x)) needs to be shuffled as:
42545   // (v16i8 shuffle <0,2,4,6,8,10,12,14,u,u,...,u> (v16i8 bitcast t0), undef)
42546   MVT SExtVT;
42547   bool PropagateSExt = false;
42548   switch (SrcVT.getSimpleVT().SimpleTy) {
42549   default:
42550     return SDValue();
42551   case MVT::v2i1:
42552     SExtVT = MVT::v2i64;
42553     break;
42554   case MVT::v4i1:
42555     SExtVT = MVT::v4i32;
42556     // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
42557     // sign-extend to a 256-bit operation to avoid truncation.
42558     if (Subtarget.hasAVX() &&
42559         checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) {
42560       SExtVT = MVT::v4i64;
42561       PropagateSExt = true;
42562     }
42563     break;
42564   case MVT::v8i1:
42565     SExtVT = MVT::v8i16;
42566     // For cases such as (i8 bitcast (v8i1 setcc v8i32 v1, v2)),
42567     // sign-extend to a 256-bit operation to match the compare.
42568     // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
42569     // 256-bit because the shuffle is cheaper than sign extending the result of
42570     // the compare.
42571     if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) ||
42572                                checkBitcastSrcVectorSize(Src, 512, true))) {
42573       SExtVT = MVT::v8i32;
42574       PropagateSExt = true;
42575     }
42576     break;
42577   case MVT::v16i1:
42578     SExtVT = MVT::v16i8;
42579     // For the case (i16 bitcast (v16i1 setcc v16i16 v1, v2)),
42580     // it is not profitable to sign-extend to 256-bit because this will
42581     // require an extra cross-lane shuffle which is more expensive than
42582     // truncating the result of the compare to 128-bits.
42583     break;
42584   case MVT::v32i1:
42585     SExtVT = MVT::v32i8;
42586     break;
42587   case MVT::v64i1:
42588     // If we have AVX512F, but not AVX512BW and the input is truncated from
42589     // v64i8 checked earlier. Then split the input and make two pmovmskbs.
42590     if (Subtarget.hasAVX512()) {
42591       if (Subtarget.hasBWI())
42592         return SDValue();
42593       SExtVT = MVT::v64i8;
42594       break;
42595     }
42596     // Split if this is a <64 x i8> comparison result.
42597     if (checkBitcastSrcVectorSize(Src, 512, false)) {
42598       SExtVT = MVT::v64i8;
42599       break;
42600     }
42601     return SDValue();
42602   };
42603 
42604   SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
42605                             : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
42606 
42607   if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8 || SExtVT == MVT::v64i8) {
42608     V = getPMOVMSKB(DL, V, DAG, Subtarget);
42609   } else {
42610     if (SExtVT == MVT::v8i16) {
42611       V = widenSubVector(V, false, Subtarget, DAG, DL, 256);
42612       V = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, V);
42613     }
42614     V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
42615   }
42616 
42617   EVT IntVT =
42618       EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
42619   V = DAG.getZExtOrTrunc(V, DL, IntVT);
42620   return DAG.getBitcast(VT, V);
42621 }
42622 
42623 // Convert a vXi1 constant build vector to the same width scalar integer.
42624 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
42625   EVT SrcVT = Op.getValueType();
42626   assert(SrcVT.getVectorElementType() == MVT::i1 &&
42627          "Expected a vXi1 vector");
42628   assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
42629          "Expected a constant build vector");
42630 
42631   APInt Imm(SrcVT.getVectorNumElements(), 0);
42632   for (unsigned Idx = 0, e = Op.getNumOperands(); Idx < e; ++Idx) {
42633     SDValue In = Op.getOperand(Idx);
42634     if (!In.isUndef() && (In->getAsZExtVal() & 0x1))
42635       Imm.setBit(Idx);
42636   }
42637   EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth());
42638   return DAG.getConstant(Imm, SDLoc(Op), IntVT);
42639 }
42640 
42641 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
42642                                            TargetLowering::DAGCombinerInfo &DCI,
42643                                            const X86Subtarget &Subtarget) {
42644   assert(N->getOpcode() == ISD::BITCAST && "Expected a bitcast");
42645 
42646   if (!DCI.isBeforeLegalizeOps())
42647     return SDValue();
42648 
42649   // Only do this if we have k-registers.
42650   if (!Subtarget.hasAVX512())
42651     return SDValue();
42652 
42653   EVT DstVT = N->getValueType(0);
42654   SDValue Op = N->getOperand(0);
42655   EVT SrcVT = Op.getValueType();
42656 
42657   if (!Op.hasOneUse())
42658     return SDValue();
42659 
42660   // Look for logic ops.
42661   if (Op.getOpcode() != ISD::AND &&
42662       Op.getOpcode() != ISD::OR &&
42663       Op.getOpcode() != ISD::XOR)
42664     return SDValue();
42665 
42666   // Make sure we have a bitcast between mask registers and a scalar type.
42667   if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
42668         DstVT.isScalarInteger()) &&
42669       !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 &&
42670         SrcVT.isScalarInteger()))
42671     return SDValue();
42672 
42673   SDValue LHS = Op.getOperand(0);
42674   SDValue RHS = Op.getOperand(1);
42675 
42676   if (LHS.hasOneUse() && LHS.getOpcode() == ISD::BITCAST &&
42677       LHS.getOperand(0).getValueType() == DstVT)
42678     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0),
42679                        DAG.getBitcast(DstVT, RHS));
42680 
42681   if (RHS.hasOneUse() && RHS.getOpcode() == ISD::BITCAST &&
42682       RHS.getOperand(0).getValueType() == DstVT)
42683     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
42684                        DAG.getBitcast(DstVT, LHS), RHS.getOperand(0));
42685 
42686   // If the RHS is a vXi1 build vector, this is a good reason to flip too.
42687   // Most of these have to move a constant from the scalar domain anyway.
42688   if (ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) {
42689     RHS = combinevXi1ConstantToInteger(RHS, DAG);
42690     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
42691                        DAG.getBitcast(DstVT, LHS), RHS);
42692   }
42693 
42694   return SDValue();
42695 }
42696 
42697 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
42698                                     const X86Subtarget &Subtarget) {
42699   SDLoc DL(BV);
42700   unsigned NumElts = BV->getNumOperands();
42701   SDValue Splat = BV->getSplatValue();
42702 
42703   // Build MMX element from integer GPR or SSE float values.
42704   auto CreateMMXElement = [&](SDValue V) {
42705     if (V.isUndef())
42706       return DAG.getUNDEF(MVT::x86mmx);
42707     if (V.getValueType().isFloatingPoint()) {
42708       if (Subtarget.hasSSE1() && !isa<ConstantFPSDNode>(V)) {
42709         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
42710         V = DAG.getBitcast(MVT::v2i64, V);
42711         return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
42712       }
42713       V = DAG.getBitcast(MVT::i32, V);
42714     } else {
42715       V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
42716     }
42717     return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
42718   };
42719 
42720   // Convert build vector ops to MMX data in the bottom elements.
42721   SmallVector<SDValue, 8> Ops;
42722 
42723   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42724 
42725   // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element.
42726   if (Splat) {
42727     if (Splat.isUndef())
42728       return DAG.getUNDEF(MVT::x86mmx);
42729 
42730     Splat = CreateMMXElement(Splat);
42731 
42732     if (Subtarget.hasSSE1()) {
42733       // Unpack v8i8 to splat i8 elements to lowest 16-bits.
42734       if (NumElts == 8)
42735         Splat = DAG.getNode(
42736             ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
42737             DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL,
42738                                   TLI.getPointerTy(DAG.getDataLayout())),
42739             Splat, Splat);
42740 
42741       // Use PSHUFW to repeat 16-bit elements.
42742       unsigned ShufMask = (NumElts > 2 ? 0 : 0x44);
42743       return DAG.getNode(
42744           ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
42745           DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL,
42746                                 TLI.getPointerTy(DAG.getDataLayout())),
42747           Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
42748     }
42749     Ops.append(NumElts, Splat);
42750   } else {
42751     for (unsigned i = 0; i != NumElts; ++i)
42752       Ops.push_back(CreateMMXElement(BV->getOperand(i)));
42753   }
42754 
42755   // Use tree of PUNPCKLs to build up general MMX vector.
42756   while (Ops.size() > 1) {
42757     unsigned NumOps = Ops.size();
42758     unsigned IntrinOp =
42759         (NumOps == 2 ? Intrinsic::x86_mmx_punpckldq
42760                      : (NumOps == 4 ? Intrinsic::x86_mmx_punpcklwd
42761                                     : Intrinsic::x86_mmx_punpcklbw));
42762     SDValue Intrin = DAG.getTargetConstant(
42763         IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout()));
42764     for (unsigned i = 0; i != NumOps; i += 2)
42765       Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
42766                                Ops[i], Ops[i + 1]);
42767     Ops.resize(NumOps / 2);
42768   }
42769 
42770   return Ops[0];
42771 }
42772 
42773 // Recursive function that attempts to find if a bool vector node was originally
42774 // a vector/float/double that got truncated/extended/bitcast to/from a scalar
42775 // integer. If so, replace the scalar ops with bool vector equivalents back down
42776 // the chain.
42777 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
42778                                           SelectionDAG &DAG,
42779                                           const X86Subtarget &Subtarget) {
42780   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42781   unsigned Opc = V.getOpcode();
42782   switch (Opc) {
42783   case ISD::BITCAST: {
42784     // Bitcast from a vector/float/double, we can cheaply bitcast to VT.
42785     SDValue Src = V.getOperand(0);
42786     EVT SrcVT = Src.getValueType();
42787     if (SrcVT.isVector() || SrcVT.isFloatingPoint())
42788       return DAG.getBitcast(VT, Src);
42789     break;
42790   }
42791   case ISD::TRUNCATE: {
42792     // If we find a suitable source, a truncated scalar becomes a subvector.
42793     SDValue Src = V.getOperand(0);
42794     EVT NewSrcVT =
42795         EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits());
42796     if (TLI.isTypeLegal(NewSrcVT))
42797       if (SDValue N0 =
42798               combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
42799         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0,
42800                            DAG.getIntPtrConstant(0, DL));
42801     break;
42802   }
42803   case ISD::ANY_EXTEND:
42804   case ISD::ZERO_EXTEND: {
42805     // If we find a suitable source, an extended scalar becomes a subvector.
42806     SDValue Src = V.getOperand(0);
42807     EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
42808                                     Src.getScalarValueSizeInBits());
42809     if (TLI.isTypeLegal(NewSrcVT))
42810       if (SDValue N0 =
42811               combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
42812         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
42813                            Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT)
42814                                                   : DAG.getConstant(0, DL, VT),
42815                            N0, DAG.getIntPtrConstant(0, DL));
42816     break;
42817   }
42818   case ISD::OR: {
42819     // If we find suitable sources, we can just move an OR to the vector domain.
42820     SDValue Src0 = V.getOperand(0);
42821     SDValue Src1 = V.getOperand(1);
42822     if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
42823       if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget))
42824         return DAG.getNode(Opc, DL, VT, N0, N1);
42825     break;
42826   }
42827   case ISD::SHL: {
42828     // If we find a suitable source, a SHL becomes a KSHIFTL.
42829     SDValue Src0 = V.getOperand(0);
42830     if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) ||
42831         ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI()))
42832       break;
42833 
42834     if (auto *Amt = dyn_cast<ConstantSDNode>(V.getOperand(1)))
42835       if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
42836         return DAG.getNode(
42837             X86ISD::KSHIFTL, DL, VT, N0,
42838             DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));
42839     break;
42840   }
42841   }
42842   return SDValue();
42843 }
42844 
42845 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
42846                               TargetLowering::DAGCombinerInfo &DCI,
42847                               const X86Subtarget &Subtarget) {
42848   SDValue N0 = N->getOperand(0);
42849   EVT VT = N->getValueType(0);
42850   EVT SrcVT = N0.getValueType();
42851   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42852 
42853   // Try to match patterns such as
42854   // (i16 bitcast (v16i1 x))
42855   // ->
42856   // (i16 movmsk (16i8 sext (v16i1 x)))
42857   // before the setcc result is scalarized on subtargets that don't have legal
42858   // vxi1 types.
42859   if (DCI.isBeforeLegalize()) {
42860     SDLoc dl(N);
42861     if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
42862       return V;
42863 
42864     // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
42865     // type, widen both sides to avoid a trip through memory.
42866     if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&
42867         Subtarget.hasAVX512()) {
42868       N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);
42869       N0 = DAG.getBitcast(MVT::v8i1, N0);
42870       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,
42871                          DAG.getIntPtrConstant(0, dl));
42872     }
42873 
42874     // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
42875     // type, widen both sides to avoid a trip through memory.
42876     if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&
42877         Subtarget.hasAVX512()) {
42878       // Use zeros for the widening if we already have some zeroes. This can
42879       // allow SimplifyDemandedBits to remove scalar ANDs that may be down
42880       // stream of this.
42881       // FIXME: It might make sense to detect a concat_vectors with a mix of
42882       // zeroes and undef and turn it into insert_subvector for i1 vectors as
42883       // a separate combine. What we can't do is canonicalize the operands of
42884       // such a concat or we'll get into a loop with SimplifyDemandedBits.
42885       if (N0.getOpcode() == ISD::CONCAT_VECTORS) {
42886         SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
42887         if (ISD::isBuildVectorAllZeros(LastOp.getNode())) {
42888           SrcVT = LastOp.getValueType();
42889           unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
42890           SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end());
42891           Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT));
42892           N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
42893           N0 = DAG.getBitcast(MVT::i8, N0);
42894           return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
42895         }
42896       }
42897 
42898       unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
42899       SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
42900       Ops[0] = N0;
42901       N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
42902       N0 = DAG.getBitcast(MVT::i8, N0);
42903       return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
42904     }
42905   } else {
42906     // If we're bitcasting from iX to vXi1, see if the integer originally
42907     // began as a vXi1 and whether we can remove the bitcast entirely.
42908     if (VT.isVector() && VT.getScalarType() == MVT::i1 &&
42909         SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) {
42910       if (SDValue V =
42911               combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget))
42912         return V;
42913     }
42914   }
42915 
42916   // Look for (i8 (bitcast (v8i1 (extract_subvector (v16i1 X), 0)))) and
42917   // replace with (i8 (trunc (i16 (bitcast (v16i1 X))))). This can occur
42918   // due to insert_subvector legalization on KNL. By promoting the copy to i16
42919   // we can help with known bits propagation from the vXi1 domain to the
42920   // scalar domain.
42921   if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() &&
42922       !Subtarget.hasDQI() && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
42923       N0.getOperand(0).getValueType() == MVT::v16i1 &&
42924       isNullConstant(N0.getOperand(1)))
42925     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
42926                        DAG.getBitcast(MVT::i16, N0.getOperand(0)));
42927 
42928   // Canonicalize (bitcast (vbroadcast_load)) so that the output of the bitcast
42929   // and the vbroadcast_load are both integer or both fp. In some cases this
42930   // will remove the bitcast entirely.
42931   if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
42932        VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
42933     auto *BCast = cast<MemIntrinsicSDNode>(N0);
42934     unsigned SrcVTSize = SrcVT.getScalarSizeInBits();
42935     unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits();
42936     // Don't swap i8/i16 since don't have fp types that size.
42937     if (MemSize >= 32) {
42938       MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize)
42939                                        : MVT::getIntegerVT(MemSize);
42940       MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize)
42941                                         : MVT::getIntegerVT(SrcVTSize);
42942       LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());
42943 
42944       SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
42945       SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
42946       SDValue ResNode =
42947           DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
42948                                   MemVT, BCast->getMemOperand());
42949       DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
42950       return DAG.getBitcast(VT, ResNode);
42951     }
42952   }
42953 
42954   // Since MMX types are special and don't usually play with other vector types,
42955   // it's better to handle them early to be sure we emit efficient code by
42956   // avoiding store-load conversions.
42957   if (VT == MVT::x86mmx) {
42958     // Detect MMX constant vectors.
42959     APInt UndefElts;
42960     SmallVector<APInt, 1> EltBits;
42961     if (getTargetConstantBitsFromNode(N0, 64, UndefElts, EltBits)) {
42962       SDLoc DL(N0);
42963       // Handle zero-extension of i32 with MOVD.
42964       if (EltBits[0].countl_zero() >= 32)
42965         return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
42966                            DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
42967       // Else, bitcast to a double.
42968       // TODO - investigate supporting sext 32-bit immediates on x86_64.
42969       APFloat F64(APFloat::IEEEdouble(), EltBits[0]);
42970       return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
42971     }
42972 
42973     // Detect bitcasts to x86mmx low word.
42974     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
42975         (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) &&
42976         N0.getOperand(0).getValueType() == SrcVT.getScalarType()) {
42977       bool LowUndef = true, AllUndefOrZero = true;
42978       for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
42979         SDValue Op = N0.getOperand(i);
42980         LowUndef &= Op.isUndef() || (i >= e/2);
42981         AllUndefOrZero &= (Op.isUndef() || isNullConstant(Op));
42982       }
42983       if (AllUndefOrZero) {
42984         SDValue N00 = N0.getOperand(0);
42985         SDLoc dl(N00);
42986         N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)
42987                        : DAG.getZExtOrTrunc(N00, dl, MVT::i32);
42988         return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
42989       }
42990     }
42991 
42992     // Detect bitcasts of 64-bit build vectors and convert to a
42993     // MMX UNPCK/PSHUFW which takes MMX type inputs with the value in the
42994     // lowest element.
42995     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
42996         (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 ||
42997          SrcVT == MVT::v8i8))
42998       return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget);
42999 
43000     // Detect bitcasts between element or subvector extraction to x86mmx.
43001     if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
43002          N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) &&
43003         isNullConstant(N0.getOperand(1))) {
43004       SDValue N00 = N0.getOperand(0);
43005       if (N00.getValueType().is128BitVector())
43006         return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
43007                            DAG.getBitcast(MVT::v2i64, N00));
43008     }
43009 
43010     // Detect bitcasts from FP_TO_SINT to x86mmx.
43011     if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) {
43012       SDLoc DL(N0);
43013       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
43014                                 DAG.getUNDEF(MVT::v2i32));
43015       return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
43016                          DAG.getBitcast(MVT::v2i64, Res));
43017     }
43018   }
43019 
43020   // Try to remove a bitcast of constant vXi1 vector. We have to legalize
43021   // most of these to scalar anyway.
43022   if (Subtarget.hasAVX512() && VT.isScalarInteger() &&
43023       SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
43024       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
43025     return combinevXi1ConstantToInteger(N0, DAG);
43026   }
43027 
43028   if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
43029       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
43030       isa<ConstantSDNode>(N0)) {
43031     auto *C = cast<ConstantSDNode>(N0);
43032     if (C->isAllOnes())
43033       return DAG.getConstant(1, SDLoc(N0), VT);
43034     if (C->isZero())
43035       return DAG.getConstant(0, SDLoc(N0), VT);
43036   }
43037 
43038   // Look for MOVMSK that is maybe truncated and then bitcasted to vXi1.
43039   // Turn it into a sign bit compare that produces a k-register. This avoids
43040   // a trip through a GPR.
43041   if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
43042       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
43043       isPowerOf2_32(VT.getVectorNumElements())) {
43044     unsigned NumElts = VT.getVectorNumElements();
43045     SDValue Src = N0;
43046 
43047     // Peek through truncate.
43048     if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
43049       Src = N0.getOperand(0);
43050 
43051     if (Src.getOpcode() == X86ISD::MOVMSK && Src.hasOneUse()) {
43052       SDValue MovmskIn = Src.getOperand(0);
43053       MVT MovmskVT = MovmskIn.getSimpleValueType();
43054       unsigned MovMskElts = MovmskVT.getVectorNumElements();
43055 
43056       // We allow extra bits of the movmsk to be used since they are known zero.
43057       // We can't convert a VPMOVMSKB without avx512bw.
43058       if (MovMskElts <= NumElts &&
43059           (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) {
43060         EVT IntVT = EVT(MovmskVT).changeVectorElementTypeToInteger();
43061         MovmskIn = DAG.getBitcast(IntVT, MovmskIn);
43062         SDLoc dl(N);
43063         MVT CmpVT = MVT::getVectorVT(MVT::i1, MovMskElts);
43064         SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
43065                                    DAG.getConstant(0, dl, IntVT), ISD::SETLT);
43066         if (EVT(CmpVT) == VT)
43067           return Cmp;
43068 
43069         // Pad with zeroes up to original VT to replace the zeroes that were
43070         // being used from the MOVMSK.
43071         unsigned NumConcats = NumElts / MovMskElts;
43072         SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
43073         Ops[0] = Cmp;
43074         return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops);
43075       }
43076     }
43077   }
43078 
43079   // Try to remove bitcasts from input and output of mask arithmetic to
43080   // remove GPR<->K-register crossings.
43081   if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
43082     return V;
43083 
43084   // Convert a bitcasted integer logic operation that has one bitcasted
43085   // floating-point operand into a floating-point logic operation. This may
43086   // create a load of a constant, but that is cheaper than materializing the
43087   // constant in an integer register and transferring it to an SSE register or
43088   // transferring the SSE operand to integer register and back.
43089   unsigned FPOpcode;
43090   switch (N0.getOpcode()) {
43091     case ISD::AND: FPOpcode = X86ISD::FAND; break;
43092     case ISD::OR:  FPOpcode = X86ISD::FOR;  break;
43093     case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
43094     default: return SDValue();
43095   }
43096 
43097   // Check if we have a bitcast from another integer type as well.
43098   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
43099         (Subtarget.hasSSE2() && VT == MVT::f64) ||
43100         (Subtarget.hasFP16() && VT == MVT::f16) ||
43101         (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() &&
43102          TLI.isTypeLegal(VT))))
43103     return SDValue();
43104 
43105   SDValue LogicOp0 = N0.getOperand(0);
43106   SDValue LogicOp1 = N0.getOperand(1);
43107   SDLoc DL0(N0);
43108 
43109   // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y))
43110   if (N0.hasOneUse() && LogicOp0.getOpcode() == ISD::BITCAST &&
43111       LogicOp0.hasOneUse() && LogicOp0.getOperand(0).hasOneUse() &&
43112       LogicOp0.getOperand(0).getValueType() == VT &&
43113       !isa<ConstantSDNode>(LogicOp0.getOperand(0))) {
43114     SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
43115     unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
43116     return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);
43117   }
43118   // bitcast(logic(X, bitcast(Y))) --> logic'(bitcast(X), Y)
43119   if (N0.hasOneUse() && LogicOp1.getOpcode() == ISD::BITCAST &&
43120       LogicOp1.hasOneUse() && LogicOp1.getOperand(0).hasOneUse() &&
43121       LogicOp1.getOperand(0).getValueType() == VT &&
43122       !isa<ConstantSDNode>(LogicOp1.getOperand(0))) {
43123     SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
43124     unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
43125     return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);
43126   }
43127 
43128   return SDValue();
43129 }
43130 
43131 // (mul (zext a), (sext, b))
43132 static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
43133                          SDValue &Op1) {
43134   Op0 = Mul.getOperand(0);
43135   Op1 = Mul.getOperand(1);
43136 
43137   // The operand1 should be signed extend
43138   if (Op0.getOpcode() == ISD::SIGN_EXTEND)
43139     std::swap(Op0, Op1);
43140 
43141   auto IsFreeTruncation = [](SDValue &Op) -> bool {
43142     if ((Op.getOpcode() == ISD::ZERO_EXTEND ||
43143          Op.getOpcode() == ISD::SIGN_EXTEND) &&
43144         Op.getOperand(0).getScalarValueSizeInBits() <= 8)
43145       return true;
43146 
43147     auto *BV = dyn_cast<BuildVectorSDNode>(Op);
43148     return (BV && BV->isConstant());
43149   };
43150 
43151   // (dpbusd (zext a), (sext, b)). Since the first operand should be unsigned
43152   // value, we need to check Op0 is zero extended value. Op1 should be signed
43153   // value, so we just check the signed bits.
43154   if ((IsFreeTruncation(Op0) &&
43155        DAG.computeKnownBits(Op0).countMaxActiveBits() <= 8) &&
43156       (IsFreeTruncation(Op1) && DAG.ComputeMaxSignificantBits(Op1) <= 8))
43157     return true;
43158 
43159   return false;
43160 }
43161 
43162 // Given a ABS node, detect the following pattern:
43163 // (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
43164 // This is useful as it is the input into a SAD pattern.
43165 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
43166   SDValue AbsOp1 = Abs->getOperand(0);
43167   if (AbsOp1.getOpcode() != ISD::SUB)
43168     return false;
43169 
43170   Op0 = AbsOp1.getOperand(0);
43171   Op1 = AbsOp1.getOperand(1);
43172 
43173   // Check if the operands of the sub are zero-extended from vectors of i8.
43174   if (Op0.getOpcode() != ISD::ZERO_EXTEND ||
43175       Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||
43176       Op1.getOpcode() != ISD::ZERO_EXTEND ||
43177       Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8)
43178     return false;
43179 
43180   return true;
43181 }
43182 
43183 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
43184                               unsigned &LogBias, const SDLoc &DL,
43185                               const X86Subtarget &Subtarget) {
43186   // Extend or truncate to MVT::i8 first.
43187   MVT Vi8VT =
43188       MVT::getVectorVT(MVT::i8, LHS.getValueType().getVectorElementCount());
43189   LHS = DAG.getZExtOrTrunc(LHS, DL, Vi8VT);
43190   RHS = DAG.getSExtOrTrunc(RHS, DL, Vi8VT);
43191 
43192   // VPDPBUSD(<16 x i32>C, <16 x i8>A, <16 x i8>B). For each dst element
43193   // C[0] = C[0] + A[0]B[0] + A[1]B[1] + A[2]B[2] + A[3]B[3].
43194   // The src A, B element type is i8, but the dst C element type is i32.
43195   // When we calculate the reduce stage, we use src vector type vXi8 for it
43196   // so we need logbias 2 to avoid extra 2 stages.
43197   LogBias = 2;
43198 
43199   unsigned RegSize = std::max(128u, (unsigned)Vi8VT.getSizeInBits());
43200   if (Subtarget.hasVNNI() && !Subtarget.hasVLX())
43201     RegSize = std::max(512u, RegSize);
43202 
43203   // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
43204   // fill in the missing vector elements with 0.
43205   unsigned NumConcat = RegSize / Vi8VT.getSizeInBits();
43206   SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT));
43207   Ops[0] = LHS;
43208   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
43209   SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43210   Ops[0] = RHS;
43211   SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43212 
43213   // Actually build the DotProduct, split as 256/512 bits for
43214   // AVXVNNI/AVX512VNNI.
43215   auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
43216                        ArrayRef<SDValue> Ops) {
43217     MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
43218     return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
43219   };
43220   MVT DpVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
43221   SDValue Zero = DAG.getConstant(0, DL, DpVT);
43222 
43223   return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1},
43224                           DpBuilder, false);
43225 }
43226 
43227 // Given two zexts of <k x i8> to <k x i32>, create a PSADBW of the inputs
43228 // to these zexts.
43229 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
43230                             const SDValue &Zext1, const SDLoc &DL,
43231                             const X86Subtarget &Subtarget) {
43232   // Find the appropriate width for the PSADBW.
43233   EVT InVT = Zext0.getOperand(0).getValueType();
43234   unsigned RegSize = std::max(128u, (unsigned)InVT.getSizeInBits());
43235 
43236   // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
43237   // fill in the missing vector elements with 0.
43238   unsigned NumConcat = RegSize / InVT.getSizeInBits();
43239   SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
43240   Ops[0] = Zext0.getOperand(0);
43241   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
43242   SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43243   Ops[0] = Zext1.getOperand(0);
43244   SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43245 
43246   // Actually build the SAD, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
43247   auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
43248                           ArrayRef<SDValue> Ops) {
43249     MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
43250     return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
43251   };
43252   MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64);
43253   return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
43254                           PSADBWBuilder);
43255 }
43256 
43257 // Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
43258 // PHMINPOSUW.
43259 static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
43260                                       const X86Subtarget &Subtarget) {
43261   // Bail without SSE41.
43262   if (!Subtarget.hasSSE41())
43263     return SDValue();
43264 
43265   EVT ExtractVT = Extract->getValueType(0);
43266   if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
43267     return SDValue();
43268 
43269   // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
43270   ISD::NodeType BinOp;
43271   SDValue Src = DAG.matchBinOpReduction(
43272       Extract, BinOp, {ISD::SMAX, ISD::SMIN, ISD::UMAX, ISD::UMIN}, true);
43273   if (!Src)
43274     return SDValue();
43275 
43276   EVT SrcVT = Src.getValueType();
43277   EVT SrcSVT = SrcVT.getScalarType();
43278   if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
43279     return SDValue();
43280 
43281   SDLoc DL(Extract);
43282   SDValue MinPos = Src;
43283 
43284   // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
43285   while (SrcVT.getSizeInBits() > 128) {
43286     SDValue Lo, Hi;
43287     std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
43288     SrcVT = Lo.getValueType();
43289     MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
43290   }
43291   assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
43292           (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
43293          "Unexpected value type");
43294 
43295   // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
43296   // to flip the value accordingly.
43297   SDValue Mask;
43298   unsigned MaskEltsBits = ExtractVT.getSizeInBits();
43299   if (BinOp == ISD::SMAX)
43300     Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
43301   else if (BinOp == ISD::SMIN)
43302     Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
43303   else if (BinOp == ISD::UMAX)
43304     Mask = DAG.getAllOnesConstant(DL, SrcVT);
43305 
43306   if (Mask)
43307     MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
43308 
43309   // For v16i8 cases we need to perform UMIN on pairs of byte elements,
43310   // shuffling each upper element down and insert zeros. This means that the
43311   // v16i8 UMIN will leave the upper element as zero, performing zero-extension
43312   // ready for the PHMINPOS.
43313   if (ExtractVT == MVT::i8) {
43314     SDValue Upper = DAG.getVectorShuffle(
43315         SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
43316         {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
43317     MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
43318   }
43319 
43320   // Perform the PHMINPOS on a v8i16 vector,
43321   MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
43322   MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
43323   MinPos = DAG.getBitcast(SrcVT, MinPos);
43324 
43325   if (Mask)
43326     MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
43327 
43328   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
43329                      DAG.getIntPtrConstant(0, DL));
43330 }
43331 
43332 // Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
43333 static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
43334                                          const X86Subtarget &Subtarget) {
43335   // Bail without SSE2.
43336   if (!Subtarget.hasSSE2())
43337     return SDValue();
43338 
43339   EVT ExtractVT = Extract->getValueType(0);
43340   unsigned BitWidth = ExtractVT.getSizeInBits();
43341   if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 &&
43342       ExtractVT != MVT::i8 && ExtractVT != MVT::i1)
43343     return SDValue();
43344 
43345   // Check for OR(any_of)/AND(all_of)/XOR(parity) horizontal reduction patterns.
43346   ISD::NodeType BinOp;
43347   SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
43348   if (!Match && ExtractVT == MVT::i1)
43349     Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR});
43350   if (!Match)
43351     return SDValue();
43352 
43353   // EXTRACT_VECTOR_ELT can require implicit extension of the vector element
43354   // which we can't support here for now.
43355   if (Match.getScalarValueSizeInBits() != BitWidth)
43356     return SDValue();
43357 
43358   SDValue Movmsk;
43359   SDLoc DL(Extract);
43360   EVT MatchVT = Match.getValueType();
43361   unsigned NumElts = MatchVT.getVectorNumElements();
43362   unsigned MaxElts = Subtarget.hasInt256() ? 32 : 16;
43363   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43364   LLVMContext &Ctx = *DAG.getContext();
43365 
43366   if (ExtractVT == MVT::i1) {
43367     // Special case for (pre-legalization) vXi1 reductions.
43368     if (NumElts > 64 || !isPowerOf2_32(NumElts))
43369       return SDValue();
43370     if (Match.getOpcode() == ISD::SETCC) {
43371       ISD::CondCode CC = cast<CondCodeSDNode>(Match.getOperand(2))->get();
43372       if ((BinOp == ISD::AND && CC == ISD::CondCode::SETEQ) ||
43373           (BinOp == ISD::OR && CC == ISD::CondCode::SETNE)) {
43374         // For all_of(setcc(x,y,eq)) - use (iX)x == (iX)y.
43375         // For any_of(setcc(x,y,ne)) - use (iX)x != (iX)y.
43376         X86::CondCode X86CC;
43377         SDValue LHS = DAG.getFreeze(Match.getOperand(0));
43378         SDValue RHS = DAG.getFreeze(Match.getOperand(1));
43379         APInt Mask = APInt::getAllOnes(LHS.getScalarValueSizeInBits());
43380         if (SDValue V = LowerVectorAllEqual(DL, LHS, RHS, CC, Mask, Subtarget,
43381                                             DAG, X86CC))
43382           return DAG.getNode(ISD::TRUNCATE, DL, ExtractVT,
43383                              getSETCC(X86CC, V, DL, DAG));
43384       }
43385     }
43386     if (TLI.isTypeLegal(MatchVT)) {
43387       // If this is a legal AVX512 predicate type then we can just bitcast.
43388       EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
43389       Movmsk = DAG.getBitcast(MovmskVT, Match);
43390     } else {
43391       // Use combineBitcastvxi1 to create the MOVMSK.
43392       while (NumElts > MaxElts) {
43393         SDValue Lo, Hi;
43394         std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
43395         Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
43396         NumElts /= 2;
43397       }
43398       EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
43399       Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
43400     }
43401     if (!Movmsk)
43402       return SDValue();
43403     Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
43404   } else {
43405     // FIXME: Better handling of k-registers or 512-bit vectors?
43406     unsigned MatchSizeInBits = Match.getValueSizeInBits();
43407     if (!(MatchSizeInBits == 128 ||
43408           (MatchSizeInBits == 256 && Subtarget.hasAVX())))
43409       return SDValue();
43410 
43411     // Make sure this isn't a vector of 1 element. The perf win from using
43412     // MOVMSK diminishes with less elements in the reduction, but it is
43413     // generally better to get the comparison over to the GPRs as soon as
43414     // possible to reduce the number of vector ops.
43415     if (Match.getValueType().getVectorNumElements() < 2)
43416       return SDValue();
43417 
43418     // Check that we are extracting a reduction of all sign bits.
43419     if (DAG.ComputeNumSignBits(Match) != BitWidth)
43420       return SDValue();
43421 
43422     if (MatchSizeInBits == 256 && BitWidth < 32 && !Subtarget.hasInt256()) {
43423       SDValue Lo, Hi;
43424       std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
43425       Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
43426       MatchSizeInBits = Match.getValueSizeInBits();
43427     }
43428 
43429     // For 32/64 bit comparisons use MOVMSKPS/MOVMSKPD, else PMOVMSKB.
43430     MVT MaskSrcVT;
43431     if (64 == BitWidth || 32 == BitWidth)
43432       MaskSrcVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth),
43433                                    MatchSizeInBits / BitWidth);
43434     else
43435       MaskSrcVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8);
43436 
43437     SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
43438     Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
43439     NumElts = MaskSrcVT.getVectorNumElements();
43440   }
43441   assert((NumElts <= 32 || NumElts == 64) &&
43442          "Not expecting more than 64 elements");
43443 
43444   MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32;
43445   if (BinOp == ISD::XOR) {
43446     // parity -> (PARITY(MOVMSK X))
43447     SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk);
43448     return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
43449   }
43450 
43451   SDValue CmpC;
43452   ISD::CondCode CondCode;
43453   if (BinOp == ISD::OR) {
43454     // any_of -> MOVMSK != 0
43455     CmpC = DAG.getConstant(0, DL, CmpVT);
43456     CondCode = ISD::CondCode::SETNE;
43457   } else {
43458     // all_of -> MOVMSK == ((1 << NumElts) - 1)
43459     CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts),
43460                            DL, CmpVT);
43461     CondCode = ISD::CondCode::SETEQ;
43462   }
43463 
43464   // The setcc produces an i8 of 0/1, so extend that to the result width and
43465   // negate to get the final 0/-1 mask value.
43466   EVT SetccVT = TLI.getSetCCResultType(DAG.getDataLayout(), Ctx, CmpVT);
43467   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
43468   SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
43469   SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
43470   return DAG.getNode(ISD::SUB, DL, ExtractVT, Zero, Zext);
43471 }
43472 
43473 static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
43474                                       const X86Subtarget &Subtarget) {
43475   if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI())
43476     return SDValue();
43477 
43478   EVT ExtractVT = Extract->getValueType(0);
43479   // Verify the type we're extracting is i32, as the output element type of
43480   // vpdpbusd is i32.
43481   if (ExtractVT != MVT::i32)
43482     return SDValue();
43483 
43484   EVT VT = Extract->getOperand(0).getValueType();
43485   if (!isPowerOf2_32(VT.getVectorNumElements()))
43486     return SDValue();
43487 
43488   // Match shuffle + add pyramid.
43489   ISD::NodeType BinOp;
43490   SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
43491 
43492   // We can't combine to vpdpbusd for zext, because each of the 4 multiplies
43493   // done by vpdpbusd compute a signed 16-bit product that will be sign extended
43494   // before adding into the accumulator.
43495   // TODO:
43496   // We also need to verify that the multiply has at least 2x the number of bits
43497   // of the input. We shouldn't match
43498   // (sign_extend (mul (vXi9 (zext (vXi8 X))), (vXi9 (zext (vXi8 Y)))).
43499   // if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND))
43500   //   Root = Root.getOperand(0);
43501 
43502   // If there was a match, we want Root to be a mul.
43503   if (!Root || Root.getOpcode() != ISD::MUL)
43504     return SDValue();
43505 
43506   // Check whether we have an extend and mul pattern
43507   SDValue LHS, RHS;
43508   if (!detectExtMul(DAG, Root, LHS, RHS))
43509     return SDValue();
43510 
43511   // Create the dot product instruction.
43512   SDLoc DL(Extract);
43513   unsigned StageBias;
43514   SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
43515 
43516   // If the original vector was wider than 4 elements, sum over the results
43517   // in the DP vector.
43518   unsigned Stages = Log2_32(VT.getVectorNumElements());
43519   EVT DpVT = DP.getValueType();
43520 
43521   if (Stages > StageBias) {
43522     unsigned DpElems = DpVT.getVectorNumElements();
43523 
43524     for (unsigned i = Stages - StageBias; i > 0; --i) {
43525       SmallVector<int, 16> Mask(DpElems, -1);
43526       for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
43527         Mask[j] = MaskEnd + j;
43528 
43529       SDValue Shuffle =
43530           DAG.getVectorShuffle(DpVT, DL, DP, DAG.getUNDEF(DpVT), Mask);
43531       DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle);
43532     }
43533   }
43534 
43535   // Return the lowest ExtractSizeInBits bits.
43536   EVT ResVT =
43537       EVT::getVectorVT(*DAG.getContext(), ExtractVT,
43538                        DpVT.getSizeInBits() / ExtractVT.getSizeInBits());
43539   DP = DAG.getBitcast(ResVT, DP);
43540   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
43541                      Extract->getOperand(1));
43542 }
43543 
43544 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
43545                                       const X86Subtarget &Subtarget) {
43546   // PSADBW is only supported on SSE2 and up.
43547   if (!Subtarget.hasSSE2())
43548     return SDValue();
43549 
43550   EVT ExtractVT = Extract->getValueType(0);
43551   // Verify the type we're extracting is either i32 or i64.
43552   // FIXME: Could support other types, but this is what we have coverage for.
43553   if (ExtractVT != MVT::i32 && ExtractVT != MVT::i64)
43554     return SDValue();
43555 
43556   EVT VT = Extract->getOperand(0).getValueType();
43557   if (!isPowerOf2_32(VT.getVectorNumElements()))
43558     return SDValue();
43559 
43560   // Match shuffle + add pyramid.
43561   ISD::NodeType BinOp;
43562   SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
43563 
43564   // The operand is expected to be zero extended from i8
43565   // (verified in detectZextAbsDiff).
43566   // In order to convert to i64 and above, additional any/zero/sign
43567   // extend is expected.
43568   // The zero extend from 32 bit has no mathematical effect on the result.
43569   // Also the sign extend is basically zero extend
43570   // (extends the sign bit which is zero).
43571   // So it is correct to skip the sign/zero extend instruction.
43572   if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND ||
43573                Root.getOpcode() == ISD::ZERO_EXTEND ||
43574                Root.getOpcode() == ISD::ANY_EXTEND))
43575     Root = Root.getOperand(0);
43576 
43577   // If there was a match, we want Root to be a select that is the root of an
43578   // abs-diff pattern.
43579   if (!Root || Root.getOpcode() != ISD::ABS)
43580     return SDValue();
43581 
43582   // Check whether we have an abs-diff pattern feeding into the select.
43583   SDValue Zext0, Zext1;
43584   if (!detectZextAbsDiff(Root, Zext0, Zext1))
43585     return SDValue();
43586 
43587   // Create the SAD instruction.
43588   SDLoc DL(Extract);
43589   SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
43590 
43591   // If the original vector was wider than 8 elements, sum over the results
43592   // in the SAD vector.
43593   unsigned Stages = Log2_32(VT.getVectorNumElements());
43594   EVT SadVT = SAD.getValueType();
43595   if (Stages > 3) {
43596     unsigned SadElems = SadVT.getVectorNumElements();
43597 
43598     for(unsigned i = Stages - 3; i > 0; --i) {
43599       SmallVector<int, 16> Mask(SadElems, -1);
43600       for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
43601         Mask[j] = MaskEnd + j;
43602 
43603       SDValue Shuffle =
43604           DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
43605       SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
43606     }
43607   }
43608 
43609   unsigned ExtractSizeInBits = ExtractVT.getSizeInBits();
43610   // Return the lowest ExtractSizeInBits bits.
43611   EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
43612                                SadVT.getSizeInBits() / ExtractSizeInBits);
43613   SAD = DAG.getBitcast(ResVT, SAD);
43614   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
43615                      Extract->getOperand(1));
43616 }
43617 
43618 // Attempt to peek through a target shuffle and extract the scalar from the
43619 // source.
43620 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
43621                                          TargetLowering::DAGCombinerInfo &DCI,
43622                                          const X86Subtarget &Subtarget) {
43623   if (DCI.isBeforeLegalizeOps())
43624     return SDValue();
43625 
43626   SDLoc dl(N);
43627   SDValue Src = N->getOperand(0);
43628   SDValue Idx = N->getOperand(1);
43629 
43630   EVT VT = N->getValueType(0);
43631   EVT SrcVT = Src.getValueType();
43632   EVT SrcSVT = SrcVT.getVectorElementType();
43633   unsigned SrcEltBits = SrcSVT.getSizeInBits();
43634   unsigned NumSrcElts = SrcVT.getVectorNumElements();
43635 
43636   // Don't attempt this for boolean mask vectors or unknown extraction indices.
43637   if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))
43638     return SDValue();
43639 
43640   const APInt &IdxC = N->getConstantOperandAPInt(1);
43641   if (IdxC.uge(NumSrcElts))
43642     return SDValue();
43643 
43644   SDValue SrcBC = peekThroughBitcasts(Src);
43645 
43646   // Handle extract(bitcast(broadcast(scalar_value))).
43647   if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
43648     SDValue SrcOp = SrcBC.getOperand(0);
43649     EVT SrcOpVT = SrcOp.getValueType();
43650     if (SrcOpVT.isScalarInteger() && VT.isInteger() &&
43651         (SrcOpVT.getSizeInBits() % SrcEltBits) == 0) {
43652       unsigned Scale = SrcOpVT.getSizeInBits() / SrcEltBits;
43653       unsigned Offset = IdxC.urem(Scale) * SrcEltBits;
43654       // TODO support non-zero offsets.
43655       if (Offset == 0) {
43656         SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType());
43657         SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT);
43658         return SrcOp;
43659       }
43660     }
43661   }
43662 
43663   // If we're extracting a single element from a broadcast load and there are
43664   // no other users, just create a single load.
43665   if (SrcBC.getOpcode() == X86ISD::VBROADCAST_LOAD && SrcBC.hasOneUse()) {
43666     auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
43667     unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
43668     if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
43669         VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) {
43670       SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
43671                                  MemIntr->getBasePtr(),
43672                                  MemIntr->getPointerInfo(),
43673                                  MemIntr->getOriginalAlign(),
43674                                  MemIntr->getMemOperand()->getFlags());
43675       DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
43676       return Load;
43677     }
43678   }
43679 
43680   // Handle extract(bitcast(scalar_to_vector(scalar_value))) for integers.
43681   // TODO: Move to DAGCombine?
43682   if (SrcBC.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isInteger() &&
43683       SrcBC.getValueType().isInteger() &&
43684       (SrcBC.getScalarValueSizeInBits() % SrcEltBits) == 0 &&
43685       SrcBC.getScalarValueSizeInBits() ==
43686           SrcBC.getOperand(0).getValueSizeInBits()) {
43687     unsigned Scale = SrcBC.getScalarValueSizeInBits() / SrcEltBits;
43688     if (IdxC.ult(Scale)) {
43689       unsigned Offset = IdxC.getZExtValue() * SrcVT.getScalarSizeInBits();
43690       SDValue Scl = SrcBC.getOperand(0);
43691       EVT SclVT = Scl.getValueType();
43692       if (Offset) {
43693         Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl,
43694                           DAG.getShiftAmountConstant(Offset, SclVT, dl));
43695       }
43696       Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType());
43697       Scl = DAG.getZExtOrTrunc(Scl, dl, VT);
43698       return Scl;
43699     }
43700   }
43701 
43702   // Handle extract(truncate(x)) for 0'th index.
43703   // TODO: Treat this as a faux shuffle?
43704   // TODO: When can we use this for general indices?
43705   if (ISD::TRUNCATE == Src.getOpcode() && IdxC == 0 &&
43706       (SrcVT.getSizeInBits() % 128) == 0) {
43707     Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
43708     MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits);
43709     return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src),
43710                        Idx);
43711   }
43712 
43713   // We can only legally extract other elements from 128-bit vectors and in
43714   // certain circumstances, depending on SSE-level.
43715   // TODO: Investigate float/double extraction if it will be just stored.
43716   auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT,
43717                                                  unsigned Idx) {
43718     EVT VecSVT = VecVT.getScalarType();
43719     if ((VecVT.is256BitVector() || VecVT.is512BitVector()) &&
43720         (VecSVT == MVT::i8 || VecSVT == MVT::i16 || VecSVT == MVT::i32 ||
43721          VecSVT == MVT::i64)) {
43722       unsigned EltSizeInBits = VecSVT.getSizeInBits();
43723       unsigned NumEltsPerLane = 128 / EltSizeInBits;
43724       unsigned LaneOffset = (Idx & ~(NumEltsPerLane - 1)) * EltSizeInBits;
43725       unsigned LaneIdx = LaneOffset / Vec.getScalarValueSizeInBits();
43726       VecVT = EVT::getVectorVT(*DAG.getContext(), VecSVT, NumEltsPerLane);
43727       Vec = extract128BitVector(Vec, LaneIdx, DAG, dl);
43728       Idx &= (NumEltsPerLane - 1);
43729     }
43730     if ((VecVT == MVT::v4i32 || VecVT == MVT::v2i64) &&
43731         ((Idx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) {
43732       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(),
43733                          DAG.getBitcast(VecVT, Vec),
43734                          DAG.getIntPtrConstant(Idx, dl));
43735     }
43736     if ((VecVT == MVT::v8i16 && Subtarget.hasSSE2()) ||
43737         (VecVT == MVT::v16i8 && Subtarget.hasSSE41())) {
43738       unsigned OpCode = (VecVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
43739       return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec),
43740                          DAG.getTargetConstant(Idx, dl, MVT::i8));
43741     }
43742     return SDValue();
43743   };
43744 
43745   // Resolve the target shuffle inputs and mask.
43746   SmallVector<int, 16> Mask;
43747   SmallVector<SDValue, 2> Ops;
43748   if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
43749     return SDValue();
43750 
43751   // Shuffle inputs must be the same size as the result.
43752   if (llvm::any_of(Ops, [SrcVT](SDValue Op) {
43753         return SrcVT.getSizeInBits() != Op.getValueSizeInBits();
43754       }))
43755     return SDValue();
43756 
43757   // Attempt to narrow/widen the shuffle mask to the correct size.
43758   if (Mask.size() != NumSrcElts) {
43759     if ((NumSrcElts % Mask.size()) == 0) {
43760       SmallVector<int, 16> ScaledMask;
43761       int Scale = NumSrcElts / Mask.size();
43762       narrowShuffleMaskElts(Scale, Mask, ScaledMask);
43763       Mask = std::move(ScaledMask);
43764     } else if ((Mask.size() % NumSrcElts) == 0) {
43765       // Simplify Mask based on demanded element.
43766       int ExtractIdx = (int)IdxC.getZExtValue();
43767       int Scale = Mask.size() / NumSrcElts;
43768       int Lo = Scale * ExtractIdx;
43769       int Hi = Scale * (ExtractIdx + 1);
43770       for (int i = 0, e = (int)Mask.size(); i != e; ++i)
43771         if (i < Lo || Hi <= i)
43772           Mask[i] = SM_SentinelUndef;
43773 
43774       SmallVector<int, 16> WidenedMask;
43775       while (Mask.size() > NumSrcElts &&
43776              canWidenShuffleElements(Mask, WidenedMask))
43777         Mask = std::move(WidenedMask);
43778     }
43779   }
43780 
43781   // If narrowing/widening failed, see if we can extract+zero-extend.
43782   int ExtractIdx;
43783   EVT ExtractVT;
43784   if (Mask.size() == NumSrcElts) {
43785     ExtractIdx = Mask[IdxC.getZExtValue()];
43786     ExtractVT = SrcVT;
43787   } else {
43788     unsigned Scale = Mask.size() / NumSrcElts;
43789     if ((Mask.size() % NumSrcElts) != 0 || SrcVT.isFloatingPoint())
43790       return SDValue();
43791     unsigned ScaledIdx = Scale * IdxC.getZExtValue();
43792     if (!isUndefOrZeroInRange(Mask, ScaledIdx + 1, Scale - 1))
43793       return SDValue();
43794     ExtractIdx = Mask[ScaledIdx];
43795     EVT ExtractSVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltBits / Scale);
43796     ExtractVT = EVT::getVectorVT(*DAG.getContext(), ExtractSVT, Mask.size());
43797     assert(SrcVT.getSizeInBits() == ExtractVT.getSizeInBits() &&
43798            "Failed to widen vector type");
43799   }
43800 
43801   // If the shuffle source element is undef/zero then we can just accept it.
43802   if (ExtractIdx == SM_SentinelUndef)
43803     return DAG.getUNDEF(VT);
43804 
43805   if (ExtractIdx == SM_SentinelZero)
43806     return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)
43807                                 : DAG.getConstant(0, dl, VT);
43808 
43809   SDValue SrcOp = Ops[ExtractIdx / Mask.size()];
43810   ExtractIdx = ExtractIdx % Mask.size();
43811   if (SDValue V = GetLegalExtract(SrcOp, ExtractVT, ExtractIdx))
43812     return DAG.getZExtOrTrunc(V, dl, VT);
43813 
43814   return SDValue();
43815 }
43816 
43817 /// Extracting a scalar FP value from vector element 0 is free, so extract each
43818 /// operand first, then perform the math as a scalar op.
43819 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
43820                                  const X86Subtarget &Subtarget) {
43821   assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract");
43822   SDValue Vec = ExtElt->getOperand(0);
43823   SDValue Index = ExtElt->getOperand(1);
43824   EVT VT = ExtElt->getValueType(0);
43825   EVT VecVT = Vec.getValueType();
43826 
43827   // TODO: If this is a unary/expensive/expand op, allow extraction from a
43828   // non-zero element because the shuffle+scalar op will be cheaper?
43829   if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT)
43830     return SDValue();
43831 
43832   // Vector FP compares don't fit the pattern of FP math ops (propagate, not
43833   // extract, the condition code), so deal with those as a special-case.
43834   if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) {
43835     EVT OpVT = Vec.getOperand(0).getValueType().getScalarType();
43836     if (OpVT != MVT::f32 && OpVT != MVT::f64)
43837       return SDValue();
43838 
43839     // extract (setcc X, Y, CC), 0 --> setcc (extract X, 0), (extract Y, 0), CC
43840     SDLoc DL(ExtElt);
43841     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
43842                                Vec.getOperand(0), Index);
43843     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
43844                                Vec.getOperand(1), Index);
43845     return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
43846   }
43847 
43848   if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 &&
43849       VT != MVT::f64)
43850     return SDValue();
43851 
43852   // Vector FP selects don't fit the pattern of FP math ops (because the
43853   // condition has a different type and we have to change the opcode), so deal
43854   // with those here.
43855   // FIXME: This is restricted to pre type legalization by ensuring the setcc
43856   // has i1 elements. If we loosen this we need to convert vector bool to a
43857   // scalar bool.
43858   if (Vec.getOpcode() == ISD::VSELECT &&
43859       Vec.getOperand(0).getOpcode() == ISD::SETCC &&
43860       Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
43861       Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
43862     // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
43863     SDLoc DL(ExtElt);
43864     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
43865                                Vec.getOperand(0).getValueType().getScalarType(),
43866                                Vec.getOperand(0), Index);
43867     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
43868                                Vec.getOperand(1), Index);
43869     SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
43870                                Vec.getOperand(2), Index);
43871     return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
43872   }
43873 
43874   // TODO: This switch could include FNEG and the x86-specific FP logic ops
43875   // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid
43876   // missed load folding and fma+fneg combining.
43877   switch (Vec.getOpcode()) {
43878   case ISD::FMA: // Begin 3 operands
43879   case ISD::FMAD:
43880   case ISD::FADD: // Begin 2 operands
43881   case ISD::FSUB:
43882   case ISD::FMUL:
43883   case ISD::FDIV:
43884   case ISD::FREM:
43885   case ISD::FCOPYSIGN:
43886   case ISD::FMINNUM:
43887   case ISD::FMAXNUM:
43888   case ISD::FMINNUM_IEEE:
43889   case ISD::FMAXNUM_IEEE:
43890   case ISD::FMAXIMUM:
43891   case ISD::FMINIMUM:
43892   case X86ISD::FMAX:
43893   case X86ISD::FMIN:
43894   case ISD::FABS: // Begin 1 operand
43895   case ISD::FSQRT:
43896   case ISD::FRINT:
43897   case ISD::FCEIL:
43898   case ISD::FTRUNC:
43899   case ISD::FNEARBYINT:
43900   case ISD::FROUNDEVEN:
43901   case ISD::FROUND:
43902   case ISD::FFLOOR:
43903   case X86ISD::FRCP:
43904   case X86ISD::FRSQRT: {
43905     // extract (fp X, Y, ...), 0 --> fp (extract X, 0), (extract Y, 0), ...
43906     SDLoc DL(ExtElt);
43907     SmallVector<SDValue, 4> ExtOps;
43908     for (SDValue Op : Vec->ops())
43909       ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
43910     return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
43911   }
43912   default:
43913     return SDValue();
43914   }
43915   llvm_unreachable("All opcodes should return within switch");
43916 }
43917 
43918 /// Try to convert a vector reduction sequence composed of binops and shuffles
43919 /// into horizontal ops.
43920 static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
43921                                      const X86Subtarget &Subtarget) {
43922   assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller");
43923 
43924   // We need at least SSE2 to anything here.
43925   if (!Subtarget.hasSSE2())
43926     return SDValue();
43927 
43928   ISD::NodeType Opc;
43929   SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
43930                                         {ISD::ADD, ISD::MUL, ISD::FADD}, true);
43931   if (!Rdx)
43932     return SDValue();
43933 
43934   SDValue Index = ExtElt->getOperand(1);
43935   assert(isNullConstant(Index) &&
43936          "Reduction doesn't end in an extract from index 0");
43937 
43938   EVT VT = ExtElt->getValueType(0);
43939   EVT VecVT = Rdx.getValueType();
43940   if (VecVT.getScalarType() != VT)
43941     return SDValue();
43942 
43943   SDLoc DL(ExtElt);
43944   unsigned NumElts = VecVT.getVectorNumElements();
43945   unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
43946 
43947   // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
43948   auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
43949     if (V.getValueType() == MVT::v4i8) {
43950       if (ZeroExtend && Subtarget.hasSSE41()) {
43951         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
43952                         DAG.getConstant(0, DL, MVT::v4i32),
43953                         DAG.getBitcast(MVT::i32, V),
43954                         DAG.getIntPtrConstant(0, DL));
43955         return DAG.getBitcast(MVT::v16i8, V);
43956       }
43957       V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
43958                       ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
43959                                  : DAG.getUNDEF(MVT::v4i8));
43960     }
43961     return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
43962                        DAG.getUNDEF(MVT::v8i8));
43963   };
43964 
43965   // vXi8 mul reduction - promote to vXi16 mul reduction.
43966   if (Opc == ISD::MUL) {
43967     if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
43968       return SDValue();
43969     if (VecVT.getSizeInBits() >= 128) {
43970       EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
43971       SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
43972       SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
43973       Lo = DAG.getBitcast(WideVT, Lo);
43974       Hi = DAG.getBitcast(WideVT, Hi);
43975       Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
43976       while (Rdx.getValueSizeInBits() > 128) {
43977         std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
43978         Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
43979       }
43980     } else {
43981       Rdx = WidenToV16I8(Rdx, false);
43982       Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
43983       Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
43984     }
43985     if (NumElts >= 8)
43986       Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
43987                         DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
43988                                              {4, 5, 6, 7, -1, -1, -1, -1}));
43989     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
43990                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
43991                                            {2, 3, -1, -1, -1, -1, -1, -1}));
43992     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
43993                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
43994                                            {1, -1, -1, -1, -1, -1, -1, -1}));
43995     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
43996     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
43997   }
43998 
43999   // vXi8 add reduction - sub 128-bit vector.
44000   if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
44001     Rdx = WidenToV16I8(Rdx, true);
44002     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
44003                       DAG.getConstant(0, DL, MVT::v16i8));
44004     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
44005     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44006   }
44007 
44008   // Must be a >=128-bit vector with pow2 elements.
44009   if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
44010     return SDValue();
44011 
44012   // vXi8 add reduction - sum lo/hi halves then use PSADBW.
44013   if (VT == MVT::i8) {
44014     while (Rdx.getValueSizeInBits() > 128) {
44015       SDValue Lo, Hi;
44016       std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
44017       VecVT = Lo.getValueType();
44018       Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
44019     }
44020     assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");
44021 
44022     SDValue Hi = DAG.getVectorShuffle(
44023         MVT::v16i8, DL, Rdx, Rdx,
44024         {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
44025     Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
44026     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
44027                       getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
44028     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
44029     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44030   }
44031 
44032   // See if we can use vXi8 PSADBW add reduction for larger zext types.
44033   // If the source vector values are 0-255, then we can use PSADBW to
44034   // sum+zext v8i8 subvectors to vXi64, then perform the reduction.
44035   // TODO: See if its worth avoiding vXi16/i32 truncations?
44036   if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 &&
44037       DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
44038       (EltSizeInBits == 16 || Rdx.getOpcode() == ISD::ZERO_EXTEND ||
44039        Subtarget.hasAVX512())) {
44040     if (Rdx.getValueType() == MVT::v8i16) {
44041       Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
44042                         DAG.getUNDEF(MVT::v8i16));
44043     } else {
44044       EVT ByteVT = VecVT.changeVectorElementType(MVT::i8);
44045       Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
44046       if (ByteVT.getSizeInBits() < 128)
44047         Rdx = WidenToV16I8(Rdx, true);
44048     }
44049 
44050     // Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
44051     auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44052                             ArrayRef<SDValue> Ops) {
44053       MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
44054       SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
44055       return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
44056     };
44057     MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);
44058     Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
44059 
44060     // TODO: We could truncate to vXi16/vXi32 before performing the reduction.
44061     while (Rdx.getValueSizeInBits() > 128) {
44062       SDValue Lo, Hi;
44063       std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
44064       VecVT = Lo.getValueType();
44065       Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
44066     }
44067     assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");
44068 
44069     if (NumElts > 8) {
44070       SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
44071       Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
44072     }
44073 
44074     VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
44075     Rdx = DAG.getBitcast(VecVT, Rdx);
44076     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44077   }
44078 
44079   // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
44080   if (!shouldUseHorizontalOp(true, DAG, Subtarget))
44081     return SDValue();
44082 
44083   unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
44084 
44085   // 256-bit horizontal instructions operate on 128-bit chunks rather than
44086   // across the whole vector, so we need an extract + hop preliminary stage.
44087   // This is the only step where the operands of the hop are not the same value.
44088   // TODO: We could extend this to handle 512-bit or even longer vectors.
44089   if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
44090       ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
44091     unsigned NumElts = VecVT.getVectorNumElements();
44092     SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
44093     SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
44094     Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
44095     VecVT = Rdx.getValueType();
44096   }
44097   if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
44098       !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
44099     return SDValue();
44100 
44101   // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
44102   unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
44103   for (unsigned i = 0; i != ReductionSteps; ++i)
44104     Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
44105 
44106   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44107 }
44108 
44109 /// Detect vector gather/scatter index generation and convert it from being a
44110 /// bunch of shuffles and extracts into a somewhat faster sequence.
44111 /// For i686, the best sequence is apparently storing the value and loading
44112 /// scalars back, while for x64 we should use 64-bit extracts and shifts.
44113 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
44114                                        TargetLowering::DAGCombinerInfo &DCI,
44115                                        const X86Subtarget &Subtarget) {
44116   if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
44117     return NewOp;
44118 
44119   SDValue InputVector = N->getOperand(0);
44120   SDValue EltIdx = N->getOperand(1);
44121   auto *CIdx = dyn_cast<ConstantSDNode>(EltIdx);
44122 
44123   EVT SrcVT = InputVector.getValueType();
44124   EVT VT = N->getValueType(0);
44125   SDLoc dl(InputVector);
44126   bool IsPextr = N->getOpcode() != ISD::EXTRACT_VECTOR_ELT;
44127   unsigned NumSrcElts = SrcVT.getVectorNumElements();
44128   unsigned NumEltBits = VT.getScalarSizeInBits();
44129   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44130 
44131   if (CIdx && CIdx->getAPIntValue().uge(NumSrcElts))
44132     return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
44133 
44134   // Integer Constant Folding.
44135   if (CIdx && VT.isInteger()) {
44136     APInt UndefVecElts;
44137     SmallVector<APInt, 16> EltBits;
44138     unsigned VecEltBitWidth = SrcVT.getScalarSizeInBits();
44139     if (getTargetConstantBitsFromNode(InputVector, VecEltBitWidth, UndefVecElts,
44140                                       EltBits, true, false)) {
44141       uint64_t Idx = CIdx->getZExtValue();
44142       if (UndefVecElts[Idx])
44143         return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
44144       return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
44145     }
44146 
44147     // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()).
44148     // Improves lowering of bool masks on rust which splits them into byte array.
44149     if (InputVector.getOpcode() == ISD::BITCAST && (NumEltBits % 8) == 0) {
44150       SDValue Src = peekThroughBitcasts(InputVector);
44151       if (Src.getValueType().getScalarType() == MVT::i1 &&
44152           TLI.isTypeLegal(Src.getValueType())) {
44153         MVT SubVT = MVT::getVectorVT(MVT::i1, NumEltBits);
44154         SDValue Sub = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Src,
44155             DAG.getIntPtrConstant(CIdx->getZExtValue() * NumEltBits, dl));
44156         return DAG.getBitcast(VT, Sub);
44157       }
44158     }
44159   }
44160 
44161   if (IsPextr) {
44162     if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumEltBits),
44163                                  DCI))
44164       return SDValue(N, 0);
44165 
44166     // PEXTR*(PINSR*(v, s, c), c) -> s (with implicit zext handling).
44167     if ((InputVector.getOpcode() == X86ISD::PINSRB ||
44168          InputVector.getOpcode() == X86ISD::PINSRW) &&
44169         InputVector.getOperand(2) == EltIdx) {
44170       assert(SrcVT == InputVector.getOperand(0).getValueType() &&
44171              "Vector type mismatch");
44172       SDValue Scl = InputVector.getOperand(1);
44173       Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl);
44174       return DAG.getZExtOrTrunc(Scl, dl, VT);
44175     }
44176 
44177     // TODO - Remove this once we can handle the implicit zero-extension of
44178     // X86ISD::PEXTRW/X86ISD::PEXTRB in combinePredicateReduction and
44179     // combineBasicSADPattern.
44180     return SDValue();
44181   }
44182 
44183   // Detect mmx extraction of all bits as a i64. It works better as a bitcast.
44184   if (VT == MVT::i64 && SrcVT == MVT::v1i64 &&
44185       InputVector.getOpcode() == ISD::BITCAST &&
44186       InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
44187       isNullConstant(EltIdx) && InputVector.hasOneUse())
44188     return DAG.getBitcast(VT, InputVector);
44189 
44190   // Detect mmx to i32 conversion through a v2i32 elt extract.
44191   if (VT == MVT::i32 && SrcVT == MVT::v2i32 &&
44192       InputVector.getOpcode() == ISD::BITCAST &&
44193       InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
44194       isNullConstant(EltIdx) && InputVector.hasOneUse())
44195     return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
44196                        InputVector.getOperand(0));
44197 
44198   // Check whether this extract is the root of a sum of absolute differences
44199   // pattern. This has to be done here because we really want it to happen
44200   // pre-legalization,
44201   if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
44202     return SAD;
44203 
44204   if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
44205     return VPDPBUSD;
44206 
44207   // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
44208   if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
44209     return Cmp;
44210 
44211   // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
44212   if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
44213     return MinMax;
44214 
44215   // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
44216   if (SDValue V = combineArithReduction(N, DAG, Subtarget))
44217     return V;
44218 
44219   if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget))
44220     return V;
44221 
44222   // Attempt to extract a i1 element by using MOVMSK to extract the signbits
44223   // and then testing the relevant element.
44224   //
44225   // Note that we only combine extracts on the *same* result number, i.e.
44226   //   t0 = merge_values a0, a1, a2, a3
44227   //   i1 = extract_vector_elt t0, Constant:i64<2>
44228   //   i1 = extract_vector_elt t0, Constant:i64<3>
44229   // but not
44230   //   i1 = extract_vector_elt t0:1, Constant:i64<2>
44231   // since the latter would need its own MOVMSK.
44232   if (SrcVT.getScalarType() == MVT::i1) {
44233     bool IsVar = !CIdx;
44234     SmallVector<SDNode *, 16> BoolExtracts;
44235     unsigned ResNo = InputVector.getResNo();
44236     auto IsBoolExtract = [&BoolExtracts, &ResNo, &IsVar](SDNode *Use) {
44237       if (Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
44238           Use->getOperand(0).getResNo() == ResNo &&
44239           Use->getValueType(0) == MVT::i1) {
44240         BoolExtracts.push_back(Use);
44241         IsVar |= !isa<ConstantSDNode>(Use->getOperand(1));
44242         return true;
44243       }
44244       return false;
44245     };
44246     // TODO: Can we drop the oneuse check for constant extracts?
44247     if (all_of(InputVector->uses(), IsBoolExtract) &&
44248         (IsVar || BoolExtracts.size() > 1)) {
44249       EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts);
44250       if (SDValue BC =
44251               combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) {
44252         for (SDNode *Use : BoolExtracts) {
44253           // extractelement vXi1 X, MaskIdx --> ((movmsk X) & Mask) == Mask
44254           // Mask = 1 << MaskIdx
44255           SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);
44256           SDValue MaskBit = DAG.getConstant(1, dl, BCVT);
44257           SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx);
44258           SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
44259           Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);
44260           DCI.CombineTo(Use, Res);
44261         }
44262         return SDValue(N, 0);
44263       }
44264     }
44265   }
44266 
44267   // If this extract is from a loaded vector value and will be used as an
44268   // integer, that requires a potentially expensive XMM -> GPR transfer.
44269   // Additionally, if we can convert to a scalar integer load, that will likely
44270   // be folded into a subsequent integer op.
44271   // Note: Unlike the related fold for this in DAGCombiner, this is not limited
44272   //       to a single-use of the loaded vector. For the reasons above, we
44273   //       expect this to be profitable even if it creates an extra load.
44274   bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) {
44275     return Use->getOpcode() == ISD::STORE ||
44276            Use->getOpcode() == ISD::INSERT_VECTOR_ELT ||
44277            Use->getOpcode() == ISD::SCALAR_TO_VECTOR;
44278   });
44279   auto *LoadVec = dyn_cast<LoadSDNode>(InputVector);
44280   if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() &&
44281       SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() &&
44282       !LikelyUsedAsVector && LoadVec->isSimple()) {
44283     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44284     SDValue NewPtr =
44285         TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx);
44286     unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8;
44287     MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff);
44288     Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff);
44289     SDValue Load =
44290         DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
44291                     LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo());
44292     DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
44293     return Load;
44294   }
44295 
44296   return SDValue();
44297 }
44298 
44299 // Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
44300 // This is more or less the reverse of combineBitcastvxi1.
44301 static SDValue combineToExtendBoolVectorInReg(
44302     unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG,
44303     TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) {
44304   if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND &&
44305       Opcode != ISD::ANY_EXTEND)
44306     return SDValue();
44307   if (!DCI.isBeforeLegalizeOps())
44308     return SDValue();
44309   if (!Subtarget.hasSSE2() || Subtarget.hasAVX512())
44310     return SDValue();
44311 
44312   EVT SVT = VT.getScalarType();
44313   EVT InSVT = N0.getValueType().getScalarType();
44314   unsigned EltSizeInBits = SVT.getSizeInBits();
44315 
44316   // Input type must be extending a bool vector (bit-casted from a scalar
44317   // integer) to legal integer types.
44318   if (!VT.isVector())
44319     return SDValue();
44320   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8)
44321     return SDValue();
44322   if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST)
44323     return SDValue();
44324 
44325   SDValue N00 = N0.getOperand(0);
44326   EVT SclVT = N00.getValueType();
44327   if (!SclVT.isScalarInteger())
44328     return SDValue();
44329 
44330   SDValue Vec;
44331   SmallVector<int> ShuffleMask;
44332   unsigned NumElts = VT.getVectorNumElements();
44333   assert(NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size");
44334 
44335   // Broadcast the scalar integer to the vector elements.
44336   if (NumElts > EltSizeInBits) {
44337     // If the scalar integer is greater than the vector element size, then we
44338     // must split it down into sub-sections for broadcasting. For example:
44339     //   i16 -> v16i8 (i16 -> v8i16 -> v16i8) with 2 sub-sections.
44340     //   i32 -> v32i8 (i32 -> v8i32 -> v32i8) with 4 sub-sections.
44341     assert((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale");
44342     unsigned Scale = NumElts / EltSizeInBits;
44343     EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits);
44344     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
44345     Vec = DAG.getBitcast(VT, Vec);
44346 
44347     for (unsigned i = 0; i != Scale; ++i)
44348       ShuffleMask.append(EltSizeInBits, i);
44349     Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
44350   } else if (Subtarget.hasAVX2() && NumElts < EltSizeInBits &&
44351              (SclVT == MVT::i8 || SclVT == MVT::i16 || SclVT == MVT::i32)) {
44352     // If we have register broadcast instructions, use the scalar size as the
44353     // element type for the shuffle. Then cast to the wider element type. The
44354     // widened bits won't be used, and this might allow the use of a broadcast
44355     // load.
44356     assert((EltSizeInBits % NumElts) == 0 && "Unexpected integer scale");
44357     unsigned Scale = EltSizeInBits / NumElts;
44358     EVT BroadcastVT =
44359         EVT::getVectorVT(*DAG.getContext(), SclVT, NumElts * Scale);
44360     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
44361     ShuffleMask.append(NumElts * Scale, 0);
44362     Vec = DAG.getVectorShuffle(BroadcastVT, DL, Vec, Vec, ShuffleMask);
44363     Vec = DAG.getBitcast(VT, Vec);
44364   } else {
44365     // For smaller scalar integers, we can simply any-extend it to the vector
44366     // element size (we don't care about the upper bits) and broadcast it to all
44367     // elements.
44368     SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
44369     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
44370     ShuffleMask.append(NumElts, 0);
44371     Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
44372   }
44373 
44374   // Now, mask the relevant bit in each element.
44375   SmallVector<SDValue, 32> Bits;
44376   for (unsigned i = 0; i != NumElts; ++i) {
44377     int BitIdx = (i % EltSizeInBits);
44378     APInt Bit = APInt::getBitsSet(EltSizeInBits, BitIdx, BitIdx + 1);
44379     Bits.push_back(DAG.getConstant(Bit, DL, SVT));
44380   }
44381   SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
44382   Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
44383 
44384   // Compare against the bitmask and extend the result.
44385   EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
44386   Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
44387   Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
44388 
44389   // For SEXT, this is now done, otherwise shift the result down for
44390   // zero-extension.
44391   if (Opcode == ISD::SIGN_EXTEND)
44392     return Vec;
44393   return DAG.getNode(ISD::SRL, DL, VT, Vec,
44394                      DAG.getConstant(EltSizeInBits - 1, DL, VT));
44395 }
44396 
44397 /// If a vector select has an operand that is -1 or 0, try to simplify the
44398 /// select to a bitwise logic operation.
44399 /// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
44400 static SDValue
44401 combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG,
44402                                  TargetLowering::DAGCombinerInfo &DCI,
44403                                  const X86Subtarget &Subtarget) {
44404   SDValue Cond = N->getOperand(0);
44405   SDValue LHS = N->getOperand(1);
44406   SDValue RHS = N->getOperand(2);
44407   EVT VT = LHS.getValueType();
44408   EVT CondVT = Cond.getValueType();
44409   SDLoc DL(N);
44410   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44411 
44412   if (N->getOpcode() != ISD::VSELECT)
44413     return SDValue();
44414 
44415   assert(CondVT.isVector() && "Vector select expects a vector selector!");
44416 
44417   // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
44418   // TODO: Can we assert that both operands are not zeros (because that should
44419   //       get simplified at node creation time)?
44420   bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
44421   bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
44422 
44423   // If both inputs are 0/undef, create a complete zero vector.
44424   // FIXME: As noted above this should be handled by DAGCombiner/getNode.
44425   if (TValIsAllZeros && FValIsAllZeros) {
44426     if (VT.isFloatingPoint())
44427       return DAG.getConstantFP(0.0, DL, VT);
44428     return DAG.getConstant(0, DL, VT);
44429   }
44430 
44431   // To use the condition operand as a bitwise mask, it must have elements that
44432   // are the same size as the select elements. Ie, the condition operand must
44433   // have already been promoted from the IR select condition type <N x i1>.
44434   // Don't check if the types themselves are equal because that excludes
44435   // vector floating-point selects.
44436   if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
44437     return SDValue();
44438 
44439   // Try to invert the condition if true value is not all 1s and false value is
44440   // not all 0s. Only do this if the condition has one use.
44441   bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
44442   if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
44443       // Check if the selector will be produced by CMPP*/PCMP*.
44444       Cond.getOpcode() == ISD::SETCC &&
44445       // Check if SETCC has already been promoted.
44446       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
44447           CondVT) {
44448     bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
44449 
44450     if (TValIsAllZeros || FValIsAllOnes) {
44451       SDValue CC = Cond.getOperand(2);
44452       ISD::CondCode NewCC = ISD::getSetCCInverse(
44453           cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
44454       Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
44455                           NewCC);
44456       std::swap(LHS, RHS);
44457       TValIsAllOnes = FValIsAllOnes;
44458       FValIsAllZeros = TValIsAllZeros;
44459     }
44460   }
44461 
44462   // Cond value must be 'sign splat' to be converted to a logical op.
44463   if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
44464     return SDValue();
44465 
44466   // vselect Cond, 111..., 000... -> Cond
44467   if (TValIsAllOnes && FValIsAllZeros)
44468     return DAG.getBitcast(VT, Cond);
44469 
44470   if (!TLI.isTypeLegal(CondVT))
44471     return SDValue();
44472 
44473   // vselect Cond, 111..., X -> or Cond, X
44474   if (TValIsAllOnes) {
44475     SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
44476     SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
44477     return DAG.getBitcast(VT, Or);
44478   }
44479 
44480   // vselect Cond, X, 000... -> and Cond, X
44481   if (FValIsAllZeros) {
44482     SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
44483     SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
44484     return DAG.getBitcast(VT, And);
44485   }
44486 
44487   // vselect Cond, 000..., X -> andn Cond, X
44488   if (TValIsAllZeros) {
44489     SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
44490     SDValue AndN;
44491     // The canonical form differs for i1 vectors - x86andnp is not used
44492     if (CondVT.getScalarType() == MVT::i1)
44493       AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
44494                          CastRHS);
44495     else
44496       AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
44497     return DAG.getBitcast(VT, AndN);
44498   }
44499 
44500   return SDValue();
44501 }
44502 
44503 /// If both arms of a vector select are concatenated vectors, split the select,
44504 /// and concatenate the result to eliminate a wide (256-bit) vector instruction:
44505 ///   vselect Cond, (concat T0, T1), (concat F0, F1) -->
44506 ///   concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
44507 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG,
44508                                   const X86Subtarget &Subtarget) {
44509   unsigned Opcode = N->getOpcode();
44510   if (Opcode != X86ISD::BLENDV && Opcode != ISD::VSELECT)
44511     return SDValue();
44512 
44513   // TODO: Split 512-bit vectors too?
44514   EVT VT = N->getValueType(0);
44515   if (!VT.is256BitVector())
44516     return SDValue();
44517 
44518   // TODO: Split as long as any 2 of the 3 operands are concatenated?
44519   SDValue Cond = N->getOperand(0);
44520   SDValue TVal = N->getOperand(1);
44521   SDValue FVal = N->getOperand(2);
44522   if (!TVal.hasOneUse() || !FVal.hasOneUse() ||
44523       !isFreeToSplitVector(TVal.getNode(), DAG) ||
44524       !isFreeToSplitVector(FVal.getNode(), DAG))
44525     return SDValue();
44526 
44527   auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
44528                             ArrayRef<SDValue> Ops) {
44529     return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
44530   };
44531   return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { Cond, TVal, FVal },
44532                           makeBlend, /*CheckBWI*/ false);
44533 }
44534 
44535 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
44536   SDValue Cond = N->getOperand(0);
44537   SDValue LHS = N->getOperand(1);
44538   SDValue RHS = N->getOperand(2);
44539   SDLoc DL(N);
44540 
44541   auto *TrueC = dyn_cast<ConstantSDNode>(LHS);
44542   auto *FalseC = dyn_cast<ConstantSDNode>(RHS);
44543   if (!TrueC || !FalseC)
44544     return SDValue();
44545 
44546   // Don't do this for crazy integer types.
44547   EVT VT = N->getValueType(0);
44548   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
44549     return SDValue();
44550 
44551   // We're going to use the condition bit in math or logic ops. We could allow
44552   // this with a wider condition value (post-legalization it becomes an i8),
44553   // but if nothing is creating selects that late, it doesn't matter.
44554   if (Cond.getValueType() != MVT::i1)
44555     return SDValue();
44556 
44557   // A power-of-2 multiply is just a shift. LEA also cheaply handles multiply by
44558   // 3, 5, or 9 with i32/i64, so those get transformed too.
44559   // TODO: For constants that overflow or do not differ by power-of-2 or small
44560   // multiplier, convert to 'and' + 'add'.
44561   const APInt &TrueVal = TrueC->getAPIntValue();
44562   const APInt &FalseVal = FalseC->getAPIntValue();
44563 
44564   // We have a more efficient lowering for "(X == 0) ? Y : -1" using SBB.
44565   if ((TrueVal.isAllOnes() || FalseVal.isAllOnes()) &&
44566       Cond.getOpcode() == ISD::SETCC && isNullConstant(Cond.getOperand(1))) {
44567     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
44568     if (CC == ISD::SETEQ || CC == ISD::SETNE)
44569       return SDValue();
44570   }
44571 
44572   bool OV;
44573   APInt Diff = TrueVal.ssub_ov(FalseVal, OV);
44574   if (OV)
44575     return SDValue();
44576 
44577   APInt AbsDiff = Diff.abs();
44578   if (AbsDiff.isPowerOf2() ||
44579       ((VT == MVT::i32 || VT == MVT::i64) &&
44580        (AbsDiff == 3 || AbsDiff == 5 || AbsDiff == 9))) {
44581 
44582     // We need a positive multiplier constant for shift/LEA codegen. The 'not'
44583     // of the condition can usually be folded into a compare predicate, but even
44584     // without that, the sequence should be cheaper than a CMOV alternative.
44585     if (TrueVal.slt(FalseVal)) {
44586       Cond = DAG.getNOT(DL, Cond, MVT::i1);
44587       std::swap(TrueC, FalseC);
44588     }
44589 
44590     // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC
44591     SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
44592 
44593     // Multiply condition by the difference if non-one.
44594     if (!AbsDiff.isOne())
44595       R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
44596 
44597     // Add the base if non-zero.
44598     if (!FalseC->isZero())
44599       R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
44600 
44601     return R;
44602   }
44603 
44604   return SDValue();
44605 }
44606 
44607 /// If this is a *dynamic* select (non-constant condition) and we can match
44608 /// this node with one of the variable blend instructions, restructure the
44609 /// condition so that blends can use the high (sign) bit of each element.
44610 /// This function will also call SimplifyDemandedBits on already created
44611 /// BLENDV to perform additional simplifications.
44612 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
44613                                       TargetLowering::DAGCombinerInfo &DCI,
44614                                       const X86Subtarget &Subtarget) {
44615   SDValue Cond = N->getOperand(0);
44616   if ((N->getOpcode() != ISD::VSELECT &&
44617        N->getOpcode() != X86ISD::BLENDV) ||
44618       ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
44619     return SDValue();
44620 
44621   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44622   unsigned BitWidth = Cond.getScalarValueSizeInBits();
44623   EVT VT = N->getValueType(0);
44624 
44625   // We can only handle the cases where VSELECT is directly legal on the
44626   // subtarget. We custom lower VSELECT nodes with constant conditions and
44627   // this makes it hard to see whether a dynamic VSELECT will correctly
44628   // lower, so we both check the operation's status and explicitly handle the
44629   // cases where a *dynamic* blend will fail even though a constant-condition
44630   // blend could be custom lowered.
44631   // FIXME: We should find a better way to handle this class of problems.
44632   // Potentially, we should combine constant-condition vselect nodes
44633   // pre-legalization into shuffles and not mark as many types as custom
44634   // lowered.
44635   if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
44636     return SDValue();
44637   // FIXME: We don't support i16-element blends currently. We could and
44638   // should support them by making *all* the bits in the condition be set
44639   // rather than just the high bit and using an i8-element blend.
44640   if (VT.getVectorElementType() == MVT::i16)
44641     return SDValue();
44642   // Dynamic blending was only available from SSE4.1 onward.
44643   if (VT.is128BitVector() && !Subtarget.hasSSE41())
44644     return SDValue();
44645   // Byte blends are only available in AVX2
44646   if (VT == MVT::v32i8 && !Subtarget.hasAVX2())
44647     return SDValue();
44648   // There are no 512-bit blend instructions that use sign bits.
44649   if (VT.is512BitVector())
44650     return SDValue();
44651 
44652   // Don't optimize before the condition has been transformed to a legal type
44653   // and don't ever optimize vector selects that map to AVX512 mask-registers.
44654   if (BitWidth < 8 || BitWidth > 64)
44655     return SDValue();
44656 
44657   auto OnlyUsedAsSelectCond = [](SDValue Cond) {
44658     for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
44659          UI != UE; ++UI)
44660       if ((UI->getOpcode() != ISD::VSELECT &&
44661            UI->getOpcode() != X86ISD::BLENDV) ||
44662           UI.getOperandNo() != 0)
44663         return false;
44664 
44665     return true;
44666   };
44667 
44668   APInt DemandedBits(APInt::getSignMask(BitWidth));
44669 
44670   if (OnlyUsedAsSelectCond(Cond)) {
44671     KnownBits Known;
44672     TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
44673                                           !DCI.isBeforeLegalizeOps());
44674     if (!TLI.SimplifyDemandedBits(Cond, DemandedBits, Known, TLO, 0, true))
44675       return SDValue();
44676 
44677     // If we changed the computation somewhere in the DAG, this change will
44678     // affect all users of Cond. Update all the nodes so that we do not use
44679     // the generic VSELECT anymore. Otherwise, we may perform wrong
44680     // optimizations as we messed with the actual expectation for the vector
44681     // boolean values.
44682     for (SDNode *U : Cond->uses()) {
44683       if (U->getOpcode() == X86ISD::BLENDV)
44684         continue;
44685 
44686       SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
44687                                Cond, U->getOperand(1), U->getOperand(2));
44688       DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
44689       DCI.AddToWorklist(U);
44690     }
44691     DCI.CommitTargetLoweringOpt(TLO);
44692     return SDValue(N, 0);
44693   }
44694 
44695   // Otherwise we can still at least try to simplify multiple use bits.
44696   if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
44697       return DAG.getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), V,
44698                          N->getOperand(1), N->getOperand(2));
44699 
44700   return SDValue();
44701 }
44702 
44703 // Try to match:
44704 //   (or (and (M, (sub 0, X)), (pandn M, X)))
44705 // which is a special case of:
44706 //   (select M, (sub 0, X), X)
44707 // Per:
44708 // http://graphics.stanford.edu/~seander/bithacks.html#ConditionalNegate
44709 // We know that, if fNegate is 0 or 1:
44710 //   (fNegate ? -v : v) == ((v ^ -fNegate) + fNegate)
44711 //
44712 // Here, we have a mask, M (all 1s or 0), and, similarly, we know that:
44713 //   ((M & 1) ? -X : X) == ((X ^ -(M & 1)) + (M & 1))
44714 //   ( M      ? -X : X) == ((X ^   M     ) + (M & 1))
44715 // This lets us transform our vselect to:
44716 //   (add (xor X, M), (and M, 1))
44717 // And further to:
44718 //   (sub (xor X, M), M)
44719 static SDValue combineLogicBlendIntoConditionalNegate(
44720     EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
44721     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
44722   EVT MaskVT = Mask.getValueType();
44723   assert(MaskVT.isInteger() &&
44724          DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() &&
44725          "Mask must be zero/all-bits");
44726 
44727   if (X.getValueType() != MaskVT || Y.getValueType() != MaskVT)
44728     return SDValue();
44729   if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT))
44730     return SDValue();
44731 
44732   auto IsNegV = [](SDNode *N, SDValue V) {
44733     return N->getOpcode() == ISD::SUB && N->getOperand(1) == V &&
44734            ISD::isBuildVectorAllZeros(N->getOperand(0).getNode());
44735   };
44736 
44737   SDValue V;
44738   if (IsNegV(Y.getNode(), X))
44739     V = X;
44740   else if (IsNegV(X.getNode(), Y))
44741     V = Y;
44742   else
44743     return SDValue();
44744 
44745   SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
44746   SDValue SubOp2 = Mask;
44747 
44748   // If the negate was on the false side of the select, then
44749   // the operands of the SUB need to be swapped. PR 27251.
44750   // This is because the pattern being matched above is
44751   // (vselect M, (sub (0, X), X)  -> (sub (xor X, M), M)
44752   // but if the pattern matched was
44753   // (vselect M, X, (sub (0, X))), that is really negation of the pattern
44754   // above, -(vselect M, (sub 0, X), X), and therefore the replacement
44755   // pattern also needs to be a negation of the replacement pattern above.
44756   // And -(sub X, Y) is just sub (Y, X), so swapping the operands of the
44757   // sub accomplishes the negation of the replacement pattern.
44758   if (V == Y)
44759     std::swap(SubOp1, SubOp2);
44760 
44761   SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
44762   return DAG.getBitcast(VT, Res);
44763 }
44764 
44765 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG,
44766                                   const X86Subtarget &Subtarget) {
44767   if (!Subtarget.hasAVX512())
44768     return SDValue();
44769   if (N->getOpcode() != ISD::VSELECT)
44770     return SDValue();
44771 
44772   SDLoc DL(N);
44773   SDValue Cond = N->getOperand(0);
44774   SDValue LHS = N->getOperand(1);
44775   SDValue RHS = N->getOperand(2);
44776 
44777   if (canCombineAsMaskOperation(LHS, Subtarget))
44778     return SDValue();
44779 
44780   if (!canCombineAsMaskOperation(RHS, Subtarget))
44781     return SDValue();
44782 
44783   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
44784     return SDValue();
44785 
44786   // Commute LHS and RHS to create opportunity to select mask instruction.
44787   // (vselect M, L, R) -> (vselect ~M, R, L)
44788   ISD::CondCode NewCC =
44789       ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
44790                            Cond.getOperand(0).getValueType());
44791   Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
44792 		                        Cond.getOperand(1), NewCC);
44793   return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
44794 }
44795 
44796 /// Do target-specific dag combines on SELECT and VSELECT nodes.
44797 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
44798                              TargetLowering::DAGCombinerInfo &DCI,
44799                              const X86Subtarget &Subtarget) {
44800   SDLoc DL(N);
44801   SDValue Cond = N->getOperand(0);
44802   SDValue LHS = N->getOperand(1);
44803   SDValue RHS = N->getOperand(2);
44804 
44805   // Try simplification again because we use this function to optimize
44806   // BLENDV nodes that are not handled by the generic combiner.
44807   if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
44808     return V;
44809 
44810   // When avx512 is available the lhs operand of select instruction can be
44811   // folded with mask instruction, while the rhs operand can't. Commute the
44812   // lhs and rhs of the select instruction to create the opportunity of
44813   // folding.
44814   if (SDValue V = commuteSelect(N, DAG, Subtarget))
44815     return V;
44816 
44817   EVT VT = LHS.getValueType();
44818   EVT CondVT = Cond.getValueType();
44819   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44820   bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
44821 
44822   // Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
44823   // Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
44824   // can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
44825   if (CondVT.isVector() && CondVT.isInteger() &&
44826       CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
44827       (!CondConstantVector || CondVT.getScalarType() == MVT::i8) &&
44828       DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits())
44829     if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
44830                                                            DL, DAG, Subtarget))
44831       return V;
44832 
44833   // Convert vselects with constant condition into shuffles.
44834   if (CondConstantVector && DCI.isBeforeLegalizeOps() &&
44835       (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV)) {
44836     SmallVector<int, 64> Mask;
44837     if (createShuffleMaskFromVSELECT(Mask, Cond,
44838                                      N->getOpcode() == X86ISD::BLENDV))
44839       return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
44840   }
44841 
44842   // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y))
44843   // by forcing the unselected elements to zero.
44844   // TODO: Can we handle more shuffles with this?
44845   if (N->getOpcode() == ISD::VSELECT && CondVT.isVector() &&
44846       LHS.getOpcode() == X86ISD::PSHUFB && RHS.getOpcode() == X86ISD::PSHUFB &&
44847       LHS.hasOneUse() && RHS.hasOneUse()) {
44848     MVT SimpleVT = VT.getSimpleVT();
44849     SmallVector<SDValue, 1> LHSOps, RHSOps;
44850     SmallVector<int, 64> LHSMask, RHSMask, CondMask;
44851     if (createShuffleMaskFromVSELECT(CondMask, Cond) &&
44852         getTargetShuffleMask(LHS.getNode(), SimpleVT, true, LHSOps, LHSMask) &&
44853         getTargetShuffleMask(RHS.getNode(), SimpleVT, true, RHSOps, RHSMask)) {
44854       int NumElts = VT.getVectorNumElements();
44855       for (int i = 0; i != NumElts; ++i) {
44856         // getConstVector sets negative shuffle mask values as undef, so ensure
44857         // we hardcode SM_SentinelZero values to zero (0x80).
44858         if (CondMask[i] < NumElts) {
44859           LHSMask[i] = isUndefOrZero(LHSMask[i]) ? 0x80 : LHSMask[i];
44860           RHSMask[i] = 0x80;
44861         } else {
44862           LHSMask[i] = 0x80;
44863           RHSMask[i] = isUndefOrZero(RHSMask[i]) ? 0x80 : RHSMask[i];
44864         }
44865       }
44866       LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0),
44867                         getConstVector(LHSMask, SimpleVT, DAG, DL, true));
44868       RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0),
44869                         getConstVector(RHSMask, SimpleVT, DAG, DL, true));
44870       return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
44871     }
44872   }
44873 
44874   // If we have SSE[12] support, try to form min/max nodes. SSE min/max
44875   // instructions match the semantics of the common C idiom x<y?x:y but not
44876   // x<=y?x:y, because of how they handle negative zero (which can be
44877   // ignored in unsafe-math mode).
44878   // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
44879   if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
44880       VT != MVT::f80 && VT != MVT::f128 && !isSoftF16(VT, Subtarget) &&
44881       (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
44882       (Subtarget.hasSSE2() ||
44883        (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {
44884     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
44885 
44886     unsigned Opcode = 0;
44887     // Check for x CC y ? x : y.
44888     if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
44889         DAG.isEqualTo(RHS, Cond.getOperand(1))) {
44890       switch (CC) {
44891       default: break;
44892       case ISD::SETULT:
44893         // Converting this to a min would handle NaNs incorrectly, and swapping
44894         // the operands would cause it to handle comparisons between positive
44895         // and negative zero incorrectly.
44896         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
44897           if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44898               !(DAG.isKnownNeverZeroFloat(LHS) ||
44899                 DAG.isKnownNeverZeroFloat(RHS)))
44900             break;
44901           std::swap(LHS, RHS);
44902         }
44903         Opcode = X86ISD::FMIN;
44904         break;
44905       case ISD::SETOLE:
44906         // Converting this to a min would handle comparisons between positive
44907         // and negative zero incorrectly.
44908         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44909             !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
44910           break;
44911         Opcode = X86ISD::FMIN;
44912         break;
44913       case ISD::SETULE:
44914         // Converting this to a min would handle both negative zeros and NaNs
44915         // incorrectly, but we can swap the operands to fix both.
44916         std::swap(LHS, RHS);
44917         [[fallthrough]];
44918       case ISD::SETOLT:
44919       case ISD::SETLT:
44920       case ISD::SETLE:
44921         Opcode = X86ISD::FMIN;
44922         break;
44923 
44924       case ISD::SETOGE:
44925         // Converting this to a max would handle comparisons between positive
44926         // and negative zero incorrectly.
44927         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44928             !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
44929           break;
44930         Opcode = X86ISD::FMAX;
44931         break;
44932       case ISD::SETUGT:
44933         // Converting this to a max would handle NaNs incorrectly, and swapping
44934         // the operands would cause it to handle comparisons between positive
44935         // and negative zero incorrectly.
44936         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
44937           if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44938               !(DAG.isKnownNeverZeroFloat(LHS) ||
44939                 DAG.isKnownNeverZeroFloat(RHS)))
44940             break;
44941           std::swap(LHS, RHS);
44942         }
44943         Opcode = X86ISD::FMAX;
44944         break;
44945       case ISD::SETUGE:
44946         // Converting this to a max would handle both negative zeros and NaNs
44947         // incorrectly, but we can swap the operands to fix both.
44948         std::swap(LHS, RHS);
44949         [[fallthrough]];
44950       case ISD::SETOGT:
44951       case ISD::SETGT:
44952       case ISD::SETGE:
44953         Opcode = X86ISD::FMAX;
44954         break;
44955       }
44956     // Check for x CC y ? y : x -- a min/max with reversed arms.
44957     } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
44958                DAG.isEqualTo(RHS, Cond.getOperand(0))) {
44959       switch (CC) {
44960       default: break;
44961       case ISD::SETOGE:
44962         // Converting this to a min would handle comparisons between positive
44963         // and negative zero incorrectly, and swapping the operands would
44964         // cause it to handle NaNs incorrectly.
44965         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44966             !(DAG.isKnownNeverZeroFloat(LHS) ||
44967               DAG.isKnownNeverZeroFloat(RHS))) {
44968           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
44969             break;
44970           std::swap(LHS, RHS);
44971         }
44972         Opcode = X86ISD::FMIN;
44973         break;
44974       case ISD::SETUGT:
44975         // Converting this to a min would handle NaNs incorrectly.
44976         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
44977           break;
44978         Opcode = X86ISD::FMIN;
44979         break;
44980       case ISD::SETUGE:
44981         // Converting this to a min would handle both negative zeros and NaNs
44982         // incorrectly, but we can swap the operands to fix both.
44983         std::swap(LHS, RHS);
44984         [[fallthrough]];
44985       case ISD::SETOGT:
44986       case ISD::SETGT:
44987       case ISD::SETGE:
44988         Opcode = X86ISD::FMIN;
44989         break;
44990 
44991       case ISD::SETULT:
44992         // Converting this to a max would handle NaNs incorrectly.
44993         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
44994           break;
44995         Opcode = X86ISD::FMAX;
44996         break;
44997       case ISD::SETOLE:
44998         // Converting this to a max would handle comparisons between positive
44999         // and negative zero incorrectly, and swapping the operands would
45000         // cause it to handle NaNs incorrectly.
45001         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
45002             !DAG.isKnownNeverZeroFloat(LHS) &&
45003             !DAG.isKnownNeverZeroFloat(RHS)) {
45004           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
45005             break;
45006           std::swap(LHS, RHS);
45007         }
45008         Opcode = X86ISD::FMAX;
45009         break;
45010       case ISD::SETULE:
45011         // Converting this to a max would handle both negative zeros and NaNs
45012         // incorrectly, but we can swap the operands to fix both.
45013         std::swap(LHS, RHS);
45014         [[fallthrough]];
45015       case ISD::SETOLT:
45016       case ISD::SETLT:
45017       case ISD::SETLE:
45018         Opcode = X86ISD::FMAX;
45019         break;
45020       }
45021     }
45022 
45023     if (Opcode)
45024       return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
45025   }
45026 
45027   // Some mask scalar intrinsics rely on checking if only one bit is set
45028   // and implement it in C code like this:
45029   // A[0] = (U & 1) ? A[0] : W[0];
45030   // This creates some redundant instructions that break pattern matching.
45031   // fold (select (setcc (and (X, 1), 0, seteq), Y, Z)) -> select(and(X, 1),Z,Y)
45032   if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT &&
45033       Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {
45034     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
45035     SDValue AndNode = Cond.getOperand(0);
45036     if (AndNode.getOpcode() == ISD::AND && CC == ISD::SETEQ &&
45037         isNullConstant(Cond.getOperand(1)) &&
45038         isOneConstant(AndNode.getOperand(1))) {
45039       // LHS and RHS swapped due to
45040       // setcc outputting 1 when AND resulted in 0 and vice versa.
45041       AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
45042       return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
45043     }
45044   }
45045 
45046   // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
45047   // lowering on KNL. In this case we convert it to
45048   // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
45049   // The same situation all vectors of i8 and i16 without BWI.
45050   // Make sure we extend these even before type legalization gets a chance to
45051   // split wide vectors.
45052   // Since SKX these selects have a proper lowering.
45053   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() &&
45054       CondVT.getVectorElementType() == MVT::i1 &&
45055       (VT.getVectorElementType() == MVT::i8 ||
45056        VT.getVectorElementType() == MVT::i16)) {
45057     Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
45058     return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
45059   }
45060 
45061   // AVX512 - Extend select with zero to merge with target shuffle.
45062   // select(mask, extract_subvector(shuffle(x)), zero) -->
45063   // extract_subvector(select(insert_subvector(mask), shuffle(x), zero))
45064   // TODO - support non target shuffles as well.
45065   if (Subtarget.hasAVX512() && CondVT.isVector() &&
45066       CondVT.getVectorElementType() == MVT::i1) {
45067     auto SelectableOp = [&TLI](SDValue Op) {
45068       return Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
45069              isTargetShuffle(Op.getOperand(0).getOpcode()) &&
45070              isNullConstant(Op.getOperand(1)) &&
45071              TLI.isTypeLegal(Op.getOperand(0).getValueType()) &&
45072              Op.hasOneUse() && Op.getOperand(0).hasOneUse();
45073     };
45074 
45075     bool SelectableLHS = SelectableOp(LHS);
45076     bool SelectableRHS = SelectableOp(RHS);
45077     bool ZeroLHS = ISD::isBuildVectorAllZeros(LHS.getNode());
45078     bool ZeroRHS = ISD::isBuildVectorAllZeros(RHS.getNode());
45079 
45080     if ((SelectableLHS && ZeroRHS) || (SelectableRHS && ZeroLHS)) {
45081       EVT SrcVT = SelectableLHS ? LHS.getOperand(0).getValueType()
45082                                 : RHS.getOperand(0).getValueType();
45083       EVT SrcCondVT = SrcVT.changeVectorElementType(MVT::i1);
45084       LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
45085                             VT.getSizeInBits());
45086       RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
45087                             VT.getSizeInBits());
45088       Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
45089                          DAG.getUNDEF(SrcCondVT), Cond,
45090                          DAG.getIntPtrConstant(0, DL));
45091       SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
45092       return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
45093     }
45094   }
45095 
45096   if (SDValue V = combineSelectOfTwoConstants(N, DAG))
45097     return V;
45098 
45099   if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
45100       Cond.hasOneUse()) {
45101     EVT CondVT = Cond.getValueType();
45102     SDValue Cond0 = Cond.getOperand(0);
45103     SDValue Cond1 = Cond.getOperand(1);
45104     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
45105 
45106     // Canonicalize min/max:
45107     // (x > 0) ? x : 0 -> (x >= 0) ? x : 0
45108     // (x < -1) ? x : -1 -> (x <= -1) ? x : -1
45109     // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
45110     // the need for an extra compare against zero. e.g.
45111     // (a - b) > 0 : (a - b) ? 0 -> (a - b) >= 0 : (a - b) ? 0
45112     // subl   %esi, %edi
45113     // testl  %edi, %edi
45114     // movl   $0, %eax
45115     // cmovgl %edi, %eax
45116     // =>
45117     // xorl   %eax, %eax
45118     // subl   %esi, $edi
45119     // cmovsl %eax, %edi
45120     //
45121     // We can also canonicalize
45122     //  (x s> 1) ? x : 1 -> (x s>= 1) ? x : 1 -> (x s> 0) ? x : 1
45123     //  (x u> 1) ? x : 1 -> (x u>= 1) ? x : 1 -> (x != 0) ? x : 1
45124     // This allows the use of a test instruction for the compare.
45125     if (LHS == Cond0 && RHS == Cond1) {
45126       if ((CC == ISD::SETGT && (isNullConstant(RHS) || isOneConstant(RHS))) ||
45127           (CC == ISD::SETLT && isAllOnesConstant(RHS))) {
45128         ISD::CondCode NewCC = CC == ISD::SETGT ? ISD::SETGE : ISD::SETLE;
45129         Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
45130         return DAG.getSelect(DL, VT, Cond, LHS, RHS);
45131       }
45132       if (CC == ISD::SETUGT && isOneConstant(RHS)) {
45133         ISD::CondCode NewCC = ISD::SETUGE;
45134         Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
45135         return DAG.getSelect(DL, VT, Cond, LHS, RHS);
45136       }
45137     }
45138 
45139     // Similar to DAGCombine's select(or(CC0,CC1),X,Y) fold but for legal types.
45140     // fold eq + gt/lt nested selects into ge/le selects
45141     // select (cmpeq Cond0, Cond1), LHS, (select (cmpugt Cond0, Cond1), LHS, Y)
45142     // --> (select (cmpuge Cond0, Cond1), LHS, Y)
45143     // select (cmpslt Cond0, Cond1), LHS, (select (cmpeq Cond0, Cond1), LHS, Y)
45144     // --> (select (cmpsle Cond0, Cond1), LHS, Y)
45145     // .. etc ..
45146     if (RHS.getOpcode() == ISD::SELECT && RHS.getOperand(1) == LHS &&
45147         RHS.getOperand(0).getOpcode() == ISD::SETCC) {
45148       SDValue InnerSetCC = RHS.getOperand(0);
45149       ISD::CondCode InnerCC =
45150           cast<CondCodeSDNode>(InnerSetCC.getOperand(2))->get();
45151       if ((CC == ISD::SETEQ || InnerCC == ISD::SETEQ) &&
45152           Cond0 == InnerSetCC.getOperand(0) &&
45153           Cond1 == InnerSetCC.getOperand(1)) {
45154         ISD::CondCode NewCC;
45155         switch (CC == ISD::SETEQ ? InnerCC : CC) {
45156         case ISD::SETGT:  NewCC = ISD::SETGE; break;
45157         case ISD::SETLT:  NewCC = ISD::SETLE; break;
45158         case ISD::SETUGT: NewCC = ISD::SETUGE; break;
45159         case ISD::SETULT: NewCC = ISD::SETULE; break;
45160         default: NewCC = ISD::SETCC_INVALID; break;
45161         }
45162         if (NewCC != ISD::SETCC_INVALID) {
45163           Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
45164           return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2));
45165         }
45166       }
45167     }
45168   }
45169 
45170   // Check if the first operand is all zeros and Cond type is vXi1.
45171   // If this an avx512 target we can improve the use of zero masking by
45172   // swapping the operands and inverting the condition.
45173   if (N->getOpcode() == ISD::VSELECT && Cond.hasOneUse() &&
45174       Subtarget.hasAVX512() && CondVT.getVectorElementType() == MVT::i1 &&
45175       ISD::isBuildVectorAllZeros(LHS.getNode()) &&
45176       !ISD::isBuildVectorAllZeros(RHS.getNode())) {
45177     // Invert the cond to not(cond) : xor(op,allones)=not(op)
45178     SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
45179     // Vselect cond, op1, op2 = Vselect not(cond), op2, op1
45180     return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
45181   }
45182 
45183   // Attempt to convert a (vXi1 bitcast(iX Cond)) selection mask before it might
45184   // get split by legalization.
45185   if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::BITCAST &&
45186       CondVT.getVectorElementType() == MVT::i1 &&
45187       TLI.isTypeLegal(VT.getScalarType())) {
45188     EVT ExtCondVT = VT.changeVectorElementTypeToInteger();
45189     if (SDValue ExtCond = combineToExtendBoolVectorInReg(
45190             ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) {
45191       ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond);
45192       return DAG.getSelect(DL, VT, ExtCond, LHS, RHS);
45193     }
45194   }
45195 
45196   // Early exit check
45197   if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
45198     return SDValue();
45199 
45200   if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
45201     return V;
45202 
45203   if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
45204     return V;
45205 
45206   if (SDValue V = narrowVectorSelect(N, DAG, Subtarget))
45207     return V;
45208 
45209   // select(~Cond, X, Y) -> select(Cond, Y, X)
45210   if (CondVT.getScalarType() != MVT::i1) {
45211     if (SDValue CondNot = IsNOT(Cond, DAG))
45212       return DAG.getNode(N->getOpcode(), DL, VT,
45213                          DAG.getBitcast(CondVT, CondNot), RHS, LHS);
45214 
45215     // pcmpgt(X, -1) -> pcmpgt(0, X) to help select/blendv just use the
45216     // signbit.
45217     if (Cond.getOpcode() == X86ISD::PCMPGT &&
45218         ISD::isBuildVectorAllOnes(Cond.getOperand(1).getNode()) &&
45219         Cond.hasOneUse()) {
45220       Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT,
45221                          DAG.getConstant(0, DL, CondVT), Cond.getOperand(0));
45222       return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS);
45223     }
45224   }
45225 
45226   // Try to optimize vXi1 selects if both operands are either all constants or
45227   // bitcasts from scalar integer type. In that case we can convert the operands
45228   // to integer and use an integer select which will be converted to a CMOV.
45229   // We need to take a little bit of care to avoid creating an i64 type after
45230   // type legalization.
45231   if (N->getOpcode() == ISD::SELECT && VT.isVector() &&
45232       VT.getVectorElementType() == MVT::i1 &&
45233       (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) {
45234     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
45235     if (DCI.isBeforeLegalize() || TLI.isTypeLegal(IntVT)) {
45236       bool LHSIsConst = ISD::isBuildVectorOfConstantSDNodes(LHS.getNode());
45237       bool RHSIsConst = ISD::isBuildVectorOfConstantSDNodes(RHS.getNode());
45238 
45239       if ((LHSIsConst || (LHS.getOpcode() == ISD::BITCAST &&
45240                           LHS.getOperand(0).getValueType() == IntVT)) &&
45241           (RHSIsConst || (RHS.getOpcode() == ISD::BITCAST &&
45242                           RHS.getOperand(0).getValueType() == IntVT))) {
45243         if (LHSIsConst)
45244           LHS = combinevXi1ConstantToInteger(LHS, DAG);
45245         else
45246           LHS = LHS.getOperand(0);
45247 
45248         if (RHSIsConst)
45249           RHS = combinevXi1ConstantToInteger(RHS, DAG);
45250         else
45251           RHS = RHS.getOperand(0);
45252 
45253         SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
45254         return DAG.getBitcast(VT, Select);
45255       }
45256     }
45257   }
45258 
45259   // If this is "((X & C) == 0) ? Y : Z" and C is a constant mask vector of
45260   // single bits, then invert the predicate and swap the select operands.
45261   // This can lower using a vector shift bit-hack rather than mask and compare.
45262   if (DCI.isBeforeLegalize() && !Subtarget.hasAVX512() &&
45263       N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
45264       Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1 &&
45265       Cond.getOperand(0).getOpcode() == ISD::AND &&
45266       isNullOrNullSplat(Cond.getOperand(1)) &&
45267       cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
45268       Cond.getOperand(0).getValueType() == VT) {
45269     // The 'and' mask must be composed of power-of-2 constants.
45270     SDValue And = Cond.getOperand(0);
45271     auto *C = isConstOrConstSplat(And.getOperand(1));
45272     if (C && C->getAPIntValue().isPowerOf2()) {
45273       // vselect (X & C == 0), LHS, RHS --> vselect (X & C != 0), RHS, LHS
45274       SDValue NotCond =
45275           DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
45276       return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
45277     }
45278 
45279     // If we have a non-splat but still powers-of-2 mask, AVX1 can use pmulld
45280     // and AVX2 can use vpsllv{dq}. 8-bit lacks a proper shift or multiply.
45281     // 16-bit lacks a proper blendv.
45282     unsigned EltBitWidth = VT.getScalarSizeInBits();
45283     bool CanShiftBlend =
45284         TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
45285                                 (Subtarget.hasAVX2() && EltBitWidth == 64) ||
45286                                 (Subtarget.hasXOP()));
45287     if (CanShiftBlend &&
45288         ISD::matchUnaryPredicate(And.getOperand(1), [](ConstantSDNode *C) {
45289           return C->getAPIntValue().isPowerOf2();
45290         })) {
45291       // Create a left-shift constant to get the mask bits over to the sign-bit.
45292       SDValue Mask = And.getOperand(1);
45293       SmallVector<int, 32> ShlVals;
45294       for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
45295         auto *MaskVal = cast<ConstantSDNode>(Mask.getOperand(i));
45296         ShlVals.push_back(EltBitWidth - 1 -
45297                           MaskVal->getAPIntValue().exactLogBase2());
45298       }
45299       // vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
45300       SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
45301       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
45302       SDValue NewCond =
45303           DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
45304       return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
45305     }
45306   }
45307 
45308   return SDValue();
45309 }
45310 
45311 /// Combine:
45312 ///   (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S)
45313 /// to:
45314 ///   (brcond/cmov/setcc .., (LADD x, 1), COND_LE)
45315 /// i.e., reusing the EFLAGS produced by the LOCKed instruction.
45316 /// Note that this is only legal for some op/cc combinations.
45317 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
45318                                        SelectionDAG &DAG,
45319                                        const X86Subtarget &Subtarget) {
45320   // This combine only operates on CMP-like nodes.
45321   if (!(Cmp.getOpcode() == X86ISD::CMP ||
45322         (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
45323     return SDValue();
45324 
45325   // Can't replace the cmp if it has more uses than the one we're looking at.
45326   // FIXME: We would like to be able to handle this, but would need to make sure
45327   // all uses were updated.
45328   if (!Cmp.hasOneUse())
45329     return SDValue();
45330 
45331   // This only applies to variations of the common case:
45332   //   (icmp slt x, 0) -> (icmp sle (add x, 1), 0)
45333   //   (icmp sge x, 0) -> (icmp sgt (add x, 1), 0)
45334   //   (icmp sle x, 0) -> (icmp slt (sub x, 1), 0)
45335   //   (icmp sgt x, 0) -> (icmp sge (sub x, 1), 0)
45336   // Using the proper condcodes (see below), overflow is checked for.
45337 
45338   // FIXME: We can generalize both constraints:
45339   // - XOR/OR/AND (if they were made to survive AtomicExpand)
45340   // - LHS != 1
45341   // if the result is compared.
45342 
45343   SDValue CmpLHS = Cmp.getOperand(0);
45344   SDValue CmpRHS = Cmp.getOperand(1);
45345   EVT CmpVT = CmpLHS.getValueType();
45346 
45347   if (!CmpLHS.hasOneUse())
45348     return SDValue();
45349 
45350   unsigned Opc = CmpLHS.getOpcode();
45351   if (Opc != ISD::ATOMIC_LOAD_ADD && Opc != ISD::ATOMIC_LOAD_SUB)
45352     return SDValue();
45353 
45354   SDValue OpRHS = CmpLHS.getOperand(2);
45355   auto *OpRHSC = dyn_cast<ConstantSDNode>(OpRHS);
45356   if (!OpRHSC)
45357     return SDValue();
45358 
45359   APInt Addend = OpRHSC->getAPIntValue();
45360   if (Opc == ISD::ATOMIC_LOAD_SUB)
45361     Addend = -Addend;
45362 
45363   auto *CmpRHSC = dyn_cast<ConstantSDNode>(CmpRHS);
45364   if (!CmpRHSC)
45365     return SDValue();
45366 
45367   APInt Comparison = CmpRHSC->getAPIntValue();
45368   APInt NegAddend = -Addend;
45369 
45370   // See if we can adjust the CC to make the comparison match the negated
45371   // addend.
45372   if (Comparison != NegAddend) {
45373     APInt IncComparison = Comparison + 1;
45374     if (IncComparison == NegAddend) {
45375       if (CC == X86::COND_A && !Comparison.isMaxValue()) {
45376         Comparison = IncComparison;
45377         CC = X86::COND_AE;
45378       } else if (CC == X86::COND_LE && !Comparison.isMaxSignedValue()) {
45379         Comparison = IncComparison;
45380         CC = X86::COND_L;
45381       }
45382     }
45383     APInt DecComparison = Comparison - 1;
45384     if (DecComparison == NegAddend) {
45385       if (CC == X86::COND_AE && !Comparison.isMinValue()) {
45386         Comparison = DecComparison;
45387         CC = X86::COND_A;
45388       } else if (CC == X86::COND_L && !Comparison.isMinSignedValue()) {
45389         Comparison = DecComparison;
45390         CC = X86::COND_LE;
45391       }
45392     }
45393   }
45394 
45395   // If the addend is the negation of the comparison value, then we can do
45396   // a full comparison by emitting the atomic arithmetic as a locked sub.
45397   if (Comparison == NegAddend) {
45398     // The CC is fine, but we need to rewrite the LHS of the comparison as an
45399     // atomic sub.
45400     auto *AN = cast<AtomicSDNode>(CmpLHS.getNode());
45401     auto AtomicSub = DAG.getAtomic(
45402         ISD::ATOMIC_LOAD_SUB, SDLoc(CmpLHS), CmpVT,
45403         /*Chain*/ CmpLHS.getOperand(0), /*LHS*/ CmpLHS.getOperand(1),
45404         /*RHS*/ DAG.getConstant(NegAddend, SDLoc(CmpRHS), CmpVT),
45405         AN->getMemOperand());
45406     auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
45407     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
45408     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
45409     return LockOp;
45410   }
45411 
45412   // We can handle comparisons with zero in a number of cases by manipulating
45413   // the CC used.
45414   if (!Comparison.isZero())
45415     return SDValue();
45416 
45417   if (CC == X86::COND_S && Addend == 1)
45418     CC = X86::COND_LE;
45419   else if (CC == X86::COND_NS && Addend == 1)
45420     CC = X86::COND_G;
45421   else if (CC == X86::COND_G && Addend == -1)
45422     CC = X86::COND_GE;
45423   else if (CC == X86::COND_LE && Addend == -1)
45424     CC = X86::COND_L;
45425   else
45426     return SDValue();
45427 
45428   SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
45429   DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
45430   DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
45431   return LockOp;
45432 }
45433 
45434 // Check whether a boolean test is testing a boolean value generated by
45435 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
45436 // code.
45437 //
45438 // Simplify the following patterns:
45439 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
45440 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
45441 // to (Op EFLAGS Cond)
45442 //
45443 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
45444 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
45445 // to (Op EFLAGS !Cond)
45446 //
45447 // where Op could be BRCOND or CMOV.
45448 //
45449 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
45450   // This combine only operates on CMP-like nodes.
45451   if (!(Cmp.getOpcode() == X86ISD::CMP ||
45452         (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
45453     return SDValue();
45454 
45455   // Quit if not used as a boolean value.
45456   if (CC != X86::COND_E && CC != X86::COND_NE)
45457     return SDValue();
45458 
45459   // Check CMP operands. One of them should be 0 or 1 and the other should be
45460   // an SetCC or extended from it.
45461   SDValue Op1 = Cmp.getOperand(0);
45462   SDValue Op2 = Cmp.getOperand(1);
45463 
45464   SDValue SetCC;
45465   const ConstantSDNode* C = nullptr;
45466   bool needOppositeCond = (CC == X86::COND_E);
45467   bool checkAgainstTrue = false; // Is it a comparison against 1?
45468 
45469   if ((C = dyn_cast<ConstantSDNode>(Op1)))
45470     SetCC = Op2;
45471   else if ((C = dyn_cast<ConstantSDNode>(Op2)))
45472     SetCC = Op1;
45473   else // Quit if all operands are not constants.
45474     return SDValue();
45475 
45476   if (C->getZExtValue() == 1) {
45477     needOppositeCond = !needOppositeCond;
45478     checkAgainstTrue = true;
45479   } else if (C->getZExtValue() != 0)
45480     // Quit if the constant is neither 0 or 1.
45481     return SDValue();
45482 
45483   bool truncatedToBoolWithAnd = false;
45484   // Skip (zext $x), (trunc $x), or (and $x, 1) node.
45485   while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
45486          SetCC.getOpcode() == ISD::TRUNCATE ||
45487          SetCC.getOpcode() == ISD::AND) {
45488     if (SetCC.getOpcode() == ISD::AND) {
45489       int OpIdx = -1;
45490       if (isOneConstant(SetCC.getOperand(0)))
45491         OpIdx = 1;
45492       if (isOneConstant(SetCC.getOperand(1)))
45493         OpIdx = 0;
45494       if (OpIdx < 0)
45495         break;
45496       SetCC = SetCC.getOperand(OpIdx);
45497       truncatedToBoolWithAnd = true;
45498     } else
45499       SetCC = SetCC.getOperand(0);
45500   }
45501 
45502   switch (SetCC.getOpcode()) {
45503   case X86ISD::SETCC_CARRY:
45504     // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
45505     // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
45506     // i.e. it's a comparison against true but the result of SETCC_CARRY is not
45507     // truncated to i1 using 'and'.
45508     if (checkAgainstTrue && !truncatedToBoolWithAnd)
45509       break;
45510     assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
45511            "Invalid use of SETCC_CARRY!");
45512     [[fallthrough]];
45513   case X86ISD::SETCC:
45514     // Set the condition code or opposite one if necessary.
45515     CC = X86::CondCode(SetCC.getConstantOperandVal(0));
45516     if (needOppositeCond)
45517       CC = X86::GetOppositeBranchCondition(CC);
45518     return SetCC.getOperand(1);
45519   case X86ISD::CMOV: {
45520     // Check whether false/true value has canonical one, i.e. 0 or 1.
45521     ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
45522     ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
45523     // Quit if true value is not a constant.
45524     if (!TVal)
45525       return SDValue();
45526     // Quit if false value is not a constant.
45527     if (!FVal) {
45528       SDValue Op = SetCC.getOperand(0);
45529       // Skip 'zext' or 'trunc' node.
45530       if (Op.getOpcode() == ISD::ZERO_EXTEND ||
45531           Op.getOpcode() == ISD::TRUNCATE)
45532         Op = Op.getOperand(0);
45533       // A special case for rdrand/rdseed, where 0 is set if false cond is
45534       // found.
45535       if ((Op.getOpcode() != X86ISD::RDRAND &&
45536            Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
45537         return SDValue();
45538     }
45539     // Quit if false value is not the constant 0 or 1.
45540     bool FValIsFalse = true;
45541     if (FVal && FVal->getZExtValue() != 0) {
45542       if (FVal->getZExtValue() != 1)
45543         return SDValue();
45544       // If FVal is 1, opposite cond is needed.
45545       needOppositeCond = !needOppositeCond;
45546       FValIsFalse = false;
45547     }
45548     // Quit if TVal is not the constant opposite of FVal.
45549     if (FValIsFalse && TVal->getZExtValue() != 1)
45550       return SDValue();
45551     if (!FValIsFalse && TVal->getZExtValue() != 0)
45552       return SDValue();
45553     CC = X86::CondCode(SetCC.getConstantOperandVal(2));
45554     if (needOppositeCond)
45555       CC = X86::GetOppositeBranchCondition(CC);
45556     return SetCC.getOperand(3);
45557   }
45558   }
45559 
45560   return SDValue();
45561 }
45562 
45563 /// Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
45564 /// Match:
45565 ///   (X86or (X86setcc) (X86setcc))
45566 ///   (X86cmp (and (X86setcc) (X86setcc)), 0)
45567 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
45568                                            X86::CondCode &CC1, SDValue &Flags,
45569                                            bool &isAnd) {
45570   if (Cond->getOpcode() == X86ISD::CMP) {
45571     if (!isNullConstant(Cond->getOperand(1)))
45572       return false;
45573 
45574     Cond = Cond->getOperand(0);
45575   }
45576 
45577   isAnd = false;
45578 
45579   SDValue SetCC0, SetCC1;
45580   switch (Cond->getOpcode()) {
45581   default: return false;
45582   case ISD::AND:
45583   case X86ISD::AND:
45584     isAnd = true;
45585     [[fallthrough]];
45586   case ISD::OR:
45587   case X86ISD::OR:
45588     SetCC0 = Cond->getOperand(0);
45589     SetCC1 = Cond->getOperand(1);
45590     break;
45591   };
45592 
45593   // Make sure we have SETCC nodes, using the same flags value.
45594   if (SetCC0.getOpcode() != X86ISD::SETCC ||
45595       SetCC1.getOpcode() != X86ISD::SETCC ||
45596       SetCC0->getOperand(1) != SetCC1->getOperand(1))
45597     return false;
45598 
45599   CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
45600   CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
45601   Flags = SetCC0->getOperand(1);
45602   return true;
45603 }
45604 
45605 // When legalizing carry, we create carries via add X, -1
45606 // If that comes from an actual carry, via setcc, we use the
45607 // carry directly.
45608 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
45609   if (EFLAGS.getOpcode() == X86ISD::ADD) {
45610     if (isAllOnesConstant(EFLAGS.getOperand(1))) {
45611       bool FoundAndLSB = false;
45612       SDValue Carry = EFLAGS.getOperand(0);
45613       while (Carry.getOpcode() == ISD::TRUNCATE ||
45614              Carry.getOpcode() == ISD::ZERO_EXTEND ||
45615              (Carry.getOpcode() == ISD::AND &&
45616               isOneConstant(Carry.getOperand(1)))) {
45617         FoundAndLSB |= Carry.getOpcode() == ISD::AND;
45618         Carry = Carry.getOperand(0);
45619       }
45620       if (Carry.getOpcode() == X86ISD::SETCC ||
45621           Carry.getOpcode() == X86ISD::SETCC_CARRY) {
45622         // TODO: Merge this code with equivalent in combineAddOrSubToADCOrSBB?
45623         uint64_t CarryCC = Carry.getConstantOperandVal(0);
45624         SDValue CarryOp1 = Carry.getOperand(1);
45625         if (CarryCC == X86::COND_B)
45626           return CarryOp1;
45627         if (CarryCC == X86::COND_A) {
45628           // Try to convert COND_A into COND_B in an attempt to facilitate
45629           // materializing "setb reg".
45630           //
45631           // Do not flip "e > c", where "c" is a constant, because Cmp
45632           // instruction cannot take an immediate as its first operand.
45633           //
45634           if (CarryOp1.getOpcode() == X86ISD::SUB &&
45635               CarryOp1.getNode()->hasOneUse() &&
45636               CarryOp1.getValueType().isInteger() &&
45637               !isa<ConstantSDNode>(CarryOp1.getOperand(1))) {
45638             SDValue SubCommute =
45639                 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
45640                             CarryOp1.getOperand(1), CarryOp1.getOperand(0));
45641             return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
45642           }
45643         }
45644         // If this is a check of the z flag of an add with 1, switch to the
45645         // C flag.
45646         if (CarryCC == X86::COND_E &&
45647             CarryOp1.getOpcode() == X86ISD::ADD &&
45648             isOneConstant(CarryOp1.getOperand(1)))
45649           return CarryOp1;
45650       } else if (FoundAndLSB) {
45651         SDLoc DL(Carry);
45652         SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType());
45653         if (Carry.getOpcode() == ISD::SRL) {
45654           BitNo = Carry.getOperand(1);
45655           Carry = Carry.getOperand(0);
45656         }
45657         return getBT(Carry, BitNo, DL, DAG);
45658       }
45659     }
45660   }
45661 
45662   return SDValue();
45663 }
45664 
45665 /// If we are inverting an PTEST/TESTP operand, attempt to adjust the CC
45666 /// to avoid the inversion.
45667 static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
45668                               SelectionDAG &DAG,
45669                               const X86Subtarget &Subtarget) {
45670   // TODO: Handle X86ISD::KTEST/X86ISD::KORTEST.
45671   if (EFLAGS.getOpcode() != X86ISD::PTEST &&
45672       EFLAGS.getOpcode() != X86ISD::TESTP)
45673     return SDValue();
45674 
45675   // PTEST/TESTP sets EFLAGS as:
45676   // TESTZ: ZF = (Op0 & Op1) == 0
45677   // TESTC: CF = (~Op0 & Op1) == 0
45678   // TESTNZC: ZF == 0 && CF == 0
45679   MVT VT = EFLAGS.getSimpleValueType();
45680   SDValue Op0 = EFLAGS.getOperand(0);
45681   SDValue Op1 = EFLAGS.getOperand(1);
45682   MVT OpVT = Op0.getSimpleValueType();
45683 
45684   // TEST*(~X,Y) == TEST*(X,Y)
45685   if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
45686     X86::CondCode InvCC;
45687     switch (CC) {
45688     case X86::COND_B:
45689       // testc -> testz.
45690       InvCC = X86::COND_E;
45691       break;
45692     case X86::COND_AE:
45693       // !testc -> !testz.
45694       InvCC = X86::COND_NE;
45695       break;
45696     case X86::COND_E:
45697       // testz -> testc.
45698       InvCC = X86::COND_B;
45699       break;
45700     case X86::COND_NE:
45701       // !testz -> !testc.
45702       InvCC = X86::COND_AE;
45703       break;
45704     case X86::COND_A:
45705     case X86::COND_BE:
45706       // testnzc -> testnzc (no change).
45707       InvCC = CC;
45708       break;
45709     default:
45710       InvCC = X86::COND_INVALID;
45711       break;
45712     }
45713 
45714     if (InvCC != X86::COND_INVALID) {
45715       CC = InvCC;
45716       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45717                          DAG.getBitcast(OpVT, NotOp0), Op1);
45718     }
45719   }
45720 
45721   if (CC == X86::COND_B || CC == X86::COND_AE) {
45722     // TESTC(X,~X) == TESTC(X,-1)
45723     if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
45724       if (peekThroughBitcasts(NotOp1) == peekThroughBitcasts(Op0)) {
45725         SDLoc DL(EFLAGS);
45726         return DAG.getNode(
45727             EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1),
45728             DAG.getBitcast(OpVT,
45729                            DAG.getAllOnesConstant(DL, NotOp1.getValueType())));
45730       }
45731     }
45732   }
45733 
45734   if (CC == X86::COND_E || CC == X86::COND_NE) {
45735     // TESTZ(X,~Y) == TESTC(Y,X)
45736     if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
45737       CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
45738       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45739                          DAG.getBitcast(OpVT, NotOp1), Op0);
45740     }
45741 
45742     if (Op0 == Op1) {
45743       SDValue BC = peekThroughBitcasts(Op0);
45744       EVT BCVT = BC.getValueType();
45745 
45746       // TESTZ(AND(X,Y),AND(X,Y)) == TESTZ(X,Y)
45747       if (BC.getOpcode() == ISD::AND || BC.getOpcode() == X86ISD::FAND) {
45748         return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45749                            DAG.getBitcast(OpVT, BC.getOperand(0)),
45750                            DAG.getBitcast(OpVT, BC.getOperand(1)));
45751       }
45752 
45753       // TESTZ(AND(~X,Y),AND(~X,Y)) == TESTC(X,Y)
45754       if (BC.getOpcode() == X86ISD::ANDNP || BC.getOpcode() == X86ISD::FANDN) {
45755         CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
45756         return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45757                            DAG.getBitcast(OpVT, BC.getOperand(0)),
45758                            DAG.getBitcast(OpVT, BC.getOperand(1)));
45759       }
45760 
45761       // If every element is an all-sign value, see if we can use TESTP/MOVMSK
45762       // to more efficiently extract the sign bits and compare that.
45763       // TODO: Handle TESTC with comparison inversion.
45764       // TODO: Can we remove SimplifyMultipleUseDemandedBits and rely on
45765       // TESTP/MOVMSK combines to make sure its never worse than PTEST?
45766       if (BCVT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(BCVT)) {
45767         unsigned EltBits = BCVT.getScalarSizeInBits();
45768         if (DAG.ComputeNumSignBits(BC) == EltBits) {
45769           assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result");
45770           APInt SignMask = APInt::getSignMask(EltBits);
45771           const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45772           if (SDValue Res =
45773                   TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) {
45774             // For vXi16 cases we need to use pmovmksb and extract every other
45775             // sign bit.
45776             SDLoc DL(EFLAGS);
45777             if ((EltBits == 32 || EltBits == 64) && Subtarget.hasAVX()) {
45778               MVT FloatSVT = MVT::getFloatingPointVT(EltBits);
45779               MVT FloatVT =
45780                   MVT::getVectorVT(FloatSVT, OpVT.getSizeInBits() / EltBits);
45781               Res = DAG.getBitcast(FloatVT, Res);
45782               return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res);
45783             } else if (EltBits == 16) {
45784               MVT MovmskVT = BCVT.is128BitVector() ? MVT::v16i8 : MVT::v32i8;
45785               Res = DAG.getBitcast(MovmskVT, Res);
45786               Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
45787               Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,
45788                                 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
45789             } else {
45790               Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
45791             }
45792             return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,
45793                                DAG.getConstant(0, DL, MVT::i32));
45794           }
45795         }
45796       }
45797     }
45798 
45799     // TESTZ(-1,X) == TESTZ(X,X)
45800     if (ISD::isBuildVectorAllOnes(Op0.getNode()))
45801       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1);
45802 
45803     // TESTZ(X,-1) == TESTZ(X,X)
45804     if (ISD::isBuildVectorAllOnes(Op1.getNode()))
45805       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0);
45806 
45807     // TESTZ(OR(LO(X),HI(X)),OR(LO(Y),HI(Y))) -> TESTZ(X,Y)
45808     // TODO: Add COND_NE handling?
45809     if (CC == X86::COND_E && OpVT.is128BitVector() && Subtarget.hasAVX()) {
45810       SDValue Src0 = peekThroughBitcasts(Op0);
45811       SDValue Src1 = peekThroughBitcasts(Op1);
45812       if (Src0.getOpcode() == ISD::OR && Src1.getOpcode() == ISD::OR) {
45813         Src0 = getSplitVectorSrc(peekThroughBitcasts(Src0.getOperand(0)),
45814                                  peekThroughBitcasts(Src0.getOperand(1)), true);
45815         Src1 = getSplitVectorSrc(peekThroughBitcasts(Src1.getOperand(0)),
45816                                  peekThroughBitcasts(Src1.getOperand(1)), true);
45817         if (Src0 && Src1) {
45818           MVT OpVT2 = OpVT.getDoubleNumVectorElementsVT();
45819           return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45820                              DAG.getBitcast(OpVT2, Src0),
45821                              DAG.getBitcast(OpVT2, Src1));
45822         }
45823       }
45824     }
45825   }
45826 
45827   return SDValue();
45828 }
45829 
45830 // Attempt to simplify the MOVMSK input based on the comparison type.
45831 static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
45832                                   SelectionDAG &DAG,
45833                                   const X86Subtarget &Subtarget) {
45834   // Handle eq/ne against zero (any_of).
45835   // Handle eq/ne against -1 (all_of).
45836   if (!(CC == X86::COND_E || CC == X86::COND_NE))
45837     return SDValue();
45838   if (EFLAGS.getValueType() != MVT::i32)
45839     return SDValue();
45840   unsigned CmpOpcode = EFLAGS.getOpcode();
45841   if (CmpOpcode != X86ISD::CMP && CmpOpcode != X86ISD::SUB)
45842     return SDValue();
45843   auto *CmpConstant = dyn_cast<ConstantSDNode>(EFLAGS.getOperand(1));
45844   if (!CmpConstant)
45845     return SDValue();
45846   const APInt &CmpVal = CmpConstant->getAPIntValue();
45847 
45848   SDValue CmpOp = EFLAGS.getOperand(0);
45849   unsigned CmpBits = CmpOp.getValueSizeInBits();
45850   assert(CmpBits == CmpVal.getBitWidth() && "Value size mismatch");
45851 
45852   // Peek through any truncate.
45853   if (CmpOp.getOpcode() == ISD::TRUNCATE)
45854     CmpOp = CmpOp.getOperand(0);
45855 
45856   // Bail if we don't find a MOVMSK.
45857   if (CmpOp.getOpcode() != X86ISD::MOVMSK)
45858     return SDValue();
45859 
45860   SDValue Vec = CmpOp.getOperand(0);
45861   MVT VecVT = Vec.getSimpleValueType();
45862   assert((VecVT.is128BitVector() || VecVT.is256BitVector()) &&
45863          "Unexpected MOVMSK operand");
45864   unsigned NumElts = VecVT.getVectorNumElements();
45865   unsigned NumEltBits = VecVT.getScalarSizeInBits();
45866 
45867   bool IsAnyOf = CmpOpcode == X86ISD::CMP && CmpVal.isZero();
45868   bool IsAllOf = (CmpOpcode == X86ISD::SUB || CmpOpcode == X86ISD::CMP) &&
45869                  NumElts <= CmpBits && CmpVal.isMask(NumElts);
45870   if (!IsAnyOf && !IsAllOf)
45871     return SDValue();
45872 
45873   // TODO: Check more combining cases for me.
45874   // Here we check the cmp use number to decide do combining or not.
45875   // Currently we only get 2 tests about combining "MOVMSK(CONCAT(..))"
45876   // and "MOVMSK(PCMPEQ(..))" are fit to use this constraint.
45877   bool IsOneUse = CmpOp.getNode()->hasOneUse();
45878 
45879   // See if we can peek through to a vector with a wider element type, if the
45880   // signbits extend down to all the sub-elements as well.
45881   // Calling MOVMSK with the wider type, avoiding the bitcast, helps expose
45882   // potential SimplifyDemandedBits/Elts cases.
45883   // If we looked through a truncate that discard bits, we can't do this
45884   // transform.
45885   // FIXME: We could do this transform for truncates that discarded bits by
45886   // inserting an AND mask between the new MOVMSK and the CMP.
45887   if (Vec.getOpcode() == ISD::BITCAST && NumElts <= CmpBits) {
45888     SDValue BC = peekThroughBitcasts(Vec);
45889     MVT BCVT = BC.getSimpleValueType();
45890     unsigned BCNumElts = BCVT.getVectorNumElements();
45891     unsigned BCNumEltBits = BCVT.getScalarSizeInBits();
45892     if ((BCNumEltBits == 32 || BCNumEltBits == 64) &&
45893         BCNumEltBits > NumEltBits &&
45894         DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) {
45895       SDLoc DL(EFLAGS);
45896       APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : BCNumElts);
45897       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
45898                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),
45899                          DAG.getConstant(CmpMask, DL, MVT::i32));
45900     }
45901   }
45902 
45903   // MOVMSK(CONCAT(X,Y)) == 0 ->  MOVMSK(OR(X,Y)).
45904   // MOVMSK(CONCAT(X,Y)) != 0 ->  MOVMSK(OR(X,Y)).
45905   // MOVMSK(CONCAT(X,Y)) == -1 ->  MOVMSK(AND(X,Y)).
45906   // MOVMSK(CONCAT(X,Y)) != -1 ->  MOVMSK(AND(X,Y)).
45907   if (VecVT.is256BitVector() && NumElts <= CmpBits && IsOneUse) {
45908     SmallVector<SDValue> Ops;
45909     if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) &&
45910         Ops.size() == 2) {
45911       SDLoc DL(EFLAGS);
45912       EVT SubVT = Ops[0].getValueType().changeTypeToInteger();
45913       APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : NumElts / 2);
45914       SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT,
45915                               DAG.getBitcast(SubVT, Ops[0]),
45916                               DAG.getBitcast(SubVT, Ops[1]));
45917       V = DAG.getBitcast(VecVT.getHalfNumVectorElementsVT(), V);
45918       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
45919                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V),
45920                          DAG.getConstant(CmpMask, DL, MVT::i32));
45921     }
45922   }
45923 
45924   // MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X).
45925   // MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X).
45926   // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(XOR(X,Y),XOR(X,Y)).
45927   // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(XOR(X,Y),XOR(X,Y)).
45928   if (IsAllOf && Subtarget.hasSSE41() && IsOneUse) {
45929     MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
45930     SDValue BC = peekThroughBitcasts(Vec);
45931     // Ensure MOVMSK was testing every signbit of BC.
45932     if (BC.getValueType().getVectorNumElements() <= NumElts) {
45933       if (BC.getOpcode() == X86ISD::PCMPEQ) {
45934         SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(),
45935                                 BC.getOperand(0), BC.getOperand(1));
45936         V = DAG.getBitcast(TestVT, V);
45937         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
45938       }
45939       // Check for 256-bit split vector cases.
45940       if (BC.getOpcode() == ISD::AND &&
45941           BC.getOperand(0).getOpcode() == X86ISD::PCMPEQ &&
45942           BC.getOperand(1).getOpcode() == X86ISD::PCMPEQ) {
45943         SDValue LHS = BC.getOperand(0);
45944         SDValue RHS = BC.getOperand(1);
45945         LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), LHS.getValueType(),
45946                           LHS.getOperand(0), LHS.getOperand(1));
45947         RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), RHS.getValueType(),
45948                           RHS.getOperand(0), RHS.getOperand(1));
45949         LHS = DAG.getBitcast(TestVT, LHS);
45950         RHS = DAG.getBitcast(TestVT, RHS);
45951         SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS);
45952         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
45953       }
45954     }
45955   }
45956 
45957   // See if we can avoid a PACKSS by calling MOVMSK on the sources.
45958   // For vXi16 cases we can use a v2Xi8 PMOVMSKB. We must mask out
45959   // sign bits prior to the comparison with zero unless we know that
45960   // the vXi16 splats the sign bit down to the lower i8 half.
45961   // TODO: Handle all_of patterns.
45962   if (Vec.getOpcode() == X86ISD::PACKSS && VecVT == MVT::v16i8) {
45963     SDValue VecOp0 = Vec.getOperand(0);
45964     SDValue VecOp1 = Vec.getOperand(1);
45965     bool SignExt0 = DAG.ComputeNumSignBits(VecOp0) > 8;
45966     bool SignExt1 = DAG.ComputeNumSignBits(VecOp1) > 8;
45967     // PMOVMSKB(PACKSSBW(X, undef)) -> PMOVMSKB(BITCAST_v16i8(X)) & 0xAAAA.
45968     if (IsAnyOf && CmpBits == 8 && VecOp1.isUndef()) {
45969       SDLoc DL(EFLAGS);
45970       SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
45971       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
45972       Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);
45973       if (!SignExt0) {
45974         Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,
45975                              DAG.getConstant(0xAAAA, DL, MVT::i16));
45976       }
45977       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
45978                          DAG.getConstant(0, DL, MVT::i16));
45979     }
45980     // PMOVMSKB(PACKSSBW(LO(X), HI(X)))
45981     // -> PMOVMSKB(BITCAST_v32i8(X)) & 0xAAAAAAAA.
45982     if (CmpBits >= 16 && Subtarget.hasInt256() &&
45983         (IsAnyOf || (SignExt0 && SignExt1))) {
45984       if (SDValue Src = getSplitVectorSrc(VecOp0, VecOp1, true)) {
45985         SDLoc DL(EFLAGS);
45986         SDValue Result = peekThroughBitcasts(Src);
45987         if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ &&
45988             Result.getValueType().getVectorNumElements() <= NumElts) {
45989           SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(),
45990                                   Result.getOperand(0), Result.getOperand(1));
45991           V = DAG.getBitcast(MVT::v4i64, V);
45992           return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
45993         }
45994         Result = DAG.getBitcast(MVT::v32i8, Result);
45995         Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
45996         unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF;
45997         if (!SignExt0 || !SignExt1) {
45998           assert(IsAnyOf &&
45999                  "Only perform v16i16 signmasks for any_of patterns");
46000           Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
46001                                DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
46002         }
46003         return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
46004                            DAG.getConstant(CmpMask, DL, MVT::i32));
46005       }
46006     }
46007   }
46008 
46009   // MOVMSK(SHUFFLE(X,u)) -> MOVMSK(X) iff every element is referenced.
46010   // Since we peek through a bitcast, we need to be careful if the base vector
46011   // type has smaller elements than the MOVMSK type.  In that case, even if
46012   // all the elements are demanded by the shuffle mask, only the "high"
46013   // elements which have highbits that align with highbits in the MOVMSK vec
46014   // elements are actually demanded. A simplification of spurious operations
46015   // on the "low" elements take place during other simplifications.
46016   //
46017   // For example:
46018   // MOVMSK64(BITCAST(SHUF32 X, (1,0,3,2))) even though all the elements are
46019   // demanded, because we are swapping around the result can change.
46020   //
46021   // To address this, we check that we can scale the shuffle mask to MOVMSK
46022   // element width (this will ensure "high" elements match). Its slightly overly
46023   // conservative, but fine for an edge case fold.
46024   SmallVector<int, 32> ShuffleMask, ScaledMaskUnused;
46025   SmallVector<SDValue, 2> ShuffleInputs;
46026   if (NumElts <= CmpBits &&
46027       getTargetShuffleInputs(peekThroughBitcasts(Vec), ShuffleInputs,
46028                              ShuffleMask, DAG) &&
46029       ShuffleInputs.size() == 1 && !isAnyZeroOrUndef(ShuffleMask) &&
46030       ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits() &&
46031       scaleShuffleElements(ShuffleMask, NumElts, ScaledMaskUnused)) {
46032     unsigned NumShuffleElts = ShuffleMask.size();
46033     APInt DemandedElts = APInt::getZero(NumShuffleElts);
46034     for (int M : ShuffleMask) {
46035       assert(0 <= M && M < (int)NumShuffleElts && "Bad unary shuffle index");
46036       DemandedElts.setBit(M);
46037     }
46038     if (DemandedElts.isAllOnes()) {
46039       SDLoc DL(EFLAGS);
46040       SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
46041       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
46042       Result =
46043           DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
46044       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
46045                          EFLAGS.getOperand(1));
46046     }
46047   }
46048 
46049   // MOVMSKPS(V) !=/== 0 -> TESTPS(V,V)
46050   // MOVMSKPD(V) !=/== 0 -> TESTPD(V,V)
46051   // MOVMSKPS(V) !=/== -1 -> TESTPS(V,V)
46052   // MOVMSKPD(V) !=/== -1 -> TESTPD(V,V)
46053   // iff every element is referenced.
46054   if (NumElts <= CmpBits && Subtarget.hasAVX() &&
46055       !Subtarget.preferMovmskOverVTest() && IsOneUse &&
46056       (NumEltBits == 32 || NumEltBits == 64)) {
46057     SDLoc DL(EFLAGS);
46058     MVT FloatSVT = MVT::getFloatingPointVT(NumEltBits);
46059     MVT FloatVT = MVT::getVectorVT(FloatSVT, NumElts);
46060     MVT IntVT = FloatVT.changeVectorElementTypeToInteger();
46061     SDValue LHS = Vec;
46062     SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT);
46063     CC = IsAnyOf ? CC : (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
46064     return DAG.getNode(X86ISD::TESTP, DL, MVT::i32,
46065                        DAG.getBitcast(FloatVT, LHS),
46066                        DAG.getBitcast(FloatVT, RHS));
46067   }
46068 
46069   return SDValue();
46070 }
46071 
46072 /// Optimize an EFLAGS definition used according to the condition code \p CC
46073 /// into a simpler EFLAGS value, potentially returning a new \p CC and replacing
46074 /// uses of chain values.
46075 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
46076                                   SelectionDAG &DAG,
46077                                   const X86Subtarget &Subtarget) {
46078   if (CC == X86::COND_B)
46079     if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
46080       return Flags;
46081 
46082   if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
46083     return R;
46084 
46085   if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
46086     return R;
46087 
46088   if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
46089     return R;
46090 
46091   return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
46092 }
46093 
46094 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
46095 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
46096                            TargetLowering::DAGCombinerInfo &DCI,
46097                            const X86Subtarget &Subtarget) {
46098   SDLoc DL(N);
46099 
46100   SDValue FalseOp = N->getOperand(0);
46101   SDValue TrueOp = N->getOperand(1);
46102   X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
46103   SDValue Cond = N->getOperand(3);
46104 
46105   // cmov X, X, ?, ? --> X
46106   if (TrueOp == FalseOp)
46107     return TrueOp;
46108 
46109   // Try to simplify the EFLAGS and condition code operands.
46110   // We can't always do this as FCMOV only supports a subset of X86 cond.
46111   if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
46112     if (!(FalseOp.getValueType() == MVT::f80 ||
46113           (FalseOp.getValueType() == MVT::f64 && !Subtarget.hasSSE2()) ||
46114           (FalseOp.getValueType() == MVT::f32 && !Subtarget.hasSSE1())) ||
46115         !Subtarget.canUseCMOV() || hasFPCMov(CC)) {
46116       SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
46117                        Flags};
46118       return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46119     }
46120   }
46121 
46122   // If this is a select between two integer constants, try to do some
46123   // optimizations.  Note that the operands are ordered the opposite of SELECT
46124   // operands.
46125   if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
46126     if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
46127       // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
46128       // larger than FalseC (the false value).
46129       if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
46130         CC = X86::GetOppositeBranchCondition(CC);
46131         std::swap(TrueC, FalseC);
46132         std::swap(TrueOp, FalseOp);
46133       }
46134 
46135       // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3.  Likewise for any pow2/0.
46136       // This is efficient for any integer data type (including i8/i16) and
46137       // shift amount.
46138       if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
46139         Cond = getSETCC(CC, Cond, DL, DAG);
46140 
46141         // Zero extend the condition if needed.
46142         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
46143 
46144         unsigned ShAmt = TrueC->getAPIntValue().logBase2();
46145         Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
46146                            DAG.getConstant(ShAmt, DL, MVT::i8));
46147         return Cond;
46148       }
46149 
46150       // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.  This is efficient
46151       // for any integer data type, including i8/i16.
46152       if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
46153         Cond = getSETCC(CC, Cond, DL, DAG);
46154 
46155         // Zero extend the condition if needed.
46156         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
46157                            FalseC->getValueType(0), Cond);
46158         Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
46159                            SDValue(FalseC, 0));
46160         return Cond;
46161       }
46162 
46163       // Optimize cases that will turn into an LEA instruction.  This requires
46164       // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
46165       if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
46166         APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue();
46167         assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() &&
46168                "Implicit constant truncation");
46169 
46170         bool isFastMultiplier = false;
46171         if (Diff.ult(10)) {
46172           switch (Diff.getZExtValue()) {
46173           default: break;
46174           case 1:  // result = add base, cond
46175           case 2:  // result = lea base(    , cond*2)
46176           case 3:  // result = lea base(cond, cond*2)
46177           case 4:  // result = lea base(    , cond*4)
46178           case 5:  // result = lea base(cond, cond*4)
46179           case 8:  // result = lea base(    , cond*8)
46180           case 9:  // result = lea base(cond, cond*8)
46181             isFastMultiplier = true;
46182             break;
46183           }
46184         }
46185 
46186         if (isFastMultiplier) {
46187           Cond = getSETCC(CC, Cond, DL ,DAG);
46188           // Zero extend the condition if needed.
46189           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
46190                              Cond);
46191           // Scale the condition by the difference.
46192           if (Diff != 1)
46193             Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
46194                                DAG.getConstant(Diff, DL, Cond.getValueType()));
46195 
46196           // Add the base if non-zero.
46197           if (FalseC->getAPIntValue() != 0)
46198             Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
46199                                SDValue(FalseC, 0));
46200           return Cond;
46201         }
46202       }
46203     }
46204   }
46205 
46206   // Handle these cases:
46207   //   (select (x != c), e, c) -> select (x != c), e, x),
46208   //   (select (x == c), c, e) -> select (x == c), x, e)
46209   // where the c is an integer constant, and the "select" is the combination
46210   // of CMOV and CMP.
46211   //
46212   // The rationale for this change is that the conditional-move from a constant
46213   // needs two instructions, however, conditional-move from a register needs
46214   // only one instruction.
46215   //
46216   // CAVEAT: By replacing a constant with a symbolic value, it may obscure
46217   //  some instruction-combining opportunities. This opt needs to be
46218   //  postponed as late as possible.
46219   //
46220   if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
46221     // the DCI.xxxx conditions are provided to postpone the optimization as
46222     // late as possible.
46223 
46224     ConstantSDNode *CmpAgainst = nullptr;
46225     if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
46226         (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
46227         !isa<ConstantSDNode>(Cond.getOperand(0))) {
46228 
46229       if (CC == X86::COND_NE &&
46230           CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
46231         CC = X86::GetOppositeBranchCondition(CC);
46232         std::swap(TrueOp, FalseOp);
46233       }
46234 
46235       if (CC == X86::COND_E &&
46236           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
46237         SDValue Ops[] = {FalseOp, Cond.getOperand(0),
46238                          DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
46239         return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46240       }
46241     }
46242   }
46243 
46244   // Transform:
46245   //
46246   //   (cmov 1 T (uge T 2))
46247   //
46248   // to:
46249   //
46250   //   (adc T 0 (sub T 1))
46251   if (CC == X86::COND_AE && isOneConstant(FalseOp) &&
46252       Cond.getOpcode() == X86ISD::SUB && Cond->hasOneUse()) {
46253     SDValue Cond0 = Cond.getOperand(0);
46254     if (Cond0.getOpcode() == ISD::TRUNCATE)
46255       Cond0 = Cond0.getOperand(0);
46256     auto *Sub1C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
46257     if (Cond0 == TrueOp && Sub1C && Sub1C->getZExtValue() == 2) {
46258       EVT CondVT = Cond->getValueType(0);
46259       EVT OuterVT = N->getValueType(0);
46260       // Subtract 1 and generate a carry.
46261       SDValue NewSub =
46262           DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0),
46263                       DAG.getConstant(1, DL, CondVT));
46264       SDValue EFLAGS(NewSub.getNode(), 1);
46265       return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(OuterVT, MVT::i32),
46266                          TrueOp, DAG.getConstant(0, DL, OuterVT), EFLAGS);
46267     }
46268   }
46269 
46270   // Fold and/or of setcc's to double CMOV:
46271   //   (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
46272   //   (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
46273   //
46274   // This combine lets us generate:
46275   //   cmovcc1 (jcc1 if we don't have CMOV)
46276   //   cmovcc2 (same)
46277   // instead of:
46278   //   setcc1
46279   //   setcc2
46280   //   and/or
46281   //   cmovne (jne if we don't have CMOV)
46282   // When we can't use the CMOV instruction, it might increase branch
46283   // mispredicts.
46284   // When we can use CMOV, or when there is no mispredict, this improves
46285   // throughput and reduces register pressure.
46286   //
46287   if (CC == X86::COND_NE) {
46288     SDValue Flags;
46289     X86::CondCode CC0, CC1;
46290     bool isAndSetCC;
46291     if (checkBoolTestAndOrSetCCCombine(Cond, CC0, CC1, Flags, isAndSetCC)) {
46292       if (isAndSetCC) {
46293         std::swap(FalseOp, TrueOp);
46294         CC0 = X86::GetOppositeBranchCondition(CC0);
46295         CC1 = X86::GetOppositeBranchCondition(CC1);
46296       }
46297 
46298       SDValue LOps[] = {FalseOp, TrueOp,
46299                         DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
46300       SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
46301       SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
46302                        Flags};
46303       SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46304       return CMOV;
46305     }
46306   }
46307 
46308   // Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
46309   //      (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
46310   // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
46311   //    (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2)
46312   if ((CC == X86::COND_NE || CC == X86::COND_E) &&
46313       Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
46314     SDValue Add = TrueOp;
46315     SDValue Const = FalseOp;
46316     // Canonicalize the condition code for easier matching and output.
46317     if (CC == X86::COND_E)
46318       std::swap(Add, Const);
46319 
46320     // We might have replaced the constant in the cmov with the LHS of the
46321     // compare. If so change it to the RHS of the compare.
46322     if (Const == Cond.getOperand(0))
46323       Const = Cond.getOperand(1);
46324 
46325     // Ok, now make sure that Add is (add (cttz X), C2) and Const is a constant.
46326     if (isa<ConstantSDNode>(Const) && Add.getOpcode() == ISD::ADD &&
46327         Add.hasOneUse() && isa<ConstantSDNode>(Add.getOperand(1)) &&
46328         (Add.getOperand(0).getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
46329          Add.getOperand(0).getOpcode() == ISD::CTTZ) &&
46330         Add.getOperand(0).getOperand(0) == Cond.getOperand(0)) {
46331       EVT VT = N->getValueType(0);
46332       // This should constant fold.
46333       SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
46334       SDValue CMov =
46335           DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
46336                       DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
46337       return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
46338     }
46339   }
46340 
46341   return SDValue();
46342 }
46343 
46344 /// Different mul shrinking modes.
46345 enum class ShrinkMode { MULS8, MULU8, MULS16, MULU16 };
46346 
46347 static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) {
46348   EVT VT = N->getOperand(0).getValueType();
46349   if (VT.getScalarSizeInBits() != 32)
46350     return false;
46351 
46352   assert(N->getNumOperands() == 2 && "NumOperands of Mul are 2");
46353   unsigned SignBits[2] = {1, 1};
46354   bool IsPositive[2] = {false, false};
46355   for (unsigned i = 0; i < 2; i++) {
46356     SDValue Opd = N->getOperand(i);
46357 
46358     SignBits[i] = DAG.ComputeNumSignBits(Opd);
46359     IsPositive[i] = DAG.SignBitIsZero(Opd);
46360   }
46361 
46362   bool AllPositive = IsPositive[0] && IsPositive[1];
46363   unsigned MinSignBits = std::min(SignBits[0], SignBits[1]);
46364   // When ranges are from -128 ~ 127, use MULS8 mode.
46365   if (MinSignBits >= 25)
46366     Mode = ShrinkMode::MULS8;
46367   // When ranges are from 0 ~ 255, use MULU8 mode.
46368   else if (AllPositive && MinSignBits >= 24)
46369     Mode = ShrinkMode::MULU8;
46370   // When ranges are from -32768 ~ 32767, use MULS16 mode.
46371   else if (MinSignBits >= 17)
46372     Mode = ShrinkMode::MULS16;
46373   // When ranges are from 0 ~ 65535, use MULU16 mode.
46374   else if (AllPositive && MinSignBits >= 16)
46375     Mode = ShrinkMode::MULU16;
46376   else
46377     return false;
46378   return true;
46379 }
46380 
46381 /// When the operands of vector mul are extended from smaller size values,
46382 /// like i8 and i16, the type of mul may be shrinked to generate more
46383 /// efficient code. Two typical patterns are handled:
46384 /// Pattern1:
46385 ///     %2 = sext/zext <N x i8> %1 to <N x i32>
46386 ///     %4 = sext/zext <N x i8> %3 to <N x i32>
46387 //   or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
46388 ///     %5 = mul <N x i32> %2, %4
46389 ///
46390 /// Pattern2:
46391 ///     %2 = zext/sext <N x i16> %1 to <N x i32>
46392 ///     %4 = zext/sext <N x i16> %3 to <N x i32>
46393 ///  or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
46394 ///     %5 = mul <N x i32> %2, %4
46395 ///
46396 /// There are four mul shrinking modes:
46397 /// If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is
46398 /// -128 to 128, and the scalar value range of %4 is also -128 to 128,
46399 /// generate pmullw+sext32 for it (MULS8 mode).
46400 /// If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is
46401 /// 0 to 255, and the scalar value range of %4 is also 0 to 255,
46402 /// generate pmullw+zext32 for it (MULU8 mode).
46403 /// If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is
46404 /// -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767,
46405 /// generate pmullw+pmulhw for it (MULS16 mode).
46406 /// If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is
46407 /// 0 to 65535, and the scalar value range of %4 is also 0 to 65535,
46408 /// generate pmullw+pmulhuw for it (MULU16 mode).
46409 static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
46410                                const X86Subtarget &Subtarget) {
46411   // Check for legality
46412   // pmullw/pmulhw are not supported by SSE.
46413   if (!Subtarget.hasSSE2())
46414     return SDValue();
46415 
46416   // Check for profitability
46417   // pmulld is supported since SSE41. It is better to use pmulld
46418   // instead of pmullw+pmulhw, except for subtargets where pmulld is slower than
46419   // the expansion.
46420   bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
46421   if (Subtarget.hasSSE41() && (OptForMinSize || !Subtarget.isPMULLDSlow()))
46422     return SDValue();
46423 
46424   ShrinkMode Mode;
46425   if (!canReduceVMulWidth(N, DAG, Mode))
46426     return SDValue();
46427 
46428   SDLoc DL(N);
46429   SDValue N0 = N->getOperand(0);
46430   SDValue N1 = N->getOperand(1);
46431   EVT VT = N->getOperand(0).getValueType();
46432   unsigned NumElts = VT.getVectorNumElements();
46433   if ((NumElts % 2) != 0)
46434     return SDValue();
46435 
46436   EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
46437 
46438   // Shrink the operands of mul.
46439   SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
46440   SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
46441 
46442   // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
46443   // lower part is needed.
46444   SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
46445   if (Mode == ShrinkMode::MULU8 || Mode == ShrinkMode::MULS8)
46446     return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND
46447                                                    : ISD::SIGN_EXTEND,
46448                        DL, VT, MulLo);
46449 
46450   EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2);
46451   // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
46452   // the higher part is also needed.
46453   SDValue MulHi =
46454       DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
46455                   ReducedVT, NewN0, NewN1);
46456 
46457   // Repack the lower part and higher part result of mul into a wider
46458   // result.
46459   // Generate shuffle functioning as punpcklwd.
46460   SmallVector<int, 16> ShuffleMask(NumElts);
46461   for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
46462     ShuffleMask[2 * i] = i;
46463     ShuffleMask[2 * i + 1] = i + NumElts;
46464   }
46465   SDValue ResLo =
46466       DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
46467   ResLo = DAG.getBitcast(ResVT, ResLo);
46468   // Generate shuffle functioning as punpckhwd.
46469   for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
46470     ShuffleMask[2 * i] = i + NumElts / 2;
46471     ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
46472   }
46473   SDValue ResHi =
46474       DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
46475   ResHi = DAG.getBitcast(ResVT, ResHi);
46476   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
46477 }
46478 
46479 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
46480                                  EVT VT, const SDLoc &DL) {
46481 
46482   auto combineMulShlAddOrSub = [&](int Mult, int Shift, bool isAdd) {
46483     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46484                                  DAG.getConstant(Mult, DL, VT));
46485     Result = DAG.getNode(ISD::SHL, DL, VT, Result,
46486                          DAG.getConstant(Shift, DL, MVT::i8));
46487     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
46488                          N->getOperand(0));
46489     return Result;
46490   };
46491 
46492   auto combineMulMulAddOrSub = [&](int Mul1, int Mul2, bool isAdd) {
46493     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46494                                  DAG.getConstant(Mul1, DL, VT));
46495     Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
46496                          DAG.getConstant(Mul2, DL, VT));
46497     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
46498                          N->getOperand(0));
46499     return Result;
46500   };
46501 
46502   switch (MulAmt) {
46503   default:
46504     break;
46505   case 11:
46506     // mul x, 11 => add ((shl (mul x, 5), 1), x)
46507     return combineMulShlAddOrSub(5, 1, /*isAdd*/ true);
46508   case 21:
46509     // mul x, 21 => add ((shl (mul x, 5), 2), x)
46510     return combineMulShlAddOrSub(5, 2, /*isAdd*/ true);
46511   case 41:
46512     // mul x, 41 => add ((shl (mul x, 5), 3), x)
46513     return combineMulShlAddOrSub(5, 3, /*isAdd*/ true);
46514   case 22:
46515     // mul x, 22 => add (add ((shl (mul x, 5), 2), x), x)
46516     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
46517                        combineMulShlAddOrSub(5, 2, /*isAdd*/ true));
46518   case 19:
46519     // mul x, 19 => add ((shl (mul x, 9), 1), x)
46520     return combineMulShlAddOrSub(9, 1, /*isAdd*/ true);
46521   case 37:
46522     // mul x, 37 => add ((shl (mul x, 9), 2), x)
46523     return combineMulShlAddOrSub(9, 2, /*isAdd*/ true);
46524   case 73:
46525     // mul x, 73 => add ((shl (mul x, 9), 3), x)
46526     return combineMulShlAddOrSub(9, 3, /*isAdd*/ true);
46527   case 13:
46528     // mul x, 13 => add ((shl (mul x, 3), 2), x)
46529     return combineMulShlAddOrSub(3, 2, /*isAdd*/ true);
46530   case 23:
46531     // mul x, 23 => sub ((shl (mul x, 3), 3), x)
46532     return combineMulShlAddOrSub(3, 3, /*isAdd*/ false);
46533   case 26:
46534     // mul x, 26 => add ((mul (mul x, 5), 5), x)
46535     return combineMulMulAddOrSub(5, 5, /*isAdd*/ true);
46536   case 28:
46537     // mul x, 28 => add ((mul (mul x, 9), 3), x)
46538     return combineMulMulAddOrSub(9, 3, /*isAdd*/ true);
46539   case 29:
46540     // mul x, 29 => add (add ((mul (mul x, 9), 3), x), x)
46541     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
46542                        combineMulMulAddOrSub(9, 3, /*isAdd*/ true));
46543   }
46544 
46545   // Another trick. If this is a power 2 + 2/4/8, we can use a shift followed
46546   // by a single LEA.
46547   // First check if this a sum of two power of 2s because that's easy. Then
46548   // count how many zeros are up to the first bit.
46549   // TODO: We can do this even without LEA at a cost of two shifts and an add.
46550   if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
46551     unsigned ScaleShift = llvm::countr_zero(MulAmt);
46552     if (ScaleShift >= 1 && ScaleShift < 4) {
46553       unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
46554       SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46555                                    DAG.getConstant(ShiftAmt, DL, MVT::i8));
46556       SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46557                                    DAG.getConstant(ScaleShift, DL, MVT::i8));
46558       return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
46559     }
46560   }
46561 
46562   return SDValue();
46563 }
46564 
46565 // If the upper 17 bits of either element are zero and the other element are
46566 // zero/sign bits then we can use PMADDWD, which is always at least as quick as
46567 // PMULLD, except on KNL.
46568 static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
46569                                    const X86Subtarget &Subtarget) {
46570   if (!Subtarget.hasSSE2())
46571     return SDValue();
46572 
46573   if (Subtarget.isPMADDWDSlow())
46574     return SDValue();
46575 
46576   EVT VT = N->getValueType(0);
46577 
46578   // Only support vXi32 vectors.
46579   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)
46580     return SDValue();
46581 
46582   // Make sure the type is legal or can split/widen to a legal type.
46583   // With AVX512 but without BWI, we would need to split v32i16.
46584   unsigned NumElts = VT.getVectorNumElements();
46585   if (NumElts == 1 || !isPowerOf2_32(NumElts))
46586     return SDValue();
46587 
46588   // With AVX512 but without BWI, we would need to split v32i16.
46589   if (32 <= (2 * NumElts) && Subtarget.hasAVX512() && !Subtarget.hasBWI())
46590     return SDValue();
46591 
46592   SDValue N0 = N->getOperand(0);
46593   SDValue N1 = N->getOperand(1);
46594 
46595   // If we are zero/sign extending two steps without SSE4.1, its better to
46596   // reduce the vmul width instead.
46597   if (!Subtarget.hasSSE41() &&
46598       (((N0.getOpcode() == ISD::ZERO_EXTEND &&
46599          N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
46600         (N1.getOpcode() == ISD::ZERO_EXTEND &&
46601          N1.getOperand(0).getScalarValueSizeInBits() <= 8)) ||
46602        ((N0.getOpcode() == ISD::SIGN_EXTEND &&
46603          N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
46604         (N1.getOpcode() == ISD::SIGN_EXTEND &&
46605          N1.getOperand(0).getScalarValueSizeInBits() <= 8))))
46606     return SDValue();
46607 
46608   // If we are sign extending a wide vector without SSE4.1, its better to reduce
46609   // the vmul width instead.
46610   if (!Subtarget.hasSSE41() &&
46611       (N0.getOpcode() == ISD::SIGN_EXTEND &&
46612        N0.getOperand(0).getValueSizeInBits() > 128) &&
46613       (N1.getOpcode() == ISD::SIGN_EXTEND &&
46614        N1.getOperand(0).getValueSizeInBits() > 128))
46615     return SDValue();
46616 
46617   // Sign bits must extend down to the lowest i16.
46618   if (DAG.ComputeMaxSignificantBits(N1) > 16 ||
46619       DAG.ComputeMaxSignificantBits(N0) > 16)
46620     return SDValue();
46621 
46622   // At least one of the elements must be zero in the upper 17 bits, or can be
46623   // safely made zero without altering the final result.
46624   auto GetZeroableOp = [&](SDValue Op) {
46625     APInt Mask17 = APInt::getHighBitsSet(32, 17);
46626     if (DAG.MaskedValueIsZero(Op, Mask17))
46627       return Op;
46628     // Mask off upper 16-bits of sign-extended constants.
46629     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()))
46630       return DAG.getNode(ISD::AND, SDLoc(N), VT, Op,
46631                          DAG.getConstant(0xFFFF, SDLoc(N), VT));
46632     if (Op.getOpcode() == ISD::SIGN_EXTEND && N->isOnlyUserOf(Op.getNode())) {
46633       SDValue Src = Op.getOperand(0);
46634       // Convert sext(vXi16) to zext(vXi16).
46635       if (Src.getScalarValueSizeInBits() == 16 && VT.getSizeInBits() <= 128)
46636         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src);
46637       // Convert sext(vXi8) to zext(vXi16 sext(vXi8)) on pre-SSE41 targets
46638       // which will expand the extension.
46639       if (Src.getScalarValueSizeInBits() < 16 && !Subtarget.hasSSE41()) {
46640         EVT ExtVT = VT.changeVectorElementType(MVT::i16);
46641         Src = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), ExtVT, Src);
46642         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src);
46643       }
46644     }
46645     // Convert SIGN_EXTEND_VECTOR_INREG to ZEXT_EXTEND_VECTOR_INREG.
46646     if (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG &&
46647         N->isOnlyUserOf(Op.getNode())) {
46648       SDValue Src = Op.getOperand(0);
46649       if (Src.getScalarValueSizeInBits() == 16)
46650         return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, Src);
46651     }
46652     // Convert VSRAI(Op, 16) to VSRLI(Op, 16).
46653     if (Op.getOpcode() == X86ISD::VSRAI && Op.getConstantOperandVal(1) == 16 &&
46654         N->isOnlyUserOf(Op.getNode())) {
46655       return DAG.getNode(X86ISD::VSRLI, SDLoc(N), VT, Op.getOperand(0),
46656                          Op.getOperand(1));
46657     }
46658     return SDValue();
46659   };
46660   SDValue ZeroN0 = GetZeroableOp(N0);
46661   SDValue ZeroN1 = GetZeroableOp(N1);
46662   if (!ZeroN0 && !ZeroN1)
46663     return SDValue();
46664   N0 = ZeroN0 ? ZeroN0 : N0;
46665   N1 = ZeroN1 ? ZeroN1 : N1;
46666 
46667   // Use SplitOpsAndApply to handle AVX splitting.
46668   auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46669                            ArrayRef<SDValue> Ops) {
46670     MVT ResVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
46671     MVT OpVT = MVT::getVectorVT(MVT::i16, Ops[0].getValueSizeInBits() / 16);
46672     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
46673                        DAG.getBitcast(OpVT, Ops[0]),
46674                        DAG.getBitcast(OpVT, Ops[1]));
46675   };
46676   return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {N0, N1},
46677                           PMADDWDBuilder);
46678 }
46679 
46680 static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
46681                                   const X86Subtarget &Subtarget) {
46682   if (!Subtarget.hasSSE2())
46683     return SDValue();
46684 
46685   EVT VT = N->getValueType(0);
46686 
46687   // Only support vXi64 vectors.
46688   if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||
46689       VT.getVectorNumElements() < 2 ||
46690       !isPowerOf2_32(VT.getVectorNumElements()))
46691     return SDValue();
46692 
46693   SDValue N0 = N->getOperand(0);
46694   SDValue N1 = N->getOperand(1);
46695 
46696   // MULDQ returns the 64-bit result of the signed multiplication of the lower
46697   // 32-bits. We can lower with this if the sign bits stretch that far.
46698   if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 &&
46699       DAG.ComputeNumSignBits(N1) > 32) {
46700     auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46701                             ArrayRef<SDValue> Ops) {
46702       return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
46703     };
46704     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
46705                             PMULDQBuilder, /*CheckBWI*/false);
46706   }
46707 
46708   // If the upper bits are zero we can use a single pmuludq.
46709   APInt Mask = APInt::getHighBitsSet(64, 32);
46710   if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) {
46711     auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46712                              ArrayRef<SDValue> Ops) {
46713       return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
46714     };
46715     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
46716                             PMULUDQBuilder, /*CheckBWI*/false);
46717   }
46718 
46719   return SDValue();
46720 }
46721 
46722 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
46723                           TargetLowering::DAGCombinerInfo &DCI,
46724                           const X86Subtarget &Subtarget) {
46725   EVT VT = N->getValueType(0);
46726 
46727   if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
46728     return V;
46729 
46730   if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget))
46731     return V;
46732 
46733   if (DCI.isBeforeLegalize() && VT.isVector())
46734     return reduceVMULWidth(N, DAG, Subtarget);
46735 
46736   // Optimize a single multiply with constant into two operations in order to
46737   // implement it with two cheaper instructions, e.g. LEA + SHL, LEA + LEA.
46738   if (!MulConstantOptimization)
46739     return SDValue();
46740 
46741   // An imul is usually smaller than the alternative sequence.
46742   if (DAG.getMachineFunction().getFunction().hasMinSize())
46743     return SDValue();
46744 
46745   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
46746     return SDValue();
46747 
46748   if (VT != MVT::i64 && VT != MVT::i32 &&
46749       (!VT.isVector() || !VT.isSimple() || !VT.isInteger()))
46750     return SDValue();
46751 
46752   ConstantSDNode *CNode = isConstOrConstSplat(
46753       N->getOperand(1), /*AllowUndefs*/ true, /*AllowTrunc*/ false);
46754   const APInt *C = nullptr;
46755   if (!CNode) {
46756     if (VT.isVector())
46757       if (auto *RawC = getTargetConstantFromNode(N->getOperand(1)))
46758         if (auto *SplatC = RawC->getSplatValue())
46759           C = &(SplatC->getUniqueInteger());
46760 
46761     if (!C || C->getBitWidth() != VT.getScalarSizeInBits())
46762       return SDValue();
46763   } else {
46764     C = &(CNode->getAPIntValue());
46765   }
46766 
46767   if (isPowerOf2_64(C->getZExtValue()))
46768     return SDValue();
46769 
46770   int64_t SignMulAmt = C->getSExtValue();
46771   assert(SignMulAmt != INT64_MIN && "Int min should have been handled!");
46772   uint64_t AbsMulAmt = SignMulAmt < 0 ? -SignMulAmt : SignMulAmt;
46773 
46774   SDLoc DL(N);
46775   SDValue NewMul = SDValue();
46776   if (VT == MVT::i64 || VT == MVT::i32) {
46777     if (AbsMulAmt == 3 || AbsMulAmt == 5 || AbsMulAmt == 9) {
46778       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46779                            DAG.getConstant(AbsMulAmt, DL, VT));
46780       if (SignMulAmt < 0)
46781         NewMul =
46782             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46783 
46784       return NewMul;
46785     }
46786 
46787     uint64_t MulAmt1 = 0;
46788     uint64_t MulAmt2 = 0;
46789     if ((AbsMulAmt % 9) == 0) {
46790       MulAmt1 = 9;
46791       MulAmt2 = AbsMulAmt / 9;
46792     } else if ((AbsMulAmt % 5) == 0) {
46793       MulAmt1 = 5;
46794       MulAmt2 = AbsMulAmt / 5;
46795     } else if ((AbsMulAmt % 3) == 0) {
46796       MulAmt1 = 3;
46797       MulAmt2 = AbsMulAmt / 3;
46798     }
46799 
46800     // For negative multiply amounts, only allow MulAmt2 to be a power of 2.
46801     if (MulAmt2 &&
46802         (isPowerOf2_64(MulAmt2) ||
46803          (SignMulAmt >= 0 && (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)))) {
46804 
46805       if (isPowerOf2_64(MulAmt2) && !(SignMulAmt >= 0 && N->hasOneUse() &&
46806                                       N->use_begin()->getOpcode() == ISD::ADD))
46807         // If second multiplifer is pow2, issue it first. We want the multiply
46808         // by 3, 5, or 9 to be folded into the addressing mode unless the lone
46809         // use is an add. Only do this for positive multiply amounts since the
46810         // negate would prevent it from being used as an address mode anyway.
46811         std::swap(MulAmt1, MulAmt2);
46812 
46813       if (isPowerOf2_64(MulAmt1))
46814         NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46815                              DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
46816       else
46817         NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46818                              DAG.getConstant(MulAmt1, DL, VT));
46819 
46820       if (isPowerOf2_64(MulAmt2))
46821         NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
46822                              DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
46823       else
46824         NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
46825                              DAG.getConstant(MulAmt2, DL, VT));
46826 
46827       // Negate the result.
46828       if (SignMulAmt < 0)
46829         NewMul =
46830             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46831     } else if (!Subtarget.slowLEA())
46832       NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);
46833   }
46834   if (!NewMul) {
46835     EVT ShiftVT = VT.isVector() ? VT : MVT::i8;
46836     assert(C->getZExtValue() != 0 &&
46837            C->getZExtValue() != maxUIntN(VT.getScalarSizeInBits()) &&
46838            "Both cases that could cause potential overflows should have "
46839            "already been handled.");
46840     if (isPowerOf2_64(AbsMulAmt - 1)) {
46841       // (mul x, 2^N + 1) => (add (shl x, N), x)
46842       NewMul = DAG.getNode(
46843           ISD::ADD, DL, VT, N->getOperand(0),
46844           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46845                       DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT)));
46846       // To negate, subtract the number from zero
46847       if (SignMulAmt < 0)
46848         NewMul =
46849             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46850     } else if (isPowerOf2_64(AbsMulAmt + 1)) {
46851       // (mul x, 2^N - 1) => (sub (shl x, N), x)
46852       NewMul =
46853           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46854                       DAG.getConstant(Log2_64(AbsMulAmt + 1), DL, ShiftVT));
46855       // To negate, reverse the operands of the subtract.
46856       if (SignMulAmt < 0)
46857         NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
46858       else
46859         NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
46860     } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt - 2) &&
46861                (!VT.isVector() || Subtarget.fastImmVectorShift())) {
46862       // (mul x, 2^N + 2) => (add (shl x, N), (add x, x))
46863       NewMul =
46864           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46865                       DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT));
46866       NewMul = DAG.getNode(
46867           ISD::ADD, DL, VT, NewMul,
46868           DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
46869     } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt + 2) &&
46870                (!VT.isVector() || Subtarget.fastImmVectorShift())) {
46871       // (mul x, 2^N - 2) => (sub (shl x, N), (add x, x))
46872       NewMul =
46873           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46874                       DAG.getConstant(Log2_64(AbsMulAmt + 2), DL, ShiftVT));
46875       NewMul = DAG.getNode(
46876           ISD::SUB, DL, VT, NewMul,
46877           DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
46878     } else if (SignMulAmt >= 0 && VT.isVector() &&
46879                Subtarget.fastImmVectorShift()) {
46880       uint64_t AbsMulAmtLowBit = AbsMulAmt & (-AbsMulAmt);
46881       uint64_t ShiftAmt1;
46882       std::optional<unsigned> Opc;
46883       if (isPowerOf2_64(AbsMulAmt - AbsMulAmtLowBit)) {
46884         ShiftAmt1 = AbsMulAmt - AbsMulAmtLowBit;
46885         Opc = ISD::ADD;
46886       } else if (isPowerOf2_64(AbsMulAmt + AbsMulAmtLowBit)) {
46887         ShiftAmt1 = AbsMulAmt + AbsMulAmtLowBit;
46888         Opc = ISD::SUB;
46889       }
46890 
46891       if (Opc) {
46892         SDValue Shift1 =
46893             DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46894                         DAG.getConstant(Log2_64(ShiftAmt1), DL, ShiftVT));
46895         SDValue Shift2 =
46896             DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46897                         DAG.getConstant(Log2_64(AbsMulAmtLowBit), DL, ShiftVT));
46898         NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2);
46899       }
46900     }
46901   }
46902 
46903   return NewMul;
46904 }
46905 
46906 // Try to form a MULHU or MULHS node by looking for
46907 // (srl (mul ext, ext), 16)
46908 // TODO: This is X86 specific because we want to be able to handle wide types
46909 // before type legalization. But we can only do it if the vector will be
46910 // legalized via widening/splitting. Type legalization can't handle promotion
46911 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
46912 // combiner.
46913 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
46914                                    const X86Subtarget &Subtarget) {
46915   assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
46916            "SRL or SRA node is required here!");
46917   SDLoc DL(N);
46918 
46919   if (!Subtarget.hasSSE2())
46920     return SDValue();
46921 
46922   // The operation feeding into the shift must be a multiply.
46923   SDValue ShiftOperand = N->getOperand(0);
46924   if (ShiftOperand.getOpcode() != ISD::MUL || !ShiftOperand.hasOneUse())
46925     return SDValue();
46926 
46927   // Input type should be at least vXi32.
46928   EVT VT = N->getValueType(0);
46929   if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32)
46930     return SDValue();
46931 
46932   // Need a shift by 16.
46933   APInt ShiftAmt;
46934   if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) ||
46935       ShiftAmt != 16)
46936     return SDValue();
46937 
46938   SDValue LHS = ShiftOperand.getOperand(0);
46939   SDValue RHS = ShiftOperand.getOperand(1);
46940 
46941   unsigned ExtOpc = LHS.getOpcode();
46942   if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
46943       RHS.getOpcode() != ExtOpc)
46944     return SDValue();
46945 
46946   // Peek through the extends.
46947   LHS = LHS.getOperand(0);
46948   RHS = RHS.getOperand(0);
46949 
46950   // Ensure the input types match.
46951   EVT MulVT = LHS.getValueType();
46952   if (MulVT.getVectorElementType() != MVT::i16 || RHS.getValueType() != MulVT)
46953     return SDValue();
46954 
46955   unsigned Opc = ExtOpc == ISD::SIGN_EXTEND ? ISD::MULHS : ISD::MULHU;
46956   SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
46957 
46958   ExtOpc = N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
46959   return DAG.getNode(ExtOpc, DL, VT, Mulh);
46960 }
46961 
46962 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
46963   SDValue N0 = N->getOperand(0);
46964   SDValue N1 = N->getOperand(1);
46965   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
46966   EVT VT = N0.getValueType();
46967 
46968   // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
46969   // since the result of setcc_c is all zero's or all ones.
46970   if (VT.isInteger() && !VT.isVector() &&
46971       N1C && N0.getOpcode() == ISD::AND &&
46972       N0.getOperand(1).getOpcode() == ISD::Constant) {
46973     SDValue N00 = N0.getOperand(0);
46974     APInt Mask = N0.getConstantOperandAPInt(1);
46975     Mask <<= N1C->getAPIntValue();
46976     bool MaskOK = false;
46977     // We can handle cases concerning bit-widening nodes containing setcc_c if
46978     // we carefully interrogate the mask to make sure we are semantics
46979     // preserving.
46980     // The transform is not safe if the result of C1 << C2 exceeds the bitwidth
46981     // of the underlying setcc_c operation if the setcc_c was zero extended.
46982     // Consider the following example:
46983     //   zext(setcc_c)                 -> i32 0x0000FFFF
46984     //   c1                            -> i32 0x0000FFFF
46985     //   c2                            -> i32 0x00000001
46986     //   (shl (and (setcc_c), c1), c2) -> i32 0x0001FFFE
46987     //   (and setcc_c, (c1 << c2))     -> i32 0x0000FFFE
46988     if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
46989       MaskOK = true;
46990     } else if (N00.getOpcode() == ISD::SIGN_EXTEND &&
46991                N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
46992       MaskOK = true;
46993     } else if ((N00.getOpcode() == ISD::ZERO_EXTEND ||
46994                 N00.getOpcode() == ISD::ANY_EXTEND) &&
46995                N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
46996       MaskOK = Mask.isIntN(N00.getOperand(0).getValueSizeInBits());
46997     }
46998     if (MaskOK && Mask != 0) {
46999       SDLoc DL(N);
47000       return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
47001     }
47002   }
47003 
47004   return SDValue();
47005 }
47006 
47007 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
47008                                            const X86Subtarget &Subtarget) {
47009   SDValue N0 = N->getOperand(0);
47010   SDValue N1 = N->getOperand(1);
47011   EVT VT = N0.getValueType();
47012   unsigned Size = VT.getSizeInBits();
47013 
47014   if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
47015     return V;
47016 
47017   // fold (ashr (shl, a, [56,48,32,24,16]), SarConst)
47018   // into (shl, (sext (a), [56,48,32,24,16] - SarConst)) or
47019   // into (lshr, (sext (a), SarConst - [56,48,32,24,16]))
47020   // depending on sign of (SarConst - [56,48,32,24,16])
47021 
47022   // sexts in X86 are MOVs. The MOVs have the same code size
47023   // as above SHIFTs (only SHIFT on 1 has lower code size).
47024   // However the MOVs have 2 advantages to a SHIFT:
47025   // 1. MOVs can write to a register that differs from source
47026   // 2. MOVs accept memory operands
47027 
47028   if (VT.isVector() || N1.getOpcode() != ISD::Constant ||
47029       N0.getOpcode() != ISD::SHL || !N0.hasOneUse() ||
47030       N0.getOperand(1).getOpcode() != ISD::Constant)
47031     return SDValue();
47032 
47033   SDValue N00 = N0.getOperand(0);
47034   SDValue N01 = N0.getOperand(1);
47035   APInt ShlConst = N01->getAsAPIntVal();
47036   APInt SarConst = N1->getAsAPIntVal();
47037   EVT CVT = N1.getValueType();
47038 
47039   if (SarConst.isNegative())
47040     return SDValue();
47041 
47042   for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
47043     unsigned ShiftSize = SVT.getSizeInBits();
47044     // skipping types without corresponding sext/zext and
47045     // ShlConst that is not one of [56,48,32,24,16]
47046     if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
47047       continue;
47048     SDLoc DL(N);
47049     SDValue NN =
47050         DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
47051     SarConst = SarConst - (Size - ShiftSize);
47052     if (SarConst == 0)
47053       return NN;
47054     if (SarConst.isNegative())
47055       return DAG.getNode(ISD::SHL, DL, VT, NN,
47056                          DAG.getConstant(-SarConst, DL, CVT));
47057     return DAG.getNode(ISD::SRA, DL, VT, NN,
47058                        DAG.getConstant(SarConst, DL, CVT));
47059   }
47060   return SDValue();
47061 }
47062 
47063 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
47064                                         TargetLowering::DAGCombinerInfo &DCI,
47065                                         const X86Subtarget &Subtarget) {
47066   SDValue N0 = N->getOperand(0);
47067   SDValue N1 = N->getOperand(1);
47068   EVT VT = N0.getValueType();
47069 
47070   if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
47071     return V;
47072 
47073   // Only do this on the last DAG combine as it can interfere with other
47074   // combines.
47075   if (!DCI.isAfterLegalizeDAG())
47076     return SDValue();
47077 
47078   // Try to improve a sequence of srl (and X, C1), C2 by inverting the order.
47079   // TODO: This is a generic DAG combine that became an x86-only combine to
47080   // avoid shortcomings in other folds such as bswap, bit-test ('bt'), and
47081   // and-not ('andn').
47082   if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
47083     return SDValue();
47084 
47085   auto *ShiftC = dyn_cast<ConstantSDNode>(N1);
47086   auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
47087   if (!ShiftC || !AndC)
47088     return SDValue();
47089 
47090   // If we can shrink the constant mask below 8-bits or 32-bits, then this
47091   // transform should reduce code size. It may also enable secondary transforms
47092   // from improved known-bits analysis or instruction selection.
47093   APInt MaskVal = AndC->getAPIntValue();
47094 
47095   // If this can be matched by a zero extend, don't optimize.
47096   if (MaskVal.isMask()) {
47097     unsigned TO = MaskVal.countr_one();
47098     if (TO >= 8 && isPowerOf2_32(TO))
47099       return SDValue();
47100   }
47101 
47102   APInt NewMaskVal = MaskVal.lshr(ShiftC->getAPIntValue());
47103   unsigned OldMaskSize = MaskVal.getSignificantBits();
47104   unsigned NewMaskSize = NewMaskVal.getSignificantBits();
47105   if ((OldMaskSize > 8 && NewMaskSize <= 8) ||
47106       (OldMaskSize > 32 && NewMaskSize <= 32)) {
47107     // srl (and X, AndC), ShiftC --> and (srl X, ShiftC), (AndC >> ShiftC)
47108     SDLoc DL(N);
47109     SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
47110     SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
47111     return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
47112   }
47113   return SDValue();
47114 }
47115 
47116 static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
47117                                          const X86Subtarget &Subtarget) {
47118   unsigned Opcode = N->getOpcode();
47119   assert(isHorizOp(Opcode) && "Unexpected hadd/hsub/pack opcode");
47120 
47121   SDLoc DL(N);
47122   EVT VT = N->getValueType(0);
47123   SDValue N0 = N->getOperand(0);
47124   SDValue N1 = N->getOperand(1);
47125   EVT SrcVT = N0.getValueType();
47126 
47127   SDValue BC0 =
47128       N->isOnlyUserOf(N0.getNode()) ? peekThroughOneUseBitcasts(N0) : N0;
47129   SDValue BC1 =
47130       N->isOnlyUserOf(N1.getNode()) ? peekThroughOneUseBitcasts(N1) : N1;
47131 
47132   // Attempt to fold HOP(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X)))
47133   // to SHUFFLE(HOP(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for
47134   // truncation trees that help us avoid lane crossing shuffles.
47135   // TODO: There's a lot more we can do for PACK/HADD style shuffle combines.
47136   // TODO: We don't handle vXf64 shuffles yet.
47137   if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
47138     if (SDValue BCSrc = getSplitVectorSrc(BC0, BC1, false)) {
47139       SmallVector<SDValue> ShuffleOps;
47140       SmallVector<int> ShuffleMask, ScaledMask;
47141       SDValue Vec = peekThroughBitcasts(BCSrc);
47142       if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) {
47143         resolveTargetShuffleInputsAndMask(ShuffleOps, ShuffleMask);
47144         // To keep the HOP LHS/RHS coherency, we must be able to scale the unary
47145         // shuffle to a v4X64 width - we can probably relax this in the future.
47146         if (!isAnyZero(ShuffleMask) && ShuffleOps.size() == 1 &&
47147             ShuffleOps[0].getValueType().is256BitVector() &&
47148             scaleShuffleElements(ShuffleMask, 4, ScaledMask)) {
47149           SDValue Lo, Hi;
47150           MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
47151           std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL);
47152           Lo = DAG.getBitcast(SrcVT, Lo);
47153           Hi = DAG.getBitcast(SrcVT, Hi);
47154           SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
47155           Res = DAG.getBitcast(ShufVT, Res);
47156           Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask);
47157           return DAG.getBitcast(VT, Res);
47158         }
47159       }
47160     }
47161   }
47162 
47163   // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(Z,W)) -> SHUFFLE(HOP()).
47164   if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
47165     // If either/both ops are a shuffle that can scale to v2x64,
47166     // then see if we can perform this as a v4x32 post shuffle.
47167     SmallVector<SDValue> Ops0, Ops1;
47168     SmallVector<int> Mask0, Mask1, ScaledMask0, ScaledMask1;
47169     bool IsShuf0 =
47170         getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
47171         scaleShuffleElements(Mask0, 2, ScaledMask0) &&
47172         all_of(Ops0, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
47173     bool IsShuf1 =
47174         getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
47175         scaleShuffleElements(Mask1, 2, ScaledMask1) &&
47176         all_of(Ops1, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
47177     if (IsShuf0 || IsShuf1) {
47178       if (!IsShuf0) {
47179         Ops0.assign({BC0});
47180         ScaledMask0.assign({0, 1});
47181       }
47182       if (!IsShuf1) {
47183         Ops1.assign({BC1});
47184         ScaledMask1.assign({0, 1});
47185       }
47186 
47187       SDValue LHS, RHS;
47188       int PostShuffle[4] = {-1, -1, -1, -1};
47189       auto FindShuffleOpAndIdx = [&](int M, int &Idx, ArrayRef<SDValue> Ops) {
47190         if (M < 0)
47191           return true;
47192         Idx = M % 2;
47193         SDValue Src = Ops[M / 2];
47194         if (!LHS || LHS == Src) {
47195           LHS = Src;
47196           return true;
47197         }
47198         if (!RHS || RHS == Src) {
47199           Idx += 2;
47200           RHS = Src;
47201           return true;
47202         }
47203         return false;
47204       };
47205       if (FindShuffleOpAndIdx(ScaledMask0[0], PostShuffle[0], Ops0) &&
47206           FindShuffleOpAndIdx(ScaledMask0[1], PostShuffle[1], Ops0) &&
47207           FindShuffleOpAndIdx(ScaledMask1[0], PostShuffle[2], Ops1) &&
47208           FindShuffleOpAndIdx(ScaledMask1[1], PostShuffle[3], Ops1)) {
47209         LHS = DAG.getBitcast(SrcVT, LHS);
47210         RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
47211         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
47212         SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
47213         Res = DAG.getBitcast(ShufVT, Res);
47214         Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle);
47215         return DAG.getBitcast(VT, Res);
47216       }
47217     }
47218   }
47219 
47220   // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y)).
47221   if (VT.is256BitVector() && Subtarget.hasInt256()) {
47222     SmallVector<int> Mask0, Mask1;
47223     SmallVector<SDValue> Ops0, Ops1;
47224     SmallVector<int, 2> ScaledMask0, ScaledMask1;
47225     if (getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
47226         getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
47227         !Ops0.empty() && !Ops1.empty() &&
47228         all_of(Ops0,
47229                [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
47230         all_of(Ops1,
47231                [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
47232         scaleShuffleElements(Mask0, 2, ScaledMask0) &&
47233         scaleShuffleElements(Mask1, 2, ScaledMask1)) {
47234       SDValue Op00 = peekThroughBitcasts(Ops0.front());
47235       SDValue Op10 = peekThroughBitcasts(Ops1.front());
47236       SDValue Op01 = peekThroughBitcasts(Ops0.back());
47237       SDValue Op11 = peekThroughBitcasts(Ops1.back());
47238       if ((Op00 == Op11) && (Op01 == Op10)) {
47239         std::swap(Op10, Op11);
47240         ShuffleVectorSDNode::commuteMask(ScaledMask1);
47241       }
47242       if ((Op00 == Op10) && (Op01 == Op11)) {
47243         const int Map[4] = {0, 2, 1, 3};
47244         SmallVector<int, 4> ShuffleMask(
47245             {Map[ScaledMask0[0]], Map[ScaledMask1[0]], Map[ScaledMask0[1]],
47246              Map[ScaledMask1[1]]});
47247         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
47248         SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00),
47249                                   DAG.getBitcast(SrcVT, Op01));
47250         Res = DAG.getBitcast(ShufVT, Res);
47251         Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask);
47252         return DAG.getBitcast(VT, Res);
47253       }
47254     }
47255   }
47256 
47257   return SDValue();
47258 }
47259 
47260 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
47261                                  TargetLowering::DAGCombinerInfo &DCI,
47262                                  const X86Subtarget &Subtarget) {
47263   unsigned Opcode = N->getOpcode();
47264   assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
47265          "Unexpected pack opcode");
47266 
47267   EVT VT = N->getValueType(0);
47268   SDValue N0 = N->getOperand(0);
47269   SDValue N1 = N->getOperand(1);
47270   unsigned NumDstElts = VT.getVectorNumElements();
47271   unsigned DstBitsPerElt = VT.getScalarSizeInBits();
47272   unsigned SrcBitsPerElt = 2 * DstBitsPerElt;
47273   assert(N0.getScalarValueSizeInBits() == SrcBitsPerElt &&
47274          N1.getScalarValueSizeInBits() == SrcBitsPerElt &&
47275          "Unexpected PACKSS/PACKUS input type");
47276 
47277   bool IsSigned = (X86ISD::PACKSS == Opcode);
47278 
47279   // Constant Folding.
47280   APInt UndefElts0, UndefElts1;
47281   SmallVector<APInt, 32> EltBits0, EltBits1;
47282   if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) &&
47283       (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) &&
47284       getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0) &&
47285       getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1)) {
47286     unsigned NumLanes = VT.getSizeInBits() / 128;
47287     unsigned NumSrcElts = NumDstElts / 2;
47288     unsigned NumDstEltsPerLane = NumDstElts / NumLanes;
47289     unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
47290 
47291     APInt Undefs(NumDstElts, 0);
47292     SmallVector<APInt, 32> Bits(NumDstElts, APInt::getZero(DstBitsPerElt));
47293     for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
47294       for (unsigned Elt = 0; Elt != NumDstEltsPerLane; ++Elt) {
47295         unsigned SrcIdx = Lane * NumSrcEltsPerLane + Elt % NumSrcEltsPerLane;
47296         auto &UndefElts = (Elt >= NumSrcEltsPerLane ? UndefElts1 : UndefElts0);
47297         auto &EltBits = (Elt >= NumSrcEltsPerLane ? EltBits1 : EltBits0);
47298 
47299         if (UndefElts[SrcIdx]) {
47300           Undefs.setBit(Lane * NumDstEltsPerLane + Elt);
47301           continue;
47302         }
47303 
47304         APInt &Val = EltBits[SrcIdx];
47305         if (IsSigned) {
47306           // PACKSS: Truncate signed value with signed saturation.
47307           // Source values less than dst minint are saturated to minint.
47308           // Source values greater than dst maxint are saturated to maxint.
47309           if (Val.isSignedIntN(DstBitsPerElt))
47310             Val = Val.trunc(DstBitsPerElt);
47311           else if (Val.isNegative())
47312             Val = APInt::getSignedMinValue(DstBitsPerElt);
47313           else
47314             Val = APInt::getSignedMaxValue(DstBitsPerElt);
47315         } else {
47316           // PACKUS: Truncate signed value with unsigned saturation.
47317           // Source values less than zero are saturated to zero.
47318           // Source values greater than dst maxuint are saturated to maxuint.
47319           if (Val.isIntN(DstBitsPerElt))
47320             Val = Val.trunc(DstBitsPerElt);
47321           else if (Val.isNegative())
47322             Val = APInt::getZero(DstBitsPerElt);
47323           else
47324             Val = APInt::getAllOnes(DstBitsPerElt);
47325         }
47326         Bits[Lane * NumDstEltsPerLane + Elt] = Val;
47327       }
47328     }
47329 
47330     return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
47331   }
47332 
47333   // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()).
47334   if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
47335     return V;
47336 
47337   // Try to fold PACKSS(NOT(X),NOT(Y)) -> NOT(PACKSS(X,Y)).
47338   // Currently limit this to allsignbits cases only.
47339   if (IsSigned &&
47340       (N0.isUndef() || DAG.ComputeNumSignBits(N0) == SrcBitsPerElt) &&
47341       (N1.isUndef() || DAG.ComputeNumSignBits(N1) == SrcBitsPerElt)) {
47342     SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG);
47343     SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG);
47344     if (Not0 && Not1) {
47345       SDLoc DL(N);
47346       MVT SrcVT = N0.getSimpleValueType();
47347       SDValue Pack =
47348           DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0),
47349                       DAG.getBitcast(SrcVT, Not1));
47350       return DAG.getNOT(DL, Pack, VT);
47351     }
47352   }
47353 
47354   // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
47355   // truncate to create a larger truncate.
47356   if (Subtarget.hasAVX512() &&
47357       N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
47358       N0.getOperand(0).getValueType() == MVT::v8i32) {
47359     if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
47360         (!IsSigned &&
47361          DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) {
47362       if (Subtarget.hasVLX())
47363         return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));
47364 
47365       // Widen input to v16i32 so we can truncate that.
47366       SDLoc dl(N);
47367       SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
47368                                    N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));
47369       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);
47370     }
47371   }
47372 
47373   // Try to fold PACK(EXTEND(X),EXTEND(Y)) -> CONCAT(X,Y) subvectors.
47374   if (VT.is128BitVector()) {
47375     unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
47376     SDValue Src0, Src1;
47377     if (N0.getOpcode() == ExtOpc &&
47378         N0.getOperand(0).getValueType().is64BitVector() &&
47379         N0.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
47380       Src0 = N0.getOperand(0);
47381     }
47382     if (N1.getOpcode() == ExtOpc &&
47383         N1.getOperand(0).getValueType().is64BitVector() &&
47384         N1.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
47385       Src1 = N1.getOperand(0);
47386     }
47387     if ((Src0 || N0.isUndef()) && (Src1 || N1.isUndef())) {
47388       assert((Src0 || Src1) && "Found PACK(UNDEF,UNDEF)");
47389       Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType());
47390       Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType());
47391       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1);
47392     }
47393 
47394     // Try again with pack(*_extend_vector_inreg, undef).
47395     unsigned VecInRegOpc = IsSigned ? ISD::SIGN_EXTEND_VECTOR_INREG
47396                                     : ISD::ZERO_EXTEND_VECTOR_INREG;
47397     if (N0.getOpcode() == VecInRegOpc && N1.isUndef() &&
47398         N0.getOperand(0).getScalarValueSizeInBits() < DstBitsPerElt)
47399       return getEXTEND_VECTOR_INREG(ExtOpc, SDLoc(N), VT, N0.getOperand(0),
47400                                     DAG);
47401   }
47402 
47403   // Attempt to combine as shuffle.
47404   SDValue Op(N, 0);
47405   if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47406     return Res;
47407 
47408   return SDValue();
47409 }
47410 
47411 static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
47412                                     TargetLowering::DAGCombinerInfo &DCI,
47413                                     const X86Subtarget &Subtarget) {
47414   assert((X86ISD::HADD == N->getOpcode() || X86ISD::FHADD == N->getOpcode() ||
47415           X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) &&
47416          "Unexpected horizontal add/sub opcode");
47417 
47418   if (!shouldUseHorizontalOp(true, DAG, Subtarget)) {
47419     MVT VT = N->getSimpleValueType(0);
47420     SDValue LHS = N->getOperand(0);
47421     SDValue RHS = N->getOperand(1);
47422 
47423     // HOP(HOP'(X,X),HOP'(Y,Y)) -> HOP(PERMUTE(HOP'(X,Y)),PERMUTE(HOP'(X,Y)).
47424     if (LHS != RHS && LHS.getOpcode() == N->getOpcode() &&
47425         LHS.getOpcode() == RHS.getOpcode() &&
47426         LHS.getValueType() == RHS.getValueType() &&
47427         N->isOnlyUserOf(LHS.getNode()) && N->isOnlyUserOf(RHS.getNode())) {
47428       SDValue LHS0 = LHS.getOperand(0);
47429       SDValue LHS1 = LHS.getOperand(1);
47430       SDValue RHS0 = RHS.getOperand(0);
47431       SDValue RHS1 = RHS.getOperand(1);
47432       if ((LHS0 == LHS1 || LHS0.isUndef() || LHS1.isUndef()) &&
47433           (RHS0 == RHS1 || RHS0.isUndef() || RHS1.isUndef())) {
47434         SDLoc DL(N);
47435         SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
47436                                   LHS0.isUndef() ? LHS1 : LHS0,
47437                                   RHS0.isUndef() ? RHS1 : RHS0);
47438         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
47439         Res = DAG.getBitcast(ShufVT, Res);
47440         SDValue NewLHS =
47441             DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
47442                         getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG));
47443         SDValue NewRHS =
47444             DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
47445                         getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG));
47446         return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS),
47447                            DAG.getBitcast(VT, NewRHS));
47448       }
47449     }
47450   }
47451 
47452   // Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()).
47453   if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
47454     return V;
47455 
47456   return SDValue();
47457 }
47458 
47459 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
47460                                      TargetLowering::DAGCombinerInfo &DCI,
47461                                      const X86Subtarget &Subtarget) {
47462   assert((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() ||
47463           X86ISD::VSRL == N->getOpcode()) &&
47464          "Unexpected shift opcode");
47465   EVT VT = N->getValueType(0);
47466   SDValue N0 = N->getOperand(0);
47467   SDValue N1 = N->getOperand(1);
47468 
47469   // Shift zero -> zero.
47470   if (ISD::isBuildVectorAllZeros(N0.getNode()))
47471     return DAG.getConstant(0, SDLoc(N), VT);
47472 
47473   // Detect constant shift amounts.
47474   APInt UndefElts;
47475   SmallVector<APInt, 32> EltBits;
47476   if (getTargetConstantBitsFromNode(N1, 64, UndefElts, EltBits, true, false)) {
47477     unsigned X86Opc = getTargetVShiftUniformOpcode(N->getOpcode(), false);
47478     return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0,
47479                                       EltBits[0].getZExtValue(), DAG);
47480   }
47481 
47482   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47483   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
47484   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
47485     return SDValue(N, 0);
47486 
47487   return SDValue();
47488 }
47489 
47490 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
47491                                      TargetLowering::DAGCombinerInfo &DCI,
47492                                      const X86Subtarget &Subtarget) {
47493   unsigned Opcode = N->getOpcode();
47494   assert((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode ||
47495           X86ISD::VSRLI == Opcode) &&
47496          "Unexpected shift opcode");
47497   bool LogicalShift = X86ISD::VSHLI == Opcode || X86ISD::VSRLI == Opcode;
47498   EVT VT = N->getValueType(0);
47499   SDValue N0 = N->getOperand(0);
47500   SDValue N1 = N->getOperand(1);
47501   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
47502   assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 &&
47503          "Unexpected value type");
47504   assert(N1.getValueType() == MVT::i8 && "Unexpected shift amount type");
47505 
47506   // (shift undef, X) -> 0
47507   if (N0.isUndef())
47508     return DAG.getConstant(0, SDLoc(N), VT);
47509 
47510   // Out of range logical bit shifts are guaranteed to be zero.
47511   // Out of range arithmetic bit shifts splat the sign bit.
47512   unsigned ShiftVal = N->getConstantOperandVal(1);
47513   if (ShiftVal >= NumBitsPerElt) {
47514     if (LogicalShift)
47515       return DAG.getConstant(0, SDLoc(N), VT);
47516     ShiftVal = NumBitsPerElt - 1;
47517   }
47518 
47519   // (shift X, 0) -> X
47520   if (!ShiftVal)
47521     return N0;
47522 
47523   // (shift 0, C) -> 0
47524   if (ISD::isBuildVectorAllZeros(N0.getNode()))
47525     // N0 is all zeros or undef. We guarantee that the bits shifted into the
47526     // result are all zeros, not undef.
47527     return DAG.getConstant(0, SDLoc(N), VT);
47528 
47529   // (VSRAI -1, C) -> -1
47530   if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
47531     // N0 is all ones or undef. We guarantee that the bits shifted into the
47532     // result are all ones, not undef.
47533     return DAG.getConstant(-1, SDLoc(N), VT);
47534 
47535   auto MergeShifts = [&](SDValue X, uint64_t Amt0, uint64_t Amt1) {
47536     unsigned NewShiftVal = Amt0 + Amt1;
47537     if (NewShiftVal >= NumBitsPerElt) {
47538       // Out of range logical bit shifts are guaranteed to be zero.
47539       // Out of range arithmetic bit shifts splat the sign bit.
47540       if (LogicalShift)
47541         return DAG.getConstant(0, SDLoc(N), VT);
47542       NewShiftVal = NumBitsPerElt - 1;
47543     }
47544     return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0),
47545                        DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
47546   };
47547 
47548   // (shift (shift X, C2), C1) -> (shift X, (C1 + C2))
47549   if (Opcode == N0.getOpcode())
47550     return MergeShifts(N0.getOperand(0), ShiftVal, N0.getConstantOperandVal(1));
47551 
47552   // (shl (add X, X), C) -> (shl X, (C + 1))
47553   if (Opcode == X86ISD::VSHLI && N0.getOpcode() == ISD::ADD &&
47554       N0.getOperand(0) == N0.getOperand(1))
47555     return MergeShifts(N0.getOperand(0), ShiftVal, 1);
47556 
47557   // We can decode 'whole byte' logical bit shifts as shuffles.
47558   if (LogicalShift && (ShiftVal % 8) == 0) {
47559     SDValue Op(N, 0);
47560     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47561       return Res;
47562   }
47563 
47564   // Attempt to detect an expanded vXi64 SIGN_EXTEND_INREG vXi1 pattern, and
47565   // convert to a splatted v2Xi32 SIGN_EXTEND_INREG pattern:
47566   // psrad(pshufd(psllq(X,63),1,1,3,3),31) ->
47567   // pshufd(psrad(pslld(X,31),31),0,0,2,2).
47568   if (Opcode == X86ISD::VSRAI && NumBitsPerElt == 32 && ShiftVal == 31 &&
47569       N0.getOpcode() == X86ISD::PSHUFD &&
47570       N0.getConstantOperandVal(1) == getV4X86ShuffleImm({1, 1, 3, 3}) &&
47571       N0->hasOneUse()) {
47572     SDValue BC = peekThroughOneUseBitcasts(N0.getOperand(0));
47573     if (BC.getOpcode() == X86ISD::VSHLI &&
47574         BC.getScalarValueSizeInBits() == 64 &&
47575         BC.getConstantOperandVal(1) == 63) {
47576       SDLoc DL(N);
47577       SDValue Src = BC.getOperand(0);
47578       Src = DAG.getBitcast(VT, Src);
47579       Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src,
47580                         getV4X86ShuffleImm8ForMask({0, 0, 2, 2}, DL, DAG));
47581       Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1);
47582       Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1);
47583       return Src;
47584     }
47585   }
47586 
47587   auto TryConstantFold = [&](SDValue V) {
47588     APInt UndefElts;
47589     SmallVector<APInt, 32> EltBits;
47590     if (!getTargetConstantBitsFromNode(V, NumBitsPerElt, UndefElts, EltBits))
47591       return SDValue();
47592     assert(EltBits.size() == VT.getVectorNumElements() &&
47593            "Unexpected shift value type");
47594     // Undef elements need to fold to 0. It's possible SimplifyDemandedBits
47595     // created an undef input due to no input bits being demanded, but user
47596     // still expects 0 in other bits.
47597     for (unsigned i = 0, e = EltBits.size(); i != e; ++i) {
47598       APInt &Elt = EltBits[i];
47599       if (UndefElts[i])
47600         Elt = 0;
47601       else if (X86ISD::VSHLI == Opcode)
47602         Elt <<= ShiftVal;
47603       else if (X86ISD::VSRAI == Opcode)
47604         Elt.ashrInPlace(ShiftVal);
47605       else
47606         Elt.lshrInPlace(ShiftVal);
47607     }
47608     // Reset undef elements since they were zeroed above.
47609     UndefElts = 0;
47610     return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));
47611   };
47612 
47613   // Constant Folding.
47614   if (N->isOnlyUserOf(N0.getNode())) {
47615     if (SDValue C = TryConstantFold(N0))
47616       return C;
47617 
47618     // Fold (shift (logic X, C2), C1) -> (logic (shift X, C1), (shift C2, C1))
47619     // Don't break NOT patterns.
47620     SDValue BC = peekThroughOneUseBitcasts(N0);
47621     if (ISD::isBitwiseLogicOp(BC.getOpcode()) &&
47622         BC->isOnlyUserOf(BC.getOperand(1).getNode()) &&
47623         !ISD::isBuildVectorAllOnes(BC.getOperand(1).getNode())) {
47624       if (SDValue RHS = TryConstantFold(BC.getOperand(1))) {
47625         SDLoc DL(N);
47626         SDValue LHS = DAG.getNode(Opcode, DL, VT,
47627                                   DAG.getBitcast(VT, BC.getOperand(0)), N1);
47628         return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS);
47629       }
47630     }
47631   }
47632 
47633   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47634   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBitsPerElt),
47635                                DCI))
47636     return SDValue(N, 0);
47637 
47638   return SDValue();
47639 }
47640 
47641 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
47642                                    TargetLowering::DAGCombinerInfo &DCI,
47643                                    const X86Subtarget &Subtarget) {
47644   EVT VT = N->getValueType(0);
47645   unsigned Opcode = N->getOpcode();
47646   assert(((Opcode == X86ISD::PINSRB && VT == MVT::v16i8) ||
47647           (Opcode == X86ISD::PINSRW && VT == MVT::v8i16) ||
47648           Opcode == ISD::INSERT_VECTOR_ELT) &&
47649          "Unexpected vector insertion");
47650 
47651   SDValue Vec = N->getOperand(0);
47652   SDValue Scl = N->getOperand(1);
47653   SDValue Idx = N->getOperand(2);
47654 
47655   // Fold insert_vector_elt(undef, elt, 0) --> scalar_to_vector(elt).
47656   if (Opcode == ISD::INSERT_VECTOR_ELT && Vec.isUndef() && isNullConstant(Idx))
47657     return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl);
47658 
47659   if (Opcode == X86ISD::PINSRB || Opcode == X86ISD::PINSRW) {
47660     unsigned NumBitsPerElt = VT.getScalarSizeInBits();
47661     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47662     if (TLI.SimplifyDemandedBits(SDValue(N, 0),
47663                                  APInt::getAllOnes(NumBitsPerElt), DCI))
47664       return SDValue(N, 0);
47665   }
47666 
47667   // Attempt to combine insertion patterns to a shuffle.
47668   if (VT.isSimple() && DCI.isAfterLegalizeDAG()) {
47669     SDValue Op(N, 0);
47670     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47671       return Res;
47672   }
47673 
47674   return SDValue();
47675 }
47676 
47677 /// Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs
47678 /// reference the same FP CMP, and rewrite for CMPEQSS and friends. Likewise for
47679 /// OR -> CMPNEQSS.
47680 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
47681                                    TargetLowering::DAGCombinerInfo &DCI,
47682                                    const X86Subtarget &Subtarget) {
47683   unsigned opcode;
47684 
47685   // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
47686   // we're requiring SSE2 for both.
47687   if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
47688     SDValue N0 = N->getOperand(0);
47689     SDValue N1 = N->getOperand(1);
47690     SDValue CMP0 = N0.getOperand(1);
47691     SDValue CMP1 = N1.getOperand(1);
47692     SDLoc DL(N);
47693 
47694     // The SETCCs should both refer to the same CMP.
47695     if (CMP0.getOpcode() != X86ISD::FCMP || CMP0 != CMP1)
47696       return SDValue();
47697 
47698     SDValue CMP00 = CMP0->getOperand(0);
47699     SDValue CMP01 = CMP0->getOperand(1);
47700     EVT     VT    = CMP00.getValueType();
47701 
47702     if (VT == MVT::f32 || VT == MVT::f64 ||
47703         (VT == MVT::f16 && Subtarget.hasFP16())) {
47704       bool ExpectingFlags = false;
47705       // Check for any users that want flags:
47706       for (const SDNode *U : N->uses()) {
47707         if (ExpectingFlags)
47708           break;
47709 
47710         switch (U->getOpcode()) {
47711         default:
47712         case ISD::BR_CC:
47713         case ISD::BRCOND:
47714         case ISD::SELECT:
47715           ExpectingFlags = true;
47716           break;
47717         case ISD::CopyToReg:
47718         case ISD::SIGN_EXTEND:
47719         case ISD::ZERO_EXTEND:
47720         case ISD::ANY_EXTEND:
47721           break;
47722         }
47723       }
47724 
47725       if (!ExpectingFlags) {
47726         enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
47727         enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
47728 
47729         if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
47730           X86::CondCode tmp = cc0;
47731           cc0 = cc1;
47732           cc1 = tmp;
47733         }
47734 
47735         if ((cc0 == X86::COND_E  && cc1 == X86::COND_NP) ||
47736             (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
47737           // FIXME: need symbolic constants for these magic numbers.
47738           // See X86ATTInstPrinter.cpp:printSSECC().
47739           unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
47740           if (Subtarget.hasAVX512()) {
47741             SDValue FSetCC =
47742                 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
47743                             DAG.getTargetConstant(x86cc, DL, MVT::i8));
47744             // Need to fill with zeros to ensure the bitcast will produce zeroes
47745             // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
47746             SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
47747                                       DAG.getConstant(0, DL, MVT::v16i1),
47748                                       FSetCC, DAG.getIntPtrConstant(0, DL));
47749             return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
47750                                       N->getSimpleValueType(0));
47751           }
47752           SDValue OnesOrZeroesF =
47753               DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
47754                           CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
47755 
47756           bool is64BitFP = (CMP00.getValueType() == MVT::f64);
47757           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
47758 
47759           if (is64BitFP && !Subtarget.is64Bit()) {
47760             // On a 32-bit target, we cannot bitcast the 64-bit float to a
47761             // 64-bit integer, since that's not a legal type. Since
47762             // OnesOrZeroesF is all ones or all zeroes, we don't need all the
47763             // bits, but can do this little dance to extract the lowest 32 bits
47764             // and work with those going forward.
47765             SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
47766                                            OnesOrZeroesF);
47767             SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
47768             OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
47769                                         Vector32, DAG.getIntPtrConstant(0, DL));
47770             IntVT = MVT::i32;
47771           }
47772 
47773           SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
47774           SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
47775                                       DAG.getConstant(1, DL, IntVT));
47776           SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
47777                                               ANDed);
47778           return OneBitOfTruth;
47779         }
47780       }
47781     }
47782   }
47783   return SDValue();
47784 }
47785 
47786 /// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
47787 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
47788   assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
47789 
47790   MVT VT = N->getSimpleValueType(0);
47791   if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())
47792     return SDValue();
47793 
47794   SDValue X, Y;
47795   SDValue N0 = N->getOperand(0);
47796   SDValue N1 = N->getOperand(1);
47797 
47798   if (SDValue Not = IsNOT(N0, DAG)) {
47799     X = Not;
47800     Y = N1;
47801   } else if (SDValue Not = IsNOT(N1, DAG)) {
47802     X = Not;
47803     Y = N0;
47804   } else
47805     return SDValue();
47806 
47807   X = DAG.getBitcast(VT, X);
47808   Y = DAG.getBitcast(VT, Y);
47809   return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
47810 }
47811 
47812 /// Try to fold:
47813 ///   and (vector_shuffle<Z,...,Z>
47814 ///            (insert_vector_elt undef, (xor X, -1), Z), undef), Y
47815 ///   ->
47816 ///   andnp (vector_shuffle<Z,...,Z>
47817 ///              (insert_vector_elt undef, X, Z), undef), Y
47818 static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
47819                                     const X86Subtarget &Subtarget) {
47820   assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
47821 
47822   EVT VT = N->getValueType(0);
47823   // Do not split 256 and 512 bit vectors with SSE2 as they overwrite original
47824   // value and require extra moves.
47825   if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
47826         ((VT.is256BitVector() || VT.is512BitVector()) && Subtarget.hasAVX())))
47827     return SDValue();
47828 
47829   auto GetNot = [&DAG](SDValue V) {
47830     auto *SVN = dyn_cast<ShuffleVectorSDNode>(peekThroughOneUseBitcasts(V));
47831     // TODO: SVN->hasOneUse() is a strong condition. It can be relaxed if all
47832     // end-users are ISD::AND including cases
47833     // (and(extract_vector_element(SVN), Y)).
47834     if (!SVN || !SVN->hasOneUse() || !SVN->isSplat() ||
47835         !SVN->getOperand(1).isUndef()) {
47836       return SDValue();
47837     }
47838     SDValue IVEN = SVN->getOperand(0);
47839     if (IVEN.getOpcode() != ISD::INSERT_VECTOR_ELT ||
47840         !IVEN.getOperand(0).isUndef() || !IVEN.hasOneUse())
47841       return SDValue();
47842     if (!isa<ConstantSDNode>(IVEN.getOperand(2)) ||
47843         IVEN.getConstantOperandAPInt(2) != SVN->getSplatIndex())
47844       return SDValue();
47845     SDValue Src = IVEN.getOperand(1);
47846     if (SDValue Not = IsNOT(Src, DAG)) {
47847       SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
47848       SDValue NotIVEN =
47849           DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(),
47850                       IVEN.getOperand(0), NotSrc, IVEN.getOperand(2));
47851       return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN,
47852                                   SVN->getOperand(1), SVN->getMask());
47853     }
47854     return SDValue();
47855   };
47856 
47857   SDValue X, Y;
47858   SDValue N0 = N->getOperand(0);
47859   SDValue N1 = N->getOperand(1);
47860 
47861   if (SDValue Not = GetNot(N0)) {
47862     X = Not;
47863     Y = N1;
47864   } else if (SDValue Not = GetNot(N1)) {
47865     X = Not;
47866     Y = N0;
47867   } else
47868     return SDValue();
47869 
47870   X = DAG.getBitcast(VT, X);
47871   Y = DAG.getBitcast(VT, Y);
47872   SDLoc DL(N);
47873   // We do not split for SSE at all, but we need to split vectors for AVX1 and
47874   // AVX2.
47875   if (!Subtarget.useAVX512Regs() && VT.is512BitVector()) {
47876     SDValue LoX, HiX;
47877     std::tie(LoX, HiX) = splitVector(X, DAG, DL);
47878     SDValue LoY, HiY;
47879     std::tie(LoY, HiY) = splitVector(Y, DAG, DL);
47880     EVT SplitVT = LoX.getValueType();
47881     SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
47882     SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
47883     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV});
47884   }
47885   return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y});
47886 }
47887 
47888 // Try to widen AND, OR and XOR nodes to VT in order to remove casts around
47889 // logical operations, like in the example below.
47890 //   or (and (truncate x, truncate y)),
47891 //      (xor (truncate z, build_vector (constants)))
47892 // Given a target type \p VT, we generate
47893 //   or (and x, y), (xor z, zext(build_vector (constants)))
47894 // given x, y and z are of type \p VT. We can do so, if operands are either
47895 // truncates from VT types, the second operand is a vector of constants or can
47896 // be recursively promoted.
47897 static SDValue PromoteMaskArithmetic(SDNode *N, EVT VT, SelectionDAG &DAG,
47898                                      unsigned Depth) {
47899   // Limit recursion to avoid excessive compile times.
47900   if (Depth >= SelectionDAG::MaxRecursionDepth)
47901     return SDValue();
47902 
47903   if (N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND &&
47904       N->getOpcode() != ISD::OR)
47905     return SDValue();
47906 
47907   SDValue N0 = N->getOperand(0);
47908   SDValue N1 = N->getOperand(1);
47909   SDLoc DL(N);
47910 
47911   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47912   if (!TLI.isOperationLegalOrPromote(N->getOpcode(), VT))
47913     return SDValue();
47914 
47915   if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1))
47916     N0 = NN0;
47917   else {
47918     // The Left side has to be a trunc.
47919     if (N0.getOpcode() != ISD::TRUNCATE)
47920       return SDValue();
47921 
47922     // The type of the truncated inputs.
47923     if (N0.getOperand(0).getValueType() != VT)
47924       return SDValue();
47925 
47926     N0 = N0.getOperand(0);
47927   }
47928 
47929   if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1))
47930     N1 = NN1;
47931   else {
47932     // The right side has to be a 'trunc' or a constant vector.
47933     bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
47934                     N1.getOperand(0).getValueType() == VT;
47935     if (!RHSTrunc && !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()))
47936       return SDValue();
47937 
47938     if (RHSTrunc)
47939       N1 = N1.getOperand(0);
47940     else
47941       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);
47942   }
47943 
47944   return DAG.getNode(N->getOpcode(), DL, VT, N0, N1);
47945 }
47946 
47947 // On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
47948 // register. In most cases we actually compare or select YMM-sized registers
47949 // and mixing the two types creates horrible code. This method optimizes
47950 // some of the transition sequences.
47951 // Even with AVX-512 this is still useful for removing casts around logical
47952 // operations on vXi1 mask types.
47953 static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
47954                                      const X86Subtarget &Subtarget) {
47955   EVT VT = N->getValueType(0);
47956   assert(VT.isVector() && "Expected vector type");
47957 
47958   SDLoc DL(N);
47959   assert((N->getOpcode() == ISD::ANY_EXTEND ||
47960           N->getOpcode() == ISD::ZERO_EXTEND ||
47961           N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
47962 
47963   SDValue Narrow = N->getOperand(0);
47964   EVT NarrowVT = Narrow.getValueType();
47965 
47966   // Generate the wide operation.
47967   SDValue Op = PromoteMaskArithmetic(Narrow.getNode(), VT, DAG, 0);
47968   if (!Op)
47969     return SDValue();
47970   switch (N->getOpcode()) {
47971   default: llvm_unreachable("Unexpected opcode");
47972   case ISD::ANY_EXTEND:
47973     return Op;
47974   case ISD::ZERO_EXTEND:
47975     return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
47976   case ISD::SIGN_EXTEND:
47977     return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
47978                        Op, DAG.getValueType(NarrowVT));
47979   }
47980 }
47981 
47982 static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
47983   unsigned FPOpcode;
47984   switch (Opcode) {
47985   default: llvm_unreachable("Unexpected input node for FP logic conversion");
47986   case ISD::AND: FPOpcode = X86ISD::FAND; break;
47987   case ISD::OR:  FPOpcode = X86ISD::FOR;  break;
47988   case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
47989   }
47990   return FPOpcode;
47991 }
47992 
47993 /// If both input operands of a logic op are being cast from floating-point
47994 /// types or FP compares, try to convert this into a floating-point logic node
47995 /// to avoid unnecessary moves from SSE to integer registers.
47996 static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
47997                                         TargetLowering::DAGCombinerInfo &DCI,
47998                                         const X86Subtarget &Subtarget) {
47999   EVT VT = N->getValueType(0);
48000   SDValue N0 = N->getOperand(0);
48001   SDValue N1 = N->getOperand(1);
48002   SDLoc DL(N);
48003 
48004   if (!((N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) ||
48005         (N0.getOpcode() == ISD::SETCC && N1.getOpcode() == ISD::SETCC)))
48006     return SDValue();
48007 
48008   SDValue N00 = N0.getOperand(0);
48009   SDValue N10 = N1.getOperand(0);
48010   EVT N00Type = N00.getValueType();
48011   EVT N10Type = N10.getValueType();
48012 
48013   // Ensure that both types are the same and are legal scalar fp types.
48014   if (N00Type != N10Type || !((Subtarget.hasSSE1() && N00Type == MVT::f32) ||
48015                               (Subtarget.hasSSE2() && N00Type == MVT::f64) ||
48016                               (Subtarget.hasFP16() && N00Type == MVT::f16)))
48017     return SDValue();
48018 
48019   if (N0.getOpcode() == ISD::BITCAST && !DCI.isBeforeLegalizeOps()) {
48020     unsigned FPOpcode = convertIntLogicToFPLogicOpcode(N->getOpcode());
48021     SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
48022     return DAG.getBitcast(VT, FPLogic);
48023   }
48024 
48025   if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() ||
48026       !N1.hasOneUse())
48027     return SDValue();
48028 
48029   ISD::CondCode CC0 = cast<CondCodeSDNode>(N0.getOperand(2))->get();
48030   ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get();
48031 
48032   // The vector ISA for FP predicates is incomplete before AVX, so converting
48033   // COMIS* to CMPS* may not be a win before AVX.
48034   if (!Subtarget.hasAVX() &&
48035       !(cheapX86FSETCC_SSE(CC0) && cheapX86FSETCC_SSE(CC1)))
48036     return SDValue();
48037 
48038   // Convert scalar FP compares and logic to vector compares (COMIS* to CMPS*)
48039   // and vector logic:
48040   // logic (setcc N00, N01), (setcc N10, N11) -->
48041   // extelt (logic (setcc (s2v N00), (s2v N01)), setcc (s2v N10), (s2v N11))), 0
48042   unsigned NumElts = 128 / N00Type.getSizeInBits();
48043   EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts);
48044   EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
48045   SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL);
48046   SDValue N01 = N0.getOperand(1);
48047   SDValue N11 = N1.getOperand(1);
48048   SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00);
48049   SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01);
48050   SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10);
48051   SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11);
48052   SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0);
48053   SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1);
48054   SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1);
48055   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex);
48056 }
48057 
48058 // Attempt to fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
48059 // to reduce XMM->GPR traffic.
48060 static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) {
48061   unsigned Opc = N->getOpcode();
48062   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48063          "Unexpected bit opcode");
48064 
48065   SDValue N0 = N->getOperand(0);
48066   SDValue N1 = N->getOperand(1);
48067 
48068   // Both operands must be single use MOVMSK.
48069   if (N0.getOpcode() != X86ISD::MOVMSK || !N0.hasOneUse() ||
48070       N1.getOpcode() != X86ISD::MOVMSK || !N1.hasOneUse())
48071     return SDValue();
48072 
48073   SDValue Vec0 = N0.getOperand(0);
48074   SDValue Vec1 = N1.getOperand(0);
48075   EVT VecVT0 = Vec0.getValueType();
48076   EVT VecVT1 = Vec1.getValueType();
48077 
48078   // Both MOVMSK operands must be from vectors of the same size and same element
48079   // size, but its OK for a fp/int diff.
48080   if (VecVT0.getSizeInBits() != VecVT1.getSizeInBits() ||
48081       VecVT0.getScalarSizeInBits() != VecVT1.getScalarSizeInBits())
48082     return SDValue();
48083 
48084   SDLoc DL(N);
48085   unsigned VecOpc =
48086       VecVT0.isFloatingPoint() ? convertIntLogicToFPLogicOpcode(Opc) : Opc;
48087   SDValue Result =
48088       DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
48089   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
48090 }
48091 
48092 // Attempt to fold BITOP(SHIFT(X,Z),SHIFT(Y,Z)) -> SHIFT(BITOP(X,Y),Z).
48093 // NOTE: This is a very limited case of what SimplifyUsingDistributiveLaws
48094 // handles in InstCombine.
48095 static SDValue combineBitOpWithShift(SDNode *N, SelectionDAG &DAG) {
48096   unsigned Opc = N->getOpcode();
48097   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48098          "Unexpected bit opcode");
48099 
48100   SDValue N0 = N->getOperand(0);
48101   SDValue N1 = N->getOperand(1);
48102   EVT VT = N->getValueType(0);
48103 
48104   // Both operands must be single use.
48105   if (!N0.hasOneUse() || !N1.hasOneUse())
48106     return SDValue();
48107 
48108   // Search for matching shifts.
48109   SDValue BC0 = peekThroughOneUseBitcasts(N0);
48110   SDValue BC1 = peekThroughOneUseBitcasts(N1);
48111 
48112   unsigned BCOpc = BC0.getOpcode();
48113   EVT BCVT = BC0.getValueType();
48114   if (BCOpc != BC1->getOpcode() || BCVT != BC1.getValueType())
48115     return SDValue();
48116 
48117   switch (BCOpc) {
48118   case X86ISD::VSHLI:
48119   case X86ISD::VSRLI:
48120   case X86ISD::VSRAI: {
48121     if (BC0.getOperand(1) != BC1.getOperand(1))
48122       return SDValue();
48123 
48124     SDLoc DL(N);
48125     SDValue BitOp =
48126         DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0));
48127     SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1));
48128     return DAG.getBitcast(VT, Shift);
48129   }
48130   }
48131 
48132   return SDValue();
48133 }
48134 
48135 // Attempt to fold:
48136 // BITOP(PACKSS(X,Z),PACKSS(Y,W)) --> PACKSS(BITOP(X,Y),BITOP(Z,W)).
48137 // TODO: Handle PACKUS handling.
48138 static SDValue combineBitOpWithPACK(SDNode *N, SelectionDAG &DAG) {
48139   unsigned Opc = N->getOpcode();
48140   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48141          "Unexpected bit opcode");
48142 
48143   SDValue N0 = N->getOperand(0);
48144   SDValue N1 = N->getOperand(1);
48145   EVT VT = N->getValueType(0);
48146 
48147   // Both operands must be single use.
48148   if (!N0.hasOneUse() || !N1.hasOneUse())
48149     return SDValue();
48150 
48151   // Search for matching packs.
48152   N0 = peekThroughOneUseBitcasts(N0);
48153   N1 = peekThroughOneUseBitcasts(N1);
48154 
48155   if (N0.getOpcode() != X86ISD::PACKSS || N1.getOpcode() != X86ISD::PACKSS)
48156     return SDValue();
48157 
48158   MVT DstVT = N0.getSimpleValueType();
48159   if (DstVT != N1.getSimpleValueType())
48160     return SDValue();
48161 
48162   MVT SrcVT = N0.getOperand(0).getSimpleValueType();
48163   unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
48164 
48165   // Limit to allsignbits packing.
48166   if (DAG.ComputeNumSignBits(N0.getOperand(0)) != NumSrcBits ||
48167       DAG.ComputeNumSignBits(N0.getOperand(1)) != NumSrcBits ||
48168       DAG.ComputeNumSignBits(N1.getOperand(0)) != NumSrcBits ||
48169       DAG.ComputeNumSignBits(N1.getOperand(1)) != NumSrcBits)
48170     return SDValue();
48171 
48172   SDLoc DL(N);
48173   SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0));
48174   SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1));
48175   return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS));
48176 }
48177 
48178 /// If this is a zero/all-bits result that is bitwise-anded with a low bits
48179 /// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
48180 /// with a shift-right to eliminate loading the vector constant mask value.
48181 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
48182                                      const X86Subtarget &Subtarget) {
48183   SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
48184   SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
48185   EVT VT = Op0.getValueType();
48186   if (VT != Op1.getValueType() || !VT.isSimple() || !VT.isInteger())
48187     return SDValue();
48188 
48189   // Try to convert an "is positive" signbit masking operation into arithmetic
48190   // shift and "andn". This saves a materialization of a -1 vector constant.
48191   // The "is negative" variant should be handled more generally because it only
48192   // requires "and" rather than "andn":
48193   // and (pcmpgt X, -1), Y --> pandn (vsrai X, BitWidth - 1), Y
48194   //
48195   // This is limited to the original type to avoid producing even more bitcasts.
48196   // If the bitcasts can't be eliminated, then it is unlikely that this fold
48197   // will be profitable.
48198   if (N->getValueType(0) == VT &&
48199       supportedVectorShiftWithImm(VT, Subtarget, ISD::SRA)) {
48200     SDValue X, Y;
48201     if (Op1.getOpcode() == X86ISD::PCMPGT &&
48202         isAllOnesOrAllOnesSplat(Op1.getOperand(1)) && Op1.hasOneUse()) {
48203       X = Op1.getOperand(0);
48204       Y = Op0;
48205     } else if (Op0.getOpcode() == X86ISD::PCMPGT &&
48206                isAllOnesOrAllOnesSplat(Op0.getOperand(1)) && Op0.hasOneUse()) {
48207       X = Op0.getOperand(0);
48208       Y = Op1;
48209     }
48210     if (X && Y) {
48211       SDLoc DL(N);
48212       SDValue Sra =
48213           getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X,
48214                                      VT.getScalarSizeInBits() - 1, DAG);
48215       return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y);
48216     }
48217   }
48218 
48219   APInt SplatVal;
48220   if (!X86::isConstantSplat(Op1, SplatVal, false) || !SplatVal.isMask())
48221     return SDValue();
48222 
48223   // Don't prevent creation of ANDN.
48224   if (isBitwiseNot(Op0))
48225     return SDValue();
48226 
48227   if (!supportedVectorShiftWithImm(VT, Subtarget, ISD::SRL))
48228     return SDValue();
48229 
48230   unsigned EltBitWidth = VT.getScalarSizeInBits();
48231   if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
48232     return SDValue();
48233 
48234   SDLoc DL(N);
48235   unsigned ShiftVal = SplatVal.countr_one();
48236   SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
48237   SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
48238   return DAG.getBitcast(N->getValueType(0), Shift);
48239 }
48240 
48241 // Get the index node from the lowered DAG of a GEP IR instruction with one
48242 // indexing dimension.
48243 static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
48244   if (Ld->isIndexed())
48245     return SDValue();
48246 
48247   SDValue Base = Ld->getBasePtr();
48248 
48249   if (Base.getOpcode() != ISD::ADD)
48250     return SDValue();
48251 
48252   SDValue ShiftedIndex = Base.getOperand(0);
48253 
48254   if (ShiftedIndex.getOpcode() != ISD::SHL)
48255     return SDValue();
48256 
48257   return ShiftedIndex.getOperand(0);
48258 
48259 }
48260 
48261 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
48262   if (Subtarget.hasBMI2() && VT.isScalarInteger()) {
48263     switch (VT.getSizeInBits()) {
48264     default: return false;
48265     case 64: return Subtarget.is64Bit() ? true : false;
48266     case 32: return true;
48267     }
48268   }
48269   return false;
48270 }
48271 
48272 // This function recognizes cases where X86 bzhi instruction can replace and
48273 // 'and-load' sequence.
48274 // In case of loading integer value from an array of constants which is defined
48275 // as follows:
48276 //
48277 //   int array[SIZE] = {0x0, 0x1, 0x3, 0x7, 0xF ..., 2^(SIZE-1) - 1}
48278 //
48279 // then applying a bitwise and on the result with another input.
48280 // It's equivalent to performing bzhi (zero high bits) on the input, with the
48281 // same index of the load.
48282 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
48283                                     const X86Subtarget &Subtarget) {
48284   MVT VT = Node->getSimpleValueType(0);
48285   SDLoc dl(Node);
48286 
48287   // Check if subtarget has BZHI instruction for the node's type
48288   if (!hasBZHI(Subtarget, VT))
48289     return SDValue();
48290 
48291   // Try matching the pattern for both operands.
48292   for (unsigned i = 0; i < 2; i++) {
48293     SDValue N = Node->getOperand(i);
48294     LoadSDNode *Ld = dyn_cast<LoadSDNode>(N.getNode());
48295 
48296      // continue if the operand is not a load instruction
48297     if (!Ld)
48298       return SDValue();
48299 
48300     const Value *MemOp = Ld->getMemOperand()->getValue();
48301 
48302     if (!MemOp)
48303       return SDValue();
48304 
48305     if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(MemOp)) {
48306       if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) {
48307         if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
48308 
48309           Constant *Init = GV->getInitializer();
48310           Type *Ty = Init->getType();
48311           if (!isa<ConstantDataArray>(Init) ||
48312               !Ty->getArrayElementType()->isIntegerTy() ||
48313               Ty->getArrayElementType()->getScalarSizeInBits() !=
48314                   VT.getSizeInBits() ||
48315               Ty->getArrayNumElements() >
48316                   Ty->getArrayElementType()->getScalarSizeInBits())
48317             continue;
48318 
48319           // Check if the array's constant elements are suitable to our case.
48320           uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
48321           bool ConstantsMatch = true;
48322           for (uint64_t j = 0; j < ArrayElementCount; j++) {
48323             auto *Elem = cast<ConstantInt>(Init->getAggregateElement(j));
48324             if (Elem->getZExtValue() != (((uint64_t)1 << j) - 1)) {
48325               ConstantsMatch = false;
48326               break;
48327             }
48328           }
48329           if (!ConstantsMatch)
48330             continue;
48331 
48332           // Do the transformation (For 32-bit type):
48333           // -> (and (load arr[idx]), inp)
48334           // <- (and (srl 0xFFFFFFFF, (sub 32, idx)))
48335           //    that will be replaced with one bzhi instruction.
48336           SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
48337           SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
48338 
48339           // Get the Node which indexes into the array.
48340           SDValue Index = getIndexFromUnindexedLoad(Ld);
48341           if (!Index)
48342             return SDValue();
48343           Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);
48344 
48345           SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
48346           Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);
48347 
48348           SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
48349           SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
48350 
48351           return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);
48352         }
48353       }
48354     }
48355   }
48356   return SDValue();
48357 }
48358 
48359 // Look for (and (bitcast (vXi1 (concat_vectors (vYi1 setcc), undef,))), C)
48360 // Where C is a mask containing the same number of bits as the setcc and
48361 // where the setcc will freely 0 upper bits of k-register. We can replace the
48362 // undef in the concat with 0s and remove the AND. This mainly helps with
48363 // v2i1/v4i1 setcc being casted to scalar.
48364 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
48365                                              const X86Subtarget &Subtarget) {
48366   assert(N->getOpcode() == ISD::AND && "Unexpected opcode!");
48367 
48368   EVT VT = N->getValueType(0);
48369 
48370   // Make sure this is an AND with constant. We will check the value of the
48371   // constant later.
48372   auto *C1 = dyn_cast<ConstantSDNode>(N->getOperand(1));
48373   if (!C1)
48374     return SDValue();
48375 
48376   // This is implied by the ConstantSDNode.
48377   assert(!VT.isVector() && "Expected scalar VT!");
48378 
48379   SDValue Src = N->getOperand(0);
48380   if (!Src.hasOneUse())
48381     return SDValue();
48382 
48383   // (Optionally) peek through any_extend().
48384   if (Src.getOpcode() == ISD::ANY_EXTEND) {
48385     if (!Src.getOperand(0).hasOneUse())
48386       return SDValue();
48387     Src = Src.getOperand(0);
48388   }
48389 
48390   if (Src.getOpcode() != ISD::BITCAST || !Src.getOperand(0).hasOneUse())
48391     return SDValue();
48392 
48393   Src = Src.getOperand(0);
48394   EVT SrcVT = Src.getValueType();
48395 
48396   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48397   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 ||
48398       !TLI.isTypeLegal(SrcVT))
48399     return SDValue();
48400 
48401   if (Src.getOpcode() != ISD::CONCAT_VECTORS)
48402     return SDValue();
48403 
48404   // We only care about the first subvector of the concat, we expect the
48405   // other subvectors to be ignored due to the AND if we make the change.
48406   SDValue SubVec = Src.getOperand(0);
48407   EVT SubVecVT = SubVec.getValueType();
48408 
48409   // The RHS of the AND should be a mask with as many bits as SubVec.
48410   if (!TLI.isTypeLegal(SubVecVT) ||
48411       !C1->getAPIntValue().isMask(SubVecVT.getVectorNumElements()))
48412     return SDValue();
48413 
48414   // First subvector should be a setcc with a legal result type or a
48415   // AND containing at least one setcc with a legal result type.
48416   auto IsLegalSetCC = [&](SDValue V) {
48417     if (V.getOpcode() != ISD::SETCC)
48418       return false;
48419     EVT SetccVT = V.getOperand(0).getValueType();
48420     if (!TLI.isTypeLegal(SetccVT) ||
48421         !(Subtarget.hasVLX() || SetccVT.is512BitVector()))
48422       return false;
48423     if (!(Subtarget.hasBWI() || SetccVT.getScalarSizeInBits() >= 32))
48424       return false;
48425     return true;
48426   };
48427   if (!(IsLegalSetCC(SubVec) || (SubVec.getOpcode() == ISD::AND &&
48428                                  (IsLegalSetCC(SubVec.getOperand(0)) ||
48429                                   IsLegalSetCC(SubVec.getOperand(1))))))
48430     return SDValue();
48431 
48432   // We passed all the checks. Rebuild the concat_vectors with zeroes
48433   // and cast it back to VT.
48434   SDLoc dl(N);
48435   SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
48436                               DAG.getConstant(0, dl, SubVecVT));
48437   Ops[0] = SubVec;
48438   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
48439                                Ops);
48440   EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
48441   return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
48442 }
48443 
48444 static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG,
48445                                 SDValue OpMustEq, SDValue Op, unsigned Depth) {
48446   // We don't want to go crazy with the recursion here. This isn't a super
48447   // important optimization.
48448   static constexpr unsigned kMaxDepth = 2;
48449 
48450   // Only do this re-ordering if op has one use.
48451   if (!Op.hasOneUse())
48452     return SDValue();
48453 
48454   SDLoc DL(Op);
48455   // If we hit another assosiative op, recurse further.
48456   if (Op.getOpcode() == Opc) {
48457     // Done recursing.
48458     if (Depth++ >= kMaxDepth)
48459       return SDValue();
48460 
48461     for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
48462       if (SDValue R =
48463               getBMIMatchingOp(Opc, DAG, OpMustEq, Op.getOperand(OpIdx), Depth))
48464         return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), R,
48465                            Op.getOperand(1 - OpIdx));
48466 
48467   } else if (Op.getOpcode() == ISD::SUB) {
48468     if (Opc == ISD::AND) {
48469       // BLSI: (and x, (sub 0, x))
48470       if (isNullConstant(Op.getOperand(0)) && Op.getOperand(1) == OpMustEq)
48471         return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48472     }
48473     // Opc must be ISD::AND or ISD::XOR
48474     // BLSR: (and x, (sub x, 1))
48475     // BLSMSK: (xor x, (sub x, 1))
48476     if (isOneConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
48477       return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48478 
48479   } else if (Op.getOpcode() == ISD::ADD) {
48480     // Opc must be ISD::AND or ISD::XOR
48481     // BLSR: (and x, (add x, -1))
48482     // BLSMSK: (xor x, (add x, -1))
48483     if (isAllOnesConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
48484       return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48485   }
48486   return SDValue();
48487 }
48488 
48489 static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG,
48490                                  const X86Subtarget &Subtarget) {
48491   EVT VT = N->getValueType(0);
48492   // Make sure this node is a candidate for BMI instructions.
48493   if (!Subtarget.hasBMI() || !VT.isScalarInteger() ||
48494       (VT != MVT::i32 && VT != MVT::i64))
48495     return SDValue();
48496 
48497   assert(N->getOpcode() == ISD::AND || N->getOpcode() == ISD::XOR);
48498 
48499   // Try and match LHS and RHS.
48500   for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
48501     if (SDValue OpMatch =
48502             getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx),
48503                              N->getOperand(1 - OpIdx), 0))
48504       return OpMatch;
48505   return SDValue();
48506 }
48507 
48508 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
48509                           TargetLowering::DAGCombinerInfo &DCI,
48510                           const X86Subtarget &Subtarget) {
48511   SDValue N0 = N->getOperand(0);
48512   SDValue N1 = N->getOperand(1);
48513   EVT VT = N->getValueType(0);
48514   SDLoc dl(N);
48515   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48516 
48517   // If this is SSE1 only convert to FAND to avoid scalarization.
48518   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
48519     return DAG.getBitcast(MVT::v4i32,
48520                           DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,
48521                                       DAG.getBitcast(MVT::v4f32, N0),
48522                                       DAG.getBitcast(MVT::v4f32, N1)));
48523   }
48524 
48525   // Use a 32-bit and+zext if upper bits known zero.
48526   if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) {
48527     APInt HiMask = APInt::getHighBitsSet(64, 32);
48528     if (DAG.MaskedValueIsZero(N1, HiMask) ||
48529         DAG.MaskedValueIsZero(N0, HiMask)) {
48530       SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
48531       SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
48532       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
48533                          DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
48534     }
48535   }
48536 
48537   // Match all-of bool scalar reductions into a bitcast/movmsk + cmp.
48538   // TODO: Support multiple SrcOps.
48539   if (VT == MVT::i1) {
48540     SmallVector<SDValue, 2> SrcOps;
48541     SmallVector<APInt, 2> SrcPartials;
48542     if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
48543         SrcOps.size() == 1) {
48544       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
48545       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
48546       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
48547       if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
48548         Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
48549       if (Mask) {
48550         assert(SrcPartials[0].getBitWidth() == NumElts &&
48551                "Unexpected partial reduction mask");
48552         SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
48553         Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
48554         return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ);
48555       }
48556     }
48557   }
48558 
48559   // InstCombine converts:
48560   //    `(-x << C0) & C1`
48561   // to
48562   //    `(x * (Pow2_Ceil(C1) - (1 << C0))) & C1`
48563   // This saves an IR instruction but on x86 the neg/shift version is preferable
48564   // so undo the transform.
48565 
48566   if (N0.getOpcode() == ISD::MUL && N0.hasOneUse()) {
48567     // TODO: We don't actually need a splat for this, we just need the checks to
48568     // hold for each element.
48569     ConstantSDNode *N1C = isConstOrConstSplat(N1, /*AllowUndefs*/ true,
48570                                               /*AllowTruncation*/ false);
48571     ConstantSDNode *N01C =
48572         isConstOrConstSplat(N0.getOperand(1), /*AllowUndefs*/ true,
48573                             /*AllowTruncation*/ false);
48574     if (N1C && N01C) {
48575       const APInt &MulC = N01C->getAPIntValue();
48576       const APInt &AndC = N1C->getAPIntValue();
48577       APInt MulCLowBit = MulC & (-MulC);
48578       if (MulC.uge(AndC) && !MulC.isPowerOf2() &&
48579           (MulCLowBit + MulC).isPowerOf2()) {
48580         SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
48581                                   N0.getOperand(0));
48582         int32_t MulCLowBitLog = MulCLowBit.exactLogBase2();
48583         assert(MulCLowBitLog != -1 &&
48584                "Isolated lowbit is somehow not a power of 2!");
48585         SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg,
48586                                     DAG.getConstant(MulCLowBitLog, dl, VT));
48587         return DAG.getNode(ISD::AND, dl, VT, Shift, N1);
48588       }
48589     }
48590   }
48591 
48592   if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
48593     return V;
48594 
48595   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
48596     return R;
48597 
48598   if (SDValue R = combineBitOpWithShift(N, DAG))
48599     return R;
48600 
48601   if (SDValue R = combineBitOpWithPACK(N, DAG))
48602     return R;
48603 
48604   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
48605     return FPLogic;
48606 
48607   if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
48608     return R;
48609 
48610   if (DCI.isBeforeLegalizeOps())
48611     return SDValue();
48612 
48613   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
48614     return R;
48615 
48616   if (SDValue R = combineAndNotIntoANDNP(N, DAG))
48617     return R;
48618 
48619   if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
48620     return ShiftRight;
48621 
48622   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
48623     return R;
48624 
48625   // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2))
48626   // iff c2 is all/no bits mask - i.e. a select-with-zero mask.
48627   // TODO: Handle PMULDQ/PMULUDQ/VPMADDWD/VPMADDUBSW?
48628   if (VT.isVector() && getTargetConstantFromNode(N1)) {
48629     unsigned Opc0 = N0.getOpcode();
48630     if ((Opc0 == ISD::MUL || Opc0 == ISD::MULHU || Opc0 == ISD::MULHS) &&
48631         getTargetConstantFromNode(N0.getOperand(1)) &&
48632         DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() &&
48633         N0->hasOneUse() && N0.getOperand(1)->hasOneUse()) {
48634       SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1);
48635       return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul);
48636     }
48637   }
48638 
48639   // Fold AND(SRL(X,Y),1) -> SETCC(BT(X,Y), COND_B) iff Y is not a constant
48640   // avoids slow variable shift (moving shift amount to ECX etc.)
48641   if (isOneConstant(N1) && N0->hasOneUse()) {
48642     SDValue Src = N0;
48643     while ((Src.getOpcode() == ISD::ZERO_EXTEND ||
48644             Src.getOpcode() == ISD::TRUNCATE) &&
48645            Src.getOperand(0)->hasOneUse())
48646       Src = Src.getOperand(0);
48647     bool ContainsNOT = false;
48648     X86::CondCode X86CC = X86::COND_B;
48649     // Peek through AND(NOT(SRL(X,Y)),1).
48650     if (isBitwiseNot(Src)) {
48651       Src = Src.getOperand(0);
48652       X86CC = X86::COND_AE;
48653       ContainsNOT = true;
48654     }
48655     if (Src.getOpcode() == ISD::SRL &&
48656         !isa<ConstantSDNode>(Src.getOperand(1))) {
48657       SDValue BitNo = Src.getOperand(1);
48658       Src = Src.getOperand(0);
48659       // Peek through AND(SRL(NOT(X),Y),1).
48660       if (isBitwiseNot(Src)) {
48661         Src = Src.getOperand(0);
48662         X86CC = X86CC == X86::COND_AE ? X86::COND_B : X86::COND_AE;
48663         ContainsNOT = true;
48664       }
48665       // If we have BMI2 then SHRX should be faster for i32/i64 cases.
48666       if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32))
48667         if (SDValue BT = getBT(Src, BitNo, dl, DAG))
48668           return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT);
48669     }
48670   }
48671 
48672   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
48673     // Attempt to recursively combine a bitmask AND with shuffles.
48674     SDValue Op(N, 0);
48675     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48676       return Res;
48677 
48678     // If either operand is a constant mask, then only the elements that aren't
48679     // zero are actually demanded by the other operand.
48680     auto GetDemandedMasks = [&](SDValue Op) {
48681       APInt UndefElts;
48682       SmallVector<APInt> EltBits;
48683       int NumElts = VT.getVectorNumElements();
48684       int EltSizeInBits = VT.getScalarSizeInBits();
48685       APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
48686       APInt DemandedElts = APInt::getAllOnes(NumElts);
48687       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
48688                                         EltBits)) {
48689         DemandedBits.clearAllBits();
48690         DemandedElts.clearAllBits();
48691         for (int I = 0; I != NumElts; ++I) {
48692           if (UndefElts[I]) {
48693             // We can't assume an undef src element gives an undef dst - the
48694             // other src might be zero.
48695             DemandedBits.setAllBits();
48696             DemandedElts.setBit(I);
48697           } else if (!EltBits[I].isZero()) {
48698             DemandedBits |= EltBits[I];
48699             DemandedElts.setBit(I);
48700           }
48701         }
48702       }
48703       return std::make_pair(DemandedBits, DemandedElts);
48704     };
48705     APInt Bits0, Elts0;
48706     APInt Bits1, Elts1;
48707     std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
48708     std::tie(Bits1, Elts1) = GetDemandedMasks(N0);
48709 
48710     if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
48711         TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
48712         TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
48713         TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
48714       if (N->getOpcode() != ISD::DELETED_NODE)
48715         DCI.AddToWorklist(N);
48716       return SDValue(N, 0);
48717     }
48718 
48719     SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG);
48720     SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG);
48721     if (NewN0 || NewN1)
48722       return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0,
48723                          NewN1 ? NewN1 : N1);
48724   }
48725 
48726   // Attempt to combine a scalar bitmask AND with an extracted shuffle.
48727   if ((VT.getScalarSizeInBits() % 8) == 0 &&
48728       N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
48729       isa<ConstantSDNode>(N0.getOperand(1)) && N0->hasOneUse()) {
48730     SDValue BitMask = N1;
48731     SDValue SrcVec = N0.getOperand(0);
48732     EVT SrcVecVT = SrcVec.getValueType();
48733 
48734     // Check that the constant bitmask masks whole bytes.
48735     APInt UndefElts;
48736     SmallVector<APInt, 64> EltBits;
48737     if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) &&
48738         getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
48739         llvm::all_of(EltBits, [](const APInt &M) {
48740           return M.isZero() || M.isAllOnes();
48741         })) {
48742       unsigned NumElts = SrcVecVT.getVectorNumElements();
48743       unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
48744       unsigned Idx = N0.getConstantOperandVal(1);
48745 
48746       // Create a root shuffle mask from the byte mask and the extracted index.
48747       SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
48748       for (unsigned i = 0; i != Scale; ++i) {
48749         if (UndefElts[i])
48750           continue;
48751         int VecIdx = Scale * Idx + i;
48752         ShuffleMask[VecIdx] = EltBits[i].isZero() ? SM_SentinelZero : VecIdx;
48753       }
48754 
48755       if (SDValue Shuffle = combineX86ShufflesRecursively(
48756               {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
48757               X86::MaxShuffleCombineDepth,
48758               /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true,
48759               /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
48760         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
48761                            N0.getOperand(1));
48762     }
48763   }
48764 
48765   if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
48766     return R;
48767 
48768   return SDValue();
48769 }
48770 
48771 // Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
48772 static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
48773                                      const X86Subtarget &Subtarget) {
48774   assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
48775 
48776   MVT VT = N->getSimpleValueType(0);
48777   unsigned EltSizeInBits = VT.getScalarSizeInBits();
48778   if (!VT.isVector() || (EltSizeInBits % 8) != 0)
48779     return SDValue();
48780 
48781   SDValue N0 = peekThroughBitcasts(N->getOperand(0));
48782   SDValue N1 = peekThroughBitcasts(N->getOperand(1));
48783   if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
48784     return SDValue();
48785 
48786   // On XOP we'll lower to PCMOV so accept one use. With AVX512, we can use
48787   // VPTERNLOG. Otherwise only do this if either mask has multiple uses already.
48788   if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) ||
48789         !N0.getOperand(1).hasOneUse() || !N1.getOperand(1).hasOneUse()))
48790     return SDValue();
48791 
48792   // Attempt to extract constant byte masks.
48793   APInt UndefElts0, UndefElts1;
48794   SmallVector<APInt, 32> EltBits0, EltBits1;
48795   if (!getTargetConstantBitsFromNode(N0.getOperand(1), 8, UndefElts0, EltBits0,
48796                                      false, false))
48797     return SDValue();
48798   if (!getTargetConstantBitsFromNode(N1.getOperand(1), 8, UndefElts1, EltBits1,
48799                                      false, false))
48800     return SDValue();
48801 
48802   for (unsigned i = 0, e = EltBits0.size(); i != e; ++i) {
48803     // TODO - add UNDEF elts support.
48804     if (UndefElts0[i] || UndefElts1[i])
48805       return SDValue();
48806     if (EltBits0[i] != ~EltBits1[i])
48807       return SDValue();
48808   }
48809 
48810   SDLoc DL(N);
48811 
48812   if (useVPTERNLOG(Subtarget, VT)) {
48813     // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C.
48814     // VPTERNLOG is only available as vXi32/64-bit types.
48815     MVT OpSVT = EltSizeInBits <= 32 ? MVT::i32 : MVT::i64;
48816     MVT OpVT =
48817         MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits());
48818     SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1));
48819     SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0));
48820     SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0));
48821     SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
48822     SDValue Res = getAVX512Node(X86ISD::VPTERNLOG, DL, OpVT, {A, B, C, Imm},
48823                                 DAG, Subtarget);
48824     return DAG.getBitcast(VT, Res);
48825   }
48826 
48827   SDValue X = N->getOperand(0);
48828   SDValue Y =
48829       DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
48830                   DAG.getBitcast(VT, N1.getOperand(0)));
48831   return DAG.getNode(ISD::OR, DL, VT, X, Y);
48832 }
48833 
48834 // Try to match OR(AND(~MASK,X),AND(MASK,Y)) logic pattern.
48835 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
48836   if (N->getOpcode() != ISD::OR)
48837     return false;
48838 
48839   SDValue N0 = N->getOperand(0);
48840   SDValue N1 = N->getOperand(1);
48841 
48842   // Canonicalize AND to LHS.
48843   if (N1.getOpcode() == ISD::AND)
48844     std::swap(N0, N1);
48845 
48846   // Attempt to match OR(AND(M,Y),ANDNP(M,X)).
48847   if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
48848     return false;
48849 
48850   Mask = N1.getOperand(0);
48851   X = N1.getOperand(1);
48852 
48853   // Check to see if the mask appeared in both the AND and ANDNP.
48854   if (N0.getOperand(0) == Mask)
48855     Y = N0.getOperand(1);
48856   else if (N0.getOperand(1) == Mask)
48857     Y = N0.getOperand(0);
48858   else
48859     return false;
48860 
48861   // TODO: Attempt to match against AND(XOR(-1,M),Y) as well, waiting for
48862   // ANDNP combine allows other combines to happen that prevent matching.
48863   return true;
48864 }
48865 
48866 // Try to fold:
48867 //   (or (and (m, y), (pandn m, x)))
48868 // into:
48869 //   (vselect m, x, y)
48870 // As a special case, try to fold:
48871 //   (or (and (m, (sub 0, x)), (pandn m, x)))
48872 // into:
48873 //   (sub (xor X, M), M)
48874 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
48875                                             const X86Subtarget &Subtarget) {
48876   assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
48877 
48878   EVT VT = N->getValueType(0);
48879   if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
48880         (VT.is256BitVector() && Subtarget.hasInt256())))
48881     return SDValue();
48882 
48883   SDValue X, Y, Mask;
48884   if (!matchLogicBlend(N, X, Y, Mask))
48885     return SDValue();
48886 
48887   // Validate that X, Y, and Mask are bitcasts, and see through them.
48888   Mask = peekThroughBitcasts(Mask);
48889   X = peekThroughBitcasts(X);
48890   Y = peekThroughBitcasts(Y);
48891 
48892   EVT MaskVT = Mask.getValueType();
48893   unsigned EltBits = MaskVT.getScalarSizeInBits();
48894 
48895   // TODO: Attempt to handle floating point cases as well?
48896   if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits)
48897     return SDValue();
48898 
48899   SDLoc DL(N);
48900 
48901   // Attempt to combine to conditional negate: (sub (xor X, M), M)
48902   if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
48903                                                            DAG, Subtarget))
48904     return Res;
48905 
48906   // PBLENDVB is only available on SSE 4.1.
48907   if (!Subtarget.hasSSE41())
48908     return SDValue();
48909 
48910   // If we have VPTERNLOG we should prefer that since PBLENDVB is multiple uops.
48911   if (Subtarget.hasVLX())
48912     return SDValue();
48913 
48914   MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;
48915 
48916   X = DAG.getBitcast(BlendVT, X);
48917   Y = DAG.getBitcast(BlendVT, Y);
48918   Mask = DAG.getBitcast(BlendVT, Mask);
48919   Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
48920   return DAG.getBitcast(VT, Mask);
48921 }
48922 
48923 // Helper function for combineOrCmpEqZeroToCtlzSrl
48924 // Transforms:
48925 //   seteq(cmp x, 0)
48926 //   into:
48927 //   srl(ctlz x), log2(bitsize(x))
48928 // Input pattern is checked by caller.
48929 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
48930   SDValue Cmp = Op.getOperand(1);
48931   EVT VT = Cmp.getOperand(0).getValueType();
48932   unsigned Log2b = Log2_32(VT.getSizeInBits());
48933   SDLoc dl(Op);
48934   SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
48935   // The result of the shift is true or false, and on X86, the 32-bit
48936   // encoding of shr and lzcnt is more desirable.
48937   SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
48938   SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
48939                             DAG.getConstant(Log2b, dl, MVT::i8));
48940   return Scc;
48941 }
48942 
48943 // Try to transform:
48944 //   zext(or(setcc(eq, (cmp x, 0)), setcc(eq, (cmp y, 0))))
48945 //   into:
48946 //   srl(or(ctlz(x), ctlz(y)), log2(bitsize(x))
48947 // Will also attempt to match more generic cases, eg:
48948 //   zext(or(or(setcc(eq, cmp 0), setcc(eq, cmp 0)), setcc(eq, cmp 0)))
48949 // Only applies if the target supports the FastLZCNT feature.
48950 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
48951                                            TargetLowering::DAGCombinerInfo &DCI,
48952                                            const X86Subtarget &Subtarget) {
48953   if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast())
48954     return SDValue();
48955 
48956   auto isORCandidate = [](SDValue N) {
48957     return (N->getOpcode() == ISD::OR && N->hasOneUse());
48958   };
48959 
48960   // Check the zero extend is extending to 32-bit or more. The code generated by
48961   // srl(ctlz) for 16-bit or less variants of the pattern would require extra
48962   // instructions to clear the upper bits.
48963   if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) ||
48964       !isORCandidate(N->getOperand(0)))
48965     return SDValue();
48966 
48967   // Check the node matches: setcc(eq, cmp 0)
48968   auto isSetCCCandidate = [](SDValue N) {
48969     return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() &&
48970            X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E &&
48971            N->getOperand(1).getOpcode() == X86ISD::CMP &&
48972            isNullConstant(N->getOperand(1).getOperand(1)) &&
48973            N->getOperand(1).getValueType().bitsGE(MVT::i32);
48974   };
48975 
48976   SDNode *OR = N->getOperand(0).getNode();
48977   SDValue LHS = OR->getOperand(0);
48978   SDValue RHS = OR->getOperand(1);
48979 
48980   // Save nodes matching or(or, setcc(eq, cmp 0)).
48981   SmallVector<SDNode *, 2> ORNodes;
48982   while (((isORCandidate(LHS) && isSetCCCandidate(RHS)) ||
48983           (isORCandidate(RHS) && isSetCCCandidate(LHS)))) {
48984     ORNodes.push_back(OR);
48985     OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode();
48986     LHS = OR->getOperand(0);
48987     RHS = OR->getOperand(1);
48988   }
48989 
48990   // The last OR node should match or(setcc(eq, cmp 0), setcc(eq, cmp 0)).
48991   if (!(isSetCCCandidate(LHS) && isSetCCCandidate(RHS)) ||
48992       !isORCandidate(SDValue(OR, 0)))
48993     return SDValue();
48994 
48995   // We have a or(setcc(eq, cmp 0), setcc(eq, cmp 0)) pattern, try to lower it
48996   // to
48997   // or(srl(ctlz),srl(ctlz)).
48998   // The dag combiner can then fold it into:
48999   // srl(or(ctlz, ctlz)).
49000   SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
49001   SDValue Ret, NewRHS;
49002   if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG)))
49003     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);
49004 
49005   if (!Ret)
49006     return SDValue();
49007 
49008   // Try to lower nodes matching the or(or, setcc(eq, cmp 0)) pattern.
49009   while (!ORNodes.empty()) {
49010     OR = ORNodes.pop_back_val();
49011     LHS = OR->getOperand(0);
49012     RHS = OR->getOperand(1);
49013     // Swap rhs with lhs to match or(setcc(eq, cmp, 0), or).
49014     if (RHS->getOpcode() == ISD::OR)
49015       std::swap(LHS, RHS);
49016     NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG);
49017     if (!NewRHS)
49018       return SDValue();
49019     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);
49020   }
49021 
49022   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
49023 }
49024 
49025 static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
49026                                    SDValue And1_L, SDValue And1_R,
49027                                    const SDLoc &DL, SelectionDAG &DAG) {
49028   if (!isBitwiseNot(And0_L, true) || !And0_L->hasOneUse())
49029     return SDValue();
49030   SDValue NotOp = And0_L->getOperand(0);
49031   if (NotOp == And1_R)
49032     std::swap(And1_R, And1_L);
49033   if (NotOp != And1_L)
49034     return SDValue();
49035 
49036   // (~(NotOp) & And0_R) | (NotOp & And1_R)
49037   // --> ((And0_R ^ And1_R) & NotOp) ^ And1_R
49038   EVT VT = And1_L->getValueType(0);
49039   SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
49040   SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
49041   SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
49042   SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
49043   return Xor1;
49044 }
49045 
49046 /// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
49047 /// equivalent `((x ^ y) & m) ^ y)` pattern.
49048 /// This is typically a better representation for  targets without a fused
49049 /// "and-not" operation. This function is intended to be called from a
49050 /// `TargetLowering::PerformDAGCombine` callback on `ISD::OR` nodes.
49051 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
49052   // Note that masked-merge variants using XOR or ADD expressions are
49053   // normalized to OR by InstCombine so we only check for OR.
49054   assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
49055   SDValue N0 = Node->getOperand(0);
49056   if (N0->getOpcode() != ISD::AND || !N0->hasOneUse())
49057     return SDValue();
49058   SDValue N1 = Node->getOperand(1);
49059   if (N1->getOpcode() != ISD::AND || !N1->hasOneUse())
49060     return SDValue();
49061 
49062   SDLoc DL(Node);
49063   SDValue N00 = N0->getOperand(0);
49064   SDValue N01 = N0->getOperand(1);
49065   SDValue N10 = N1->getOperand(0);
49066   SDValue N11 = N1->getOperand(1);
49067   if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
49068     return Result;
49069   if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
49070     return Result;
49071   if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
49072     return Result;
49073   if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
49074     return Result;
49075   return SDValue();
49076 }
49077 
49078 /// If this is an add or subtract where one operand is produced by a cmp+setcc,
49079 /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
49080 /// with CMP+{ADC, SBB}.
49081 /// Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.
49082 static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
49083                                          SDValue X, SDValue Y,
49084                                          SelectionDAG &DAG,
49085                                          bool ZeroSecondOpOnly = false) {
49086   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
49087     return SDValue();
49088 
49089   // Look through a one-use zext.
49090   if (Y.getOpcode() == ISD::ZERO_EXTEND && Y.hasOneUse())
49091     Y = Y.getOperand(0);
49092 
49093   X86::CondCode CC;
49094   SDValue EFLAGS;
49095   if (Y.getOpcode() == X86ISD::SETCC && Y.hasOneUse()) {
49096     CC = (X86::CondCode)Y.getConstantOperandVal(0);
49097     EFLAGS = Y.getOperand(1);
49098   } else if (Y.getOpcode() == ISD::AND && isOneConstant(Y.getOperand(1)) &&
49099              Y.hasOneUse()) {
49100     EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC);
49101   }
49102 
49103   if (!EFLAGS)
49104     return SDValue();
49105 
49106   // If X is -1 or 0, then we have an opportunity to avoid constants required in
49107   // the general case below.
49108   auto *ConstantX = dyn_cast<ConstantSDNode>(X);
49109   if (ConstantX && !ZeroSecondOpOnly) {
49110     if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnes()) ||
49111         (IsSub && CC == X86::COND_B && ConstantX->isZero())) {
49112       // This is a complicated way to get -1 or 0 from the carry flag:
49113       // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax
49114       //  0 - SETB  -->  0 -  (CF) --> CF ? -1 : 0 --> SBB %eax, %eax
49115       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49116                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49117                          EFLAGS);
49118     }
49119 
49120     if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnes()) ||
49121         (IsSub && CC == X86::COND_A && ConstantX->isZero())) {
49122       if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
49123           EFLAGS.getValueType().isInteger() &&
49124           !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49125         // Swap the operands of a SUB, and we have the same pattern as above.
49126         // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB
49127         //  0 - SETA  (SUB A, B) -->  0 - SETB  (SUB B, A) --> SUB + SBB
49128         SDValue NewSub = DAG.getNode(
49129             X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49130             EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49131         SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
49132         return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49133                            DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49134                            NewEFLAGS);
49135       }
49136     }
49137   }
49138 
49139   if (CC == X86::COND_B) {
49140     // X + SETB Z --> adc X, 0
49141     // X - SETB Z --> sbb X, 0
49142     return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
49143                        DAG.getVTList(VT, MVT::i32), X,
49144                        DAG.getConstant(0, DL, VT), EFLAGS);
49145   }
49146 
49147   if (ZeroSecondOpOnly)
49148     return SDValue();
49149 
49150   if (CC == X86::COND_A) {
49151     // Try to convert COND_A into COND_B in an attempt to facilitate
49152     // materializing "setb reg".
49153     //
49154     // Do not flip "e > c", where "c" is a constant, because Cmp instruction
49155     // cannot take an immediate as its first operand.
49156     //
49157     if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
49158         EFLAGS.getValueType().isInteger() &&
49159         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49160       SDValue NewSub =
49161           DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49162                       EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49163       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
49164       return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
49165                          DAG.getVTList(VT, MVT::i32), X,
49166                          DAG.getConstant(0, DL, VT), NewEFLAGS);
49167     }
49168   }
49169 
49170   if (CC == X86::COND_AE) {
49171     // X + SETAE --> sbb X, -1
49172     // X - SETAE --> adc X, -1
49173     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
49174                        DAG.getVTList(VT, MVT::i32), X,
49175                        DAG.getConstant(-1, DL, VT), EFLAGS);
49176   }
49177 
49178   if (CC == X86::COND_BE) {
49179     // X + SETBE --> sbb X, -1
49180     // X - SETBE --> adc X, -1
49181     // Try to convert COND_BE into COND_AE in an attempt to facilitate
49182     // materializing "setae reg".
49183     //
49184     // Do not flip "e <= c", where "c" is a constant, because Cmp instruction
49185     // cannot take an immediate as its first operand.
49186     //
49187     if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
49188         EFLAGS.getValueType().isInteger() &&
49189         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49190       SDValue NewSub =
49191           DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49192                       EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49193       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
49194       return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
49195                          DAG.getVTList(VT, MVT::i32), X,
49196                          DAG.getConstant(-1, DL, VT), NewEFLAGS);
49197     }
49198   }
49199 
49200   if (CC != X86::COND_E && CC != X86::COND_NE)
49201     return SDValue();
49202 
49203   if (EFLAGS.getOpcode() != X86ISD::CMP || !EFLAGS.hasOneUse() ||
49204       !X86::isZeroNode(EFLAGS.getOperand(1)) ||
49205       !EFLAGS.getOperand(0).getValueType().isInteger())
49206     return SDValue();
49207 
49208   SDValue Z = EFLAGS.getOperand(0);
49209   EVT ZVT = Z.getValueType();
49210 
49211   // If X is -1 or 0, then we have an opportunity to avoid constants required in
49212   // the general case below.
49213   if (ConstantX) {
49214     // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with
49215     // fake operands:
49216     //  0 - (Z != 0) --> sbb %eax, %eax, (neg Z)
49217     // -1 + (Z == 0) --> sbb %eax, %eax, (neg Z)
49218     if ((IsSub && CC == X86::COND_NE && ConstantX->isZero()) ||
49219         (!IsSub && CC == X86::COND_E && ConstantX->isAllOnes())) {
49220       SDValue Zero = DAG.getConstant(0, DL, ZVT);
49221       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49222       SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
49223       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49224                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49225                          SDValue(Neg.getNode(), 1));
49226     }
49227 
49228     // cmp with 1 sets the carry flag when Z == 0, so create 0 or -1 using 'sbb'
49229     // with fake operands:
49230     //  0 - (Z == 0) --> sbb %eax, %eax, (cmp Z, 1)
49231     // -1 + (Z != 0) --> sbb %eax, %eax, (cmp Z, 1)
49232     if ((IsSub && CC == X86::COND_E && ConstantX->isZero()) ||
49233         (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnes())) {
49234       SDValue One = DAG.getConstant(1, DL, ZVT);
49235       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49236       SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
49237       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49238                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49239                          Cmp1.getValue(1));
49240     }
49241   }
49242 
49243   // (cmp Z, 1) sets the carry flag if Z is 0.
49244   SDValue One = DAG.getConstant(1, DL, ZVT);
49245   SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49246   SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
49247 
49248   // Add the flags type for ADC/SBB nodes.
49249   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
49250 
49251   // X - (Z != 0) --> sub X, (zext(setne Z, 0)) --> adc X, -1, (cmp Z, 1)
49252   // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
49253   if (CC == X86::COND_NE)
49254     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
49255                        DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1));
49256 
49257   // X - (Z == 0) --> sub X, (zext(sete  Z, 0)) --> sbb X, 0, (cmp Z, 1)
49258   // X + (Z == 0) --> add X, (zext(sete  Z, 0)) --> adc X, 0, (cmp Z, 1)
49259   return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
49260                      DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
49261 }
49262 
49263 /// If this is an add or subtract where one operand is produced by a cmp+setcc,
49264 /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
49265 /// with CMP+{ADC, SBB}.
49266 static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
49267   bool IsSub = N->getOpcode() == ISD::SUB;
49268   SDValue X = N->getOperand(0);
49269   SDValue Y = N->getOperand(1);
49270   EVT VT = N->getValueType(0);
49271   SDLoc DL(N);
49272 
49273   if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG))
49274     return ADCOrSBB;
49275 
49276   // Commute and try again (negate the result for subtracts).
49277   if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) {
49278     if (IsSub)
49279       ADCOrSBB =
49280           DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), ADCOrSBB);
49281     return ADCOrSBB;
49282   }
49283 
49284   return SDValue();
49285 }
49286 
49287 static SDValue combineOrXorWithSETCC(SDNode *N, SDValue N0, SDValue N1,
49288                                      SelectionDAG &DAG) {
49289   assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::OR) &&
49290          "Unexpected opcode");
49291 
49292   // Delegate to combineAddOrSubToADCOrSBB if we have:
49293   //
49294   //   (xor/or (zero_extend (setcc)) imm)
49295   //
49296   // where imm is odd if and only if we have xor, in which case the XOR/OR are
49297   // equivalent to a SUB/ADD, respectively.
49298   if (N0.getOpcode() == ISD::ZERO_EXTEND &&
49299       N0.getOperand(0).getOpcode() == X86ISD::SETCC && N0.hasOneUse()) {
49300     if (auto *N1C = dyn_cast<ConstantSDNode>(N1)) {
49301       bool IsSub = N->getOpcode() == ISD::XOR;
49302       bool N1COdd = N1C->getZExtValue() & 1;
49303       if (IsSub ? N1COdd : !N1COdd) {
49304         SDLoc DL(N);
49305         EVT VT = N->getValueType(0);
49306         if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG))
49307           return R;
49308       }
49309     }
49310   }
49311 
49312   return SDValue();
49313 }
49314 
49315 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
49316                          TargetLowering::DAGCombinerInfo &DCI,
49317                          const X86Subtarget &Subtarget) {
49318   SDValue N0 = N->getOperand(0);
49319   SDValue N1 = N->getOperand(1);
49320   EVT VT = N->getValueType(0);
49321   SDLoc dl(N);
49322   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49323 
49324   // If this is SSE1 only convert to FOR to avoid scalarization.
49325   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
49326     return DAG.getBitcast(MVT::v4i32,
49327                           DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,
49328                                       DAG.getBitcast(MVT::v4f32, N0),
49329                                       DAG.getBitcast(MVT::v4f32, N1)));
49330   }
49331 
49332   // Match any-of bool scalar reductions into a bitcast/movmsk + cmp.
49333   // TODO: Support multiple SrcOps.
49334   if (VT == MVT::i1) {
49335     SmallVector<SDValue, 2> SrcOps;
49336     SmallVector<APInt, 2> SrcPartials;
49337     if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
49338         SrcOps.size() == 1) {
49339       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
49340       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
49341       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
49342       if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
49343         Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
49344       if (Mask) {
49345         assert(SrcPartials[0].getBitWidth() == NumElts &&
49346                "Unexpected partial reduction mask");
49347         SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
49348         SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
49349         Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
49350         return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE);
49351       }
49352     }
49353   }
49354 
49355   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
49356     return R;
49357 
49358   if (SDValue R = combineBitOpWithShift(N, DAG))
49359     return R;
49360 
49361   if (SDValue R = combineBitOpWithPACK(N, DAG))
49362     return R;
49363 
49364   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
49365     return FPLogic;
49366 
49367   if (DCI.isBeforeLegalizeOps())
49368     return SDValue();
49369 
49370   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
49371     return R;
49372 
49373   if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
49374     return R;
49375 
49376   if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
49377     return R;
49378 
49379   // (0 - SetCC) | C -> (zext (not SetCC)) * (C + 1) - 1 if we can get a LEA out of it.
49380   if ((VT == MVT::i32 || VT == MVT::i64) &&
49381       N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
49382       isNullConstant(N0.getOperand(0))) {
49383     SDValue Cond = N0.getOperand(1);
49384     if (Cond.getOpcode() == ISD::ZERO_EXTEND && Cond.hasOneUse())
49385       Cond = Cond.getOperand(0);
49386 
49387     if (Cond.getOpcode() == X86ISD::SETCC && Cond.hasOneUse()) {
49388       if (auto *CN = dyn_cast<ConstantSDNode>(N1)) {
49389         uint64_t Val = CN->getZExtValue();
49390         if (Val == 1 || Val == 2 || Val == 3 || Val == 4 || Val == 7 || Val == 8) {
49391           X86::CondCode CCode = (X86::CondCode)Cond.getConstantOperandVal(0);
49392           CCode = X86::GetOppositeBranchCondition(CCode);
49393           SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG);
49394 
49395           SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
49396           R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
49397           R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
49398           return R;
49399         }
49400       }
49401     }
49402   }
49403 
49404   // Combine OR(X,KSHIFTL(Y,Elts/2)) -> CONCAT_VECTORS(X,Y) == KUNPCK(X,Y).
49405   // Combine OR(KSHIFTL(X,Elts/2),Y) -> CONCAT_VECTORS(Y,X) == KUNPCK(Y,X).
49406   // iff the upper elements of the non-shifted arg are zero.
49407   // KUNPCK require 16+ bool vector elements.
49408   if (N0.getOpcode() == X86ISD::KSHIFTL || N1.getOpcode() == X86ISD::KSHIFTL) {
49409     unsigned NumElts = VT.getVectorNumElements();
49410     unsigned HalfElts = NumElts / 2;
49411     APInt UpperElts = APInt::getHighBitsSet(NumElts, HalfElts);
49412     if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL &&
49413         N1.getConstantOperandAPInt(1) == HalfElts &&
49414         DAG.MaskedVectorIsZero(N0, UpperElts)) {
49415       return DAG.getNode(
49416           ISD::CONCAT_VECTORS, dl, VT,
49417           extractSubVector(N0, 0, DAG, dl, HalfElts),
49418           extractSubVector(N1.getOperand(0), 0, DAG, dl, HalfElts));
49419     }
49420     if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL &&
49421         N0.getConstantOperandAPInt(1) == HalfElts &&
49422         DAG.MaskedVectorIsZero(N1, UpperElts)) {
49423       return DAG.getNode(
49424           ISD::CONCAT_VECTORS, dl, VT,
49425           extractSubVector(N1, 0, DAG, dl, HalfElts),
49426           extractSubVector(N0.getOperand(0), 0, DAG, dl, HalfElts));
49427     }
49428   }
49429 
49430   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
49431     // Attempt to recursively combine an OR of shuffles.
49432     SDValue Op(N, 0);
49433     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49434       return Res;
49435 
49436     // If either operand is a constant mask, then only the elements that aren't
49437     // allones are actually demanded by the other operand.
49438     auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
49439       APInt UndefElts;
49440       SmallVector<APInt> EltBits;
49441       int NumElts = VT.getVectorNumElements();
49442       int EltSizeInBits = VT.getScalarSizeInBits();
49443       if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits))
49444         return false;
49445 
49446       APInt DemandedElts = APInt::getZero(NumElts);
49447       for (int I = 0; I != NumElts; ++I)
49448         if (!EltBits[I].isAllOnes())
49449           DemandedElts.setBit(I);
49450 
49451       return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, DCI);
49452     };
49453     if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) {
49454       if (N->getOpcode() != ISD::DELETED_NODE)
49455         DCI.AddToWorklist(N);
49456       return SDValue(N, 0);
49457     }
49458   }
49459 
49460   // We should fold "masked merge" patterns when `andn` is not available.
49461   if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1)
49462     if (SDValue R = foldMaskedMerge(N, DAG))
49463       return R;
49464 
49465   if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
49466     return R;
49467 
49468   return SDValue();
49469 }
49470 
49471 /// Try to turn tests against the signbit in the form of:
49472 ///   XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
49473 /// into:
49474 ///   SETGT(X, -1)
49475 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
49476   // This is only worth doing if the output type is i8 or i1.
49477   EVT ResultType = N->getValueType(0);
49478   if (ResultType != MVT::i8 && ResultType != MVT::i1)
49479     return SDValue();
49480 
49481   SDValue N0 = N->getOperand(0);
49482   SDValue N1 = N->getOperand(1);
49483 
49484   // We should be performing an xor against a truncated shift.
49485   if (N0.getOpcode() != ISD::TRUNCATE || !N0.hasOneUse())
49486     return SDValue();
49487 
49488   // Make sure we are performing an xor against one.
49489   if (!isOneConstant(N1))
49490     return SDValue();
49491 
49492   // SetCC on x86 zero extends so only act on this if it's a logical shift.
49493   SDValue Shift = N0.getOperand(0);
49494   if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse())
49495     return SDValue();
49496 
49497   // Make sure we are truncating from one of i16, i32 or i64.
49498   EVT ShiftTy = Shift.getValueType();
49499   if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64)
49500     return SDValue();
49501 
49502   // Make sure the shift amount extracts the sign bit.
49503   if (!isa<ConstantSDNode>(Shift.getOperand(1)) ||
49504       Shift.getConstantOperandAPInt(1) != (ShiftTy.getSizeInBits() - 1))
49505     return SDValue();
49506 
49507   // Create a greater-than comparison against -1.
49508   // N.B. Using SETGE against 0 works but we want a canonical looking
49509   // comparison, using SETGT matches up with what TranslateX86CC.
49510   SDLoc DL(N);
49511   SDValue ShiftOp = Shift.getOperand(0);
49512   EVT ShiftOpTy = ShiftOp.getValueType();
49513   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49514   EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
49515                                                *DAG.getContext(), ResultType);
49516   SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
49517                               DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
49518   if (SetCCResultType != ResultType)
49519     Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
49520   return Cond;
49521 }
49522 
49523 /// Turn vector tests of the signbit in the form of:
49524 ///   xor (sra X, elt_size(X)-1), -1
49525 /// into:
49526 ///   pcmpgt X, -1
49527 ///
49528 /// This should be called before type legalization because the pattern may not
49529 /// persist after that.
49530 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
49531                                          const X86Subtarget &Subtarget) {
49532   EVT VT = N->getValueType(0);
49533   if (!VT.isSimple())
49534     return SDValue();
49535 
49536   switch (VT.getSimpleVT().SimpleTy) {
49537   default: return SDValue();
49538   case MVT::v16i8:
49539   case MVT::v8i16:
49540   case MVT::v4i32:
49541   case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
49542   case MVT::v32i8:
49543   case MVT::v16i16:
49544   case MVT::v8i32:
49545   case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
49546   }
49547 
49548   // There must be a shift right algebraic before the xor, and the xor must be a
49549   // 'not' operation.
49550   SDValue Shift = N->getOperand(0);
49551   SDValue Ones = N->getOperand(1);
49552   if (Shift.getOpcode() != ISD::SRA || !Shift.hasOneUse() ||
49553       !ISD::isBuildVectorAllOnes(Ones.getNode()))
49554     return SDValue();
49555 
49556   // The shift should be smearing the sign bit across each vector element.
49557   auto *ShiftAmt =
49558       isConstOrConstSplat(Shift.getOperand(1), /*AllowUndefs*/ true);
49559   if (!ShiftAmt ||
49560       ShiftAmt->getAPIntValue() != (Shift.getScalarValueSizeInBits() - 1))
49561     return SDValue();
49562 
49563   // Create a greater-than comparison against -1. We don't use the more obvious
49564   // greater-than-or-equal-to-zero because SSE/AVX don't have that instruction.
49565   return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT);
49566 }
49567 
49568 /// Detect patterns of truncation with unsigned saturation:
49569 ///
49570 /// 1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
49571 ///   Return the source value x to be truncated or SDValue() if the pattern was
49572 ///   not matched.
49573 ///
49574 /// 2. (truncate (smin (smax (x, C1), C2)) to dest_type),
49575 ///   where C1 >= 0 and C2 is unsigned max of destination type.
49576 ///
49577 ///    (truncate (smax (smin (x, C2), C1)) to dest_type)
49578 ///   where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
49579 ///
49580 ///   These two patterns are equivalent to:
49581 ///   (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type)
49582 ///   So return the smax(x, C1) value to be truncated or SDValue() if the
49583 ///   pattern was not matched.
49584 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
49585                                  const SDLoc &DL) {
49586   EVT InVT = In.getValueType();
49587 
49588   // Saturation with truncation. We truncate from InVT to VT.
49589   assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&
49590          "Unexpected types for truncate operation");
49591 
49592   // Match min/max and return limit value as a parameter.
49593   auto MatchMinMax = [](SDValue V, unsigned Opcode, APInt &Limit) -> SDValue {
49594     if (V.getOpcode() == Opcode &&
49595         ISD::isConstantSplatVector(V.getOperand(1).getNode(), Limit))
49596       return V.getOperand(0);
49597     return SDValue();
49598   };
49599 
49600   APInt C1, C2;
49601   if (SDValue UMin = MatchMinMax(In, ISD::UMIN, C2))
49602     // C2 should be equal to UINT32_MAX / UINT16_MAX / UINT8_MAX according
49603     // the element size of the destination type.
49604     if (C2.isMask(VT.getScalarSizeInBits()))
49605       return UMin;
49606 
49607   if (SDValue SMin = MatchMinMax(In, ISD::SMIN, C2))
49608     if (MatchMinMax(SMin, ISD::SMAX, C1))
49609       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()))
49610         return SMin;
49611 
49612   if (SDValue SMax = MatchMinMax(In, ISD::SMAX, C1))
49613     if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, C2))
49614       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) &&
49615           C2.uge(C1)) {
49616         return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
49617       }
49618 
49619   return SDValue();
49620 }
49621 
49622 /// Detect patterns of truncation with signed saturation:
49623 /// (truncate (smin ((smax (x, signed_min_of_dest_type)),
49624 ///                  signed_max_of_dest_type)) to dest_type)
49625 /// or:
49626 /// (truncate (smax ((smin (x, signed_max_of_dest_type)),
49627 ///                  signed_min_of_dest_type)) to dest_type).
49628 /// With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type].
49629 /// Return the source value to be truncated or SDValue() if the pattern was not
49630 /// matched.
49631 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
49632   unsigned NumDstBits = VT.getScalarSizeInBits();
49633   unsigned NumSrcBits = In.getScalarValueSizeInBits();
49634   assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
49635 
49636   auto MatchMinMax = [](SDValue V, unsigned Opcode,
49637                         const APInt &Limit) -> SDValue {
49638     APInt C;
49639     if (V.getOpcode() == Opcode &&
49640         ISD::isConstantSplatVector(V.getOperand(1).getNode(), C) && C == Limit)
49641       return V.getOperand(0);
49642     return SDValue();
49643   };
49644 
49645   APInt SignedMax, SignedMin;
49646   if (MatchPackUS) {
49647     SignedMax = APInt::getAllOnes(NumDstBits).zext(NumSrcBits);
49648     SignedMin = APInt(NumSrcBits, 0);
49649   } else {
49650     SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
49651     SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
49652   }
49653 
49654   if (SDValue SMin = MatchMinMax(In, ISD::SMIN, SignedMax))
49655     if (SDValue SMax = MatchMinMax(SMin, ISD::SMAX, SignedMin))
49656       return SMax;
49657 
49658   if (SDValue SMax = MatchMinMax(In, ISD::SMAX, SignedMin))
49659     if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, SignedMax))
49660       return SMin;
49661 
49662   return SDValue();
49663 }
49664 
49665 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
49666                                       SelectionDAG &DAG,
49667                                       const X86Subtarget &Subtarget) {
49668   if (!Subtarget.hasSSE2() || !VT.isVector())
49669     return SDValue();
49670 
49671   EVT SVT = VT.getVectorElementType();
49672   EVT InVT = In.getValueType();
49673   EVT InSVT = InVT.getVectorElementType();
49674 
49675   // If we're clamping a signed 32-bit vector to 0-255 and the 32-bit vector is
49676   // split across two registers. We can use a packusdw+perm to clamp to 0-65535
49677   // and concatenate at the same time. Then we can use a final vpmovuswb to
49678   // clip to 0-255.
49679   if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
49680       InVT == MVT::v16i32 && VT == MVT::v16i8) {
49681     if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
49682       // Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
49683       SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
49684                                            DL, DAG, Subtarget);
49685       assert(Mid && "Failed to pack!");
49686       return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
49687     }
49688   }
49689 
49690   // vXi32 truncate instructions are available with AVX512F.
49691   // vXi16 truncate instructions are only available with AVX512BW.
49692   // For 256-bit or smaller vectors, we require VLX.
49693   // FIXME: We could widen truncates to 512 to remove the VLX restriction.
49694   // If the result type is 256-bits or larger and we have disable 512-bit
49695   // registers, we should go ahead and use the pack instructions if possible.
49696   bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
49697                        (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
49698                       (InVT.getSizeInBits() > 128) &&
49699                       (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
49700                       !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
49701 
49702   if (!PreferAVX512 && VT.getVectorNumElements() > 1 &&
49703       isPowerOf2_32(VT.getVectorNumElements()) &&
49704       (SVT == MVT::i8 || SVT == MVT::i16) &&
49705       (InSVT == MVT::i16 || InSVT == MVT::i32)) {
49706     if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
49707       // vXi32 -> vXi8 must be performed as PACKUSWB(PACKSSDW,PACKSSDW).
49708       if (SVT == MVT::i8 && InSVT == MVT::i32) {
49709         EVT MidVT = VT.changeVectorElementType(MVT::i16);
49710         SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
49711                                              DAG, Subtarget);
49712         assert(Mid && "Failed to pack!");
49713         SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
49714                                            Subtarget);
49715         assert(V && "Failed to pack!");
49716         return V;
49717       } else if (SVT == MVT::i8 || Subtarget.hasSSE41())
49718         return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
49719                                       Subtarget);
49720     }
49721     if (SDValue SSatVal = detectSSatPattern(In, VT))
49722       return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
49723                                     Subtarget);
49724   }
49725 
49726   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49727   if (TLI.isTypeLegal(InVT) && InVT.isVector() && SVT != MVT::i1 &&
49728       Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI()) &&
49729       (SVT == MVT::i32 || SVT == MVT::i16 || SVT == MVT::i8)) {
49730     unsigned TruncOpc = 0;
49731     SDValue SatVal;
49732     if (SDValue SSatVal = detectSSatPattern(In, VT)) {
49733       SatVal = SSatVal;
49734       TruncOpc = X86ISD::VTRUNCS;
49735     } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) {
49736       SatVal = USatVal;
49737       TruncOpc = X86ISD::VTRUNCUS;
49738     }
49739     if (SatVal) {
49740       unsigned ResElts = VT.getVectorNumElements();
49741       // If the input type is less than 512 bits and we don't have VLX, we need
49742       // to widen to 512 bits.
49743       if (!Subtarget.hasVLX() && !InVT.is512BitVector()) {
49744         unsigned NumConcats = 512 / InVT.getSizeInBits();
49745         ResElts *= NumConcats;
49746         SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
49747         ConcatOps[0] = SatVal;
49748         InVT = EVT::getVectorVT(*DAG.getContext(), InSVT,
49749                                 NumConcats * InVT.getVectorNumElements());
49750         SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
49751       }
49752       // Widen the result if its narrower than 128 bits.
49753       if (ResElts * SVT.getSizeInBits() < 128)
49754         ResElts = 128 / SVT.getSizeInBits();
49755       EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts);
49756       SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
49757       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
49758                          DAG.getIntPtrConstant(0, DL));
49759     }
49760   }
49761 
49762   return SDValue();
49763 }
49764 
49765 /// This function detects the AVG pattern between vectors of unsigned i8/i16,
49766 /// which is c = (a + b + 1) / 2, and replace this operation with the efficient
49767 /// ISD::AVGCEILU (AVG) instruction.
49768 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
49769                                 const X86Subtarget &Subtarget,
49770                                 const SDLoc &DL) {
49771   if (!VT.isVector())
49772     return SDValue();
49773   EVT InVT = In.getValueType();
49774   unsigned NumElems = VT.getVectorNumElements();
49775 
49776   EVT ScalarVT = VT.getVectorElementType();
49777   if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && NumElems >= 2))
49778     return SDValue();
49779 
49780   // InScalarVT is the intermediate type in AVG pattern and it should be greater
49781   // than the original input type (i8/i16).
49782   EVT InScalarVT = InVT.getVectorElementType();
49783   if (InScalarVT.getFixedSizeInBits() <= ScalarVT.getFixedSizeInBits())
49784     return SDValue();
49785 
49786   if (!Subtarget.hasSSE2())
49787     return SDValue();
49788 
49789   // Detect the following pattern:
49790   //
49791   //   %1 = zext <N x i8> %a to <N x i32>
49792   //   %2 = zext <N x i8> %b to <N x i32>
49793   //   %3 = add nuw nsw <N x i32> %1, <i32 1 x N>
49794   //   %4 = add nuw nsw <N x i32> %3, %2
49795   //   %5 = lshr <N x i32> %N, <i32 1 x N>
49796   //   %6 = trunc <N x i32> %5 to <N x i8>
49797   //
49798   // In AVX512, the last instruction can also be a trunc store.
49799   if (In.getOpcode() != ISD::SRL)
49800     return SDValue();
49801 
49802   // A lambda checking the given SDValue is a constant vector and each element
49803   // is in the range [Min, Max].
49804   auto IsConstVectorInRange = [](SDValue V, unsigned Min, unsigned Max) {
49805     return ISD::matchUnaryPredicate(V, [Min, Max](ConstantSDNode *C) {
49806       return !(C->getAPIntValue().ult(Min) || C->getAPIntValue().ugt(Max));
49807     });
49808   };
49809 
49810   auto IsZExtLike = [DAG = &DAG, ScalarVT](SDValue V) {
49811     unsigned MaxActiveBits = DAG->computeKnownBits(V).countMaxActiveBits();
49812     return MaxActiveBits <= ScalarVT.getSizeInBits();
49813   };
49814 
49815   // Check if each element of the vector is right-shifted by one.
49816   SDValue LHS = In.getOperand(0);
49817   SDValue RHS = In.getOperand(1);
49818   if (!IsConstVectorInRange(RHS, 1, 1))
49819     return SDValue();
49820   if (LHS.getOpcode() != ISD::ADD)
49821     return SDValue();
49822 
49823   // Detect a pattern of a + b + 1 where the order doesn't matter.
49824   SDValue Operands[3];
49825   Operands[0] = LHS.getOperand(0);
49826   Operands[1] = LHS.getOperand(1);
49827 
49828   auto AVGBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
49829                        ArrayRef<SDValue> Ops) {
49830     return DAG.getNode(ISD::AVGCEILU, DL, Ops[0].getValueType(), Ops);
49831   };
49832 
49833   auto AVGSplitter = [&](std::array<SDValue, 2> Ops) {
49834     for (SDValue &Op : Ops)
49835       if (Op.getValueType() != VT)
49836         Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
49837     // Pad to a power-of-2 vector, split+apply and extract the original vector.
49838     unsigned NumElemsPow2 = PowerOf2Ceil(NumElems);
49839     EVT Pow2VT = EVT::getVectorVT(*DAG.getContext(), ScalarVT, NumElemsPow2);
49840     if (NumElemsPow2 != NumElems) {
49841       for (SDValue &Op : Ops) {
49842         SmallVector<SDValue, 32> EltsOfOp(NumElemsPow2, DAG.getUNDEF(ScalarVT));
49843         for (unsigned i = 0; i != NumElems; ++i) {
49844           SDValue Idx = DAG.getIntPtrConstant(i, DL);
49845           EltsOfOp[i] =
49846               DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Op, Idx);
49847         }
49848         Op = DAG.getBuildVector(Pow2VT, DL, EltsOfOp);
49849       }
49850     }
49851     SDValue Res = SplitOpsAndApply(DAG, Subtarget, DL, Pow2VT, Ops, AVGBuilder);
49852     if (NumElemsPow2 == NumElems)
49853       return Res;
49854     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
49855                        DAG.getIntPtrConstant(0, DL));
49856   };
49857 
49858   // Take care of the case when one of the operands is a constant vector whose
49859   // element is in the range [1, 256].
49860   if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) &&
49861       IsZExtLike(Operands[0])) {
49862     // The pattern is detected. Subtract one from the constant vector, then
49863     // demote it and emit X86ISD::AVG instruction.
49864     SDValue VecOnes = DAG.getConstant(1, DL, InVT);
49865     Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes);
49866     return AVGSplitter({Operands[0], Operands[1]});
49867   }
49868 
49869   // Matches 'add like' patterns: add(Op0,Op1) + zext(or(Op0,Op1)).
49870   // Match the or case only if its 'add-like' - can be replaced by an add.
49871   auto FindAddLike = [&](SDValue V, SDValue &Op0, SDValue &Op1) {
49872     if (ISD::ADD == V.getOpcode()) {
49873       Op0 = V.getOperand(0);
49874       Op1 = V.getOperand(1);
49875       return true;
49876     }
49877     if (ISD::ZERO_EXTEND != V.getOpcode())
49878       return false;
49879     V = V.getOperand(0);
49880     if (V.getValueType() != VT || ISD::OR != V.getOpcode() ||
49881         !DAG.haveNoCommonBitsSet(V.getOperand(0), V.getOperand(1)))
49882       return false;
49883     Op0 = V.getOperand(0);
49884     Op1 = V.getOperand(1);
49885     return true;
49886   };
49887 
49888   SDValue Op0, Op1;
49889   if (FindAddLike(Operands[0], Op0, Op1))
49890     std::swap(Operands[0], Operands[1]);
49891   else if (!FindAddLike(Operands[1], Op0, Op1))
49892     return SDValue();
49893   Operands[2] = Op0;
49894   Operands[1] = Op1;
49895 
49896   // Now we have three operands of two additions. Check that one of them is a
49897   // constant vector with ones, and the other two can be promoted from i8/i16.
49898   for (SDValue &Op : Operands) {
49899     if (!IsConstVectorInRange(Op, 1, 1))
49900       continue;
49901     std::swap(Op, Operands[2]);
49902 
49903     // Check if Operands[0] and Operands[1] are results of type promotion.
49904     for (int j = 0; j < 2; ++j)
49905       if (Operands[j].getValueType() != VT)
49906         if (!IsZExtLike(Operands[j]))
49907           return SDValue();
49908 
49909     // The pattern is detected, emit X86ISD::AVG instruction(s).
49910     return AVGSplitter({Operands[0], Operands[1]});
49911   }
49912 
49913   return SDValue();
49914 }
49915 
49916 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
49917                            TargetLowering::DAGCombinerInfo &DCI,
49918                            const X86Subtarget &Subtarget) {
49919   LoadSDNode *Ld = cast<LoadSDNode>(N);
49920   EVT RegVT = Ld->getValueType(0);
49921   EVT MemVT = Ld->getMemoryVT();
49922   SDLoc dl(Ld);
49923   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49924 
49925   // For chips with slow 32-byte unaligned loads, break the 32-byte operation
49926   // into two 16-byte operations. Also split non-temporal aligned loads on
49927   // pre-AVX2 targets as 32-byte loads will lower to regular temporal loads.
49928   ISD::LoadExtType Ext = Ld->getExtensionType();
49929   unsigned Fast;
49930   if (RegVT.is256BitVector() && !DCI.isBeforeLegalizeOps() &&
49931       Ext == ISD::NON_EXTLOAD &&
49932       ((Ld->isNonTemporal() && !Subtarget.hasInt256() &&
49933         Ld->getAlign() >= Align(16)) ||
49934        (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT,
49935                                *Ld->getMemOperand(), &Fast) &&
49936         !Fast))) {
49937     unsigned NumElems = RegVT.getVectorNumElements();
49938     if (NumElems < 2)
49939       return SDValue();
49940 
49941     unsigned HalfOffset = 16;
49942     SDValue Ptr1 = Ld->getBasePtr();
49943     SDValue Ptr2 =
49944         DAG.getMemBasePlusOffset(Ptr1, TypeSize::getFixed(HalfOffset), dl);
49945     EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
49946                                   NumElems / 2);
49947     SDValue Load1 =
49948         DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(),
49949                     Ld->getOriginalAlign(),
49950                     Ld->getMemOperand()->getFlags());
49951     SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
49952                                 Ld->getPointerInfo().getWithOffset(HalfOffset),
49953                                 Ld->getOriginalAlign(),
49954                                 Ld->getMemOperand()->getFlags());
49955     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
49956                              Load1.getValue(1), Load2.getValue(1));
49957 
49958     SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
49959     return DCI.CombineTo(N, NewVec, TF, true);
49960   }
49961 
49962   // Bool vector load - attempt to cast to an integer, as we have good
49963   // (vXiY *ext(vXi1 bitcast(iX))) handling.
49964   if (Ext == ISD::NON_EXTLOAD && !Subtarget.hasAVX512() && RegVT.isVector() &&
49965       RegVT.getScalarType() == MVT::i1 && DCI.isBeforeLegalize()) {
49966     unsigned NumElts = RegVT.getVectorNumElements();
49967     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
49968     if (TLI.isTypeLegal(IntVT)) {
49969       SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
49970                                     Ld->getPointerInfo(),
49971                                     Ld->getOriginalAlign(),
49972                                     Ld->getMemOperand()->getFlags());
49973       SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
49974       return DCI.CombineTo(N, BoolVec, IntLoad.getValue(1), true);
49975     }
49976   }
49977 
49978   // If we also load/broadcast this to a wider type, then just extract the
49979   // lowest subvector.
49980   if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() &&
49981       (RegVT.is128BitVector() || RegVT.is256BitVector())) {
49982     SDValue Ptr = Ld->getBasePtr();
49983     SDValue Chain = Ld->getChain();
49984     for (SDNode *User : Chain->uses()) {
49985       auto *UserLd = dyn_cast<MemSDNode>(User);
49986       if (User != N && UserLd &&
49987           (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
49988            User->getOpcode() == X86ISD::VBROADCAST_LOAD ||
49989            ISD::isNormalLoad(User)) &&
49990           UserLd->getChain() == Chain && !User->hasAnyUseOfValue(1) &&
49991           User->getValueSizeInBits(0).getFixedValue() >
49992               RegVT.getFixedSizeInBits()) {
49993         if (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
49994             UserLd->getBasePtr() == Ptr &&
49995             UserLd->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits()) {
49996           SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
49997                                              RegVT.getSizeInBits());
49998           Extract = DAG.getBitcast(RegVT, Extract);
49999           return DCI.CombineTo(N, Extract, SDValue(User, 1));
50000         }
50001         auto MatchingBits = [](const APInt &Undefs, const APInt &UserUndefs,
50002                                ArrayRef<APInt> Bits, ArrayRef<APInt> UserBits) {
50003           for (unsigned I = 0, E = Undefs.getBitWidth(); I != E; ++I) {
50004             if (Undefs[I])
50005               continue;
50006             if (UserUndefs[I] || Bits[I] != UserBits[I])
50007               return false;
50008           }
50009           return true;
50010         };
50011         // See if we are loading a constant that matches in the lower
50012         // bits of a longer constant (but from a different constant pool ptr).
50013         EVT UserVT = User->getValueType(0);
50014         SDValue UserPtr = UserLd->getBasePtr();
50015         const Constant *LdC = getTargetConstantFromBasePtr(Ptr);
50016         const Constant *UserC = getTargetConstantFromBasePtr(UserPtr);
50017         if (LdC && UserC && UserPtr != Ptr) {
50018           unsigned LdSize = LdC->getType()->getPrimitiveSizeInBits();
50019           unsigned UserSize = UserC->getType()->getPrimitiveSizeInBits();
50020           if (LdSize < UserSize || !ISD::isNormalLoad(User)) {
50021             APInt Undefs, UserUndefs;
50022             SmallVector<APInt> Bits, UserBits;
50023             unsigned NumBits = std::min(RegVT.getScalarSizeInBits(),
50024                                         UserVT.getScalarSizeInBits());
50025             if (getTargetConstantBitsFromNode(SDValue(N, 0), NumBits, Undefs,
50026                                               Bits) &&
50027                 getTargetConstantBitsFromNode(SDValue(User, 0), NumBits,
50028                                               UserUndefs, UserBits)) {
50029               if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) {
50030                 SDValue Extract = extractSubVector(
50031                     SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
50032                 Extract = DAG.getBitcast(RegVT, Extract);
50033                 return DCI.CombineTo(N, Extract, SDValue(User, 1));
50034               }
50035             }
50036           }
50037         }
50038       }
50039     }
50040   }
50041 
50042   // Cast ptr32 and ptr64 pointers to the default address space before a load.
50043   unsigned AddrSpace = Ld->getAddressSpace();
50044   if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
50045       AddrSpace == X86AS::PTR32_UPTR) {
50046     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
50047     if (PtrVT != Ld->getBasePtr().getSimpleValueType()) {
50048       SDValue Cast =
50049           DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
50050       return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast,
50051                             Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(),
50052                             Ld->getMemOperand()->getFlags());
50053     }
50054   }
50055 
50056   return SDValue();
50057 }
50058 
50059 /// If V is a build vector of boolean constants and exactly one of those
50060 /// constants is true, return the operand index of that true element.
50061 /// Otherwise, return -1.
50062 static int getOneTrueElt(SDValue V) {
50063   // This needs to be a build vector of booleans.
50064   // TODO: Checking for the i1 type matches the IR definition for the mask,
50065   // but the mask check could be loosened to i8 or other types. That might
50066   // also require checking more than 'allOnesValue'; eg, the x86 HW
50067   // instructions only require that the MSB is set for each mask element.
50068   // The ISD::MSTORE comments/definition do not specify how the mask operand
50069   // is formatted.
50070   auto *BV = dyn_cast<BuildVectorSDNode>(V);
50071   if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1)
50072     return -1;
50073 
50074   int TrueIndex = -1;
50075   unsigned NumElts = BV->getValueType(0).getVectorNumElements();
50076   for (unsigned i = 0; i < NumElts; ++i) {
50077     const SDValue &Op = BV->getOperand(i);
50078     if (Op.isUndef())
50079       continue;
50080     auto *ConstNode = dyn_cast<ConstantSDNode>(Op);
50081     if (!ConstNode)
50082       return -1;
50083     if (ConstNode->getAPIntValue().countr_one() >= 1) {
50084       // If we already found a one, this is too many.
50085       if (TrueIndex >= 0)
50086         return -1;
50087       TrueIndex = i;
50088     }
50089   }
50090   return TrueIndex;
50091 }
50092 
50093 /// Given a masked memory load/store operation, return true if it has one mask
50094 /// bit set. If it has one mask bit set, then also return the memory address of
50095 /// the scalar element to load/store, the vector index to insert/extract that
50096 /// scalar element, and the alignment for the scalar memory access.
50097 static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp,
50098                                          SelectionDAG &DAG, SDValue &Addr,
50099                                          SDValue &Index, Align &Alignment,
50100                                          unsigned &Offset) {
50101   int TrueMaskElt = getOneTrueElt(MaskedOp->getMask());
50102   if (TrueMaskElt < 0)
50103     return false;
50104 
50105   // Get the address of the one scalar element that is specified by the mask
50106   // using the appropriate offset from the base pointer.
50107   EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType();
50108   Offset = 0;
50109   Addr = MaskedOp->getBasePtr();
50110   if (TrueMaskElt != 0) {
50111     Offset = TrueMaskElt * EltVT.getStoreSize();
50112     Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::getFixed(Offset),
50113                                     SDLoc(MaskedOp));
50114   }
50115 
50116   Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp));
50117   Alignment = commonAlignment(MaskedOp->getOriginalAlign(),
50118                               EltVT.getStoreSize());
50119   return true;
50120 }
50121 
50122 /// If exactly one element of the mask is set for a non-extending masked load,
50123 /// it is a scalar load and vector insert.
50124 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
50125 /// mask have already been optimized in IR, so we don't bother with those here.
50126 static SDValue
50127 reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
50128                              TargetLowering::DAGCombinerInfo &DCI,
50129                              const X86Subtarget &Subtarget) {
50130   assert(ML->isUnindexed() && "Unexpected indexed masked load!");
50131   // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
50132   // However, some target hooks may need to be added to know when the transform
50133   // is profitable. Endianness would also have to be considered.
50134 
50135   SDValue Addr, VecIndex;
50136   Align Alignment;
50137   unsigned Offset;
50138   if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset))
50139     return SDValue();
50140 
50141   // Load the one scalar element that is specified by the mask using the
50142   // appropriate offset from the base pointer.
50143   SDLoc DL(ML);
50144   EVT VT = ML->getValueType(0);
50145   EVT EltVT = VT.getVectorElementType();
50146 
50147   EVT CastVT = VT;
50148   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
50149     EltVT = MVT::f64;
50150     CastVT = VT.changeVectorElementType(EltVT);
50151   }
50152 
50153   SDValue Load =
50154       DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
50155                   ML->getPointerInfo().getWithOffset(Offset),
50156                   Alignment, ML->getMemOperand()->getFlags());
50157 
50158   SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
50159 
50160   // Insert the loaded element into the appropriate place in the vector.
50161   SDValue Insert =
50162       DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex);
50163   Insert = DAG.getBitcast(VT, Insert);
50164   return DCI.CombineTo(ML, Insert, Load.getValue(1), true);
50165 }
50166 
50167 static SDValue
50168 combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
50169                               TargetLowering::DAGCombinerInfo &DCI) {
50170   assert(ML->isUnindexed() && "Unexpected indexed masked load!");
50171   if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode()))
50172     return SDValue();
50173 
50174   SDLoc DL(ML);
50175   EVT VT = ML->getValueType(0);
50176 
50177   // If we are loading the first and last elements of a vector, it is safe and
50178   // always faster to load the whole vector. Replace the masked load with a
50179   // vector load and select.
50180   unsigned NumElts = VT.getVectorNumElements();
50181   BuildVectorSDNode *MaskBV = cast<BuildVectorSDNode>(ML->getMask());
50182   bool LoadFirstElt = !isNullConstant(MaskBV->getOperand(0));
50183   bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1));
50184   if (LoadFirstElt && LoadLastElt) {
50185     SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
50186                                 ML->getMemOperand());
50187     SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
50188                                   ML->getPassThru());
50189     return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
50190   }
50191 
50192   // Convert a masked load with a constant mask into a masked load and a select.
50193   // This allows the select operation to use a faster kind of select instruction
50194   // (for example, vblendvps -> vblendps).
50195 
50196   // Don't try this if the pass-through operand is already undefined. That would
50197   // cause an infinite loop because that's what we're about to create.
50198   if (ML->getPassThru().isUndef())
50199     return SDValue();
50200 
50201   if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode()))
50202     return SDValue();
50203 
50204   // The new masked load has an undef pass-through operand. The select uses the
50205   // original pass-through operand.
50206   SDValue NewML = DAG.getMaskedLoad(
50207       VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
50208       DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
50209       ML->getAddressingMode(), ML->getExtensionType());
50210   SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
50211                                 ML->getPassThru());
50212 
50213   return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
50214 }
50215 
50216 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
50217                                  TargetLowering::DAGCombinerInfo &DCI,
50218                                  const X86Subtarget &Subtarget) {
50219   auto *Mld = cast<MaskedLoadSDNode>(N);
50220 
50221   // TODO: Expanding load with constant mask may be optimized as well.
50222   if (Mld->isExpandingLoad())
50223     return SDValue();
50224 
50225   if (Mld->getExtensionType() == ISD::NON_EXTLOAD) {
50226     if (SDValue ScalarLoad =
50227             reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget))
50228       return ScalarLoad;
50229 
50230     // TODO: Do some AVX512 subsets benefit from this transform?
50231     if (!Subtarget.hasAVX512())
50232       if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
50233         return Blend;
50234   }
50235 
50236   // If the mask value has been legalized to a non-boolean vector, try to
50237   // simplify ops leading up to it. We only demand the MSB of each lane.
50238   SDValue Mask = Mld->getMask();
50239   if (Mask.getScalarValueSizeInBits() != 1) {
50240     EVT VT = Mld->getValueType(0);
50241     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50242     APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
50243     if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
50244       if (N->getOpcode() != ISD::DELETED_NODE)
50245         DCI.AddToWorklist(N);
50246       return SDValue(N, 0);
50247     }
50248     if (SDValue NewMask =
50249             TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
50250       return DAG.getMaskedLoad(
50251           VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(),
50252           NewMask, Mld->getPassThru(), Mld->getMemoryVT(), Mld->getMemOperand(),
50253           Mld->getAddressingMode(), Mld->getExtensionType());
50254   }
50255 
50256   return SDValue();
50257 }
50258 
50259 /// If exactly one element of the mask is set for a non-truncating masked store,
50260 /// it is a vector extract and scalar store.
50261 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
50262 /// mask have already been optimized in IR, so we don't bother with those here.
50263 static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
50264                                               SelectionDAG &DAG,
50265                                               const X86Subtarget &Subtarget) {
50266   // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
50267   // However, some target hooks may need to be added to know when the transform
50268   // is profitable. Endianness would also have to be considered.
50269 
50270   SDValue Addr, VecIndex;
50271   Align Alignment;
50272   unsigned Offset;
50273   if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset))
50274     return SDValue();
50275 
50276   // Extract the one scalar element that is actually being stored.
50277   SDLoc DL(MS);
50278   SDValue Value = MS->getValue();
50279   EVT VT = Value.getValueType();
50280   EVT EltVT = VT.getVectorElementType();
50281   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
50282     EltVT = MVT::f64;
50283     EVT CastVT = VT.changeVectorElementType(EltVT);
50284     Value = DAG.getBitcast(CastVT, Value);
50285   }
50286   SDValue Extract =
50287       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex);
50288 
50289   // Store that element at the appropriate offset from the base pointer.
50290   return DAG.getStore(MS->getChain(), DL, Extract, Addr,
50291                       MS->getPointerInfo().getWithOffset(Offset),
50292                       Alignment, MS->getMemOperand()->getFlags());
50293 }
50294 
50295 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
50296                                   TargetLowering::DAGCombinerInfo &DCI,
50297                                   const X86Subtarget &Subtarget) {
50298   MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
50299   if (Mst->isCompressingStore())
50300     return SDValue();
50301 
50302   EVT VT = Mst->getValue().getValueType();
50303   SDLoc dl(Mst);
50304   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50305 
50306   if (Mst->isTruncatingStore())
50307     return SDValue();
50308 
50309   if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget))
50310     return ScalarStore;
50311 
50312   // If the mask value has been legalized to a non-boolean vector, try to
50313   // simplify ops leading up to it. We only demand the MSB of each lane.
50314   SDValue Mask = Mst->getMask();
50315   if (Mask.getScalarValueSizeInBits() != 1) {
50316     APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
50317     if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
50318       if (N->getOpcode() != ISD::DELETED_NODE)
50319         DCI.AddToWorklist(N);
50320       return SDValue(N, 0);
50321     }
50322     if (SDValue NewMask =
50323             TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
50324       return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(),
50325                                 Mst->getBasePtr(), Mst->getOffset(), NewMask,
50326                                 Mst->getMemoryVT(), Mst->getMemOperand(),
50327                                 Mst->getAddressingMode());
50328   }
50329 
50330   SDValue Value = Mst->getValue();
50331   if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
50332       TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
50333                             Mst->getMemoryVT())) {
50334     return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
50335                               Mst->getBasePtr(), Mst->getOffset(), Mask,
50336                               Mst->getMemoryVT(), Mst->getMemOperand(),
50337                               Mst->getAddressingMode(), true);
50338   }
50339 
50340   return SDValue();
50341 }
50342 
50343 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
50344                             TargetLowering::DAGCombinerInfo &DCI,
50345                             const X86Subtarget &Subtarget) {
50346   StoreSDNode *St = cast<StoreSDNode>(N);
50347   EVT StVT = St->getMemoryVT();
50348   SDLoc dl(St);
50349   SDValue StoredVal = St->getValue();
50350   EVT VT = StoredVal.getValueType();
50351   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50352 
50353   // Convert a store of vXi1 into a store of iX and a bitcast.
50354   if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() &&
50355       VT.getVectorElementType() == MVT::i1) {
50356 
50357     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
50358     StoredVal = DAG.getBitcast(NewVT, StoredVal);
50359 
50360     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50361                         St->getPointerInfo(), St->getOriginalAlign(),
50362                         St->getMemOperand()->getFlags());
50363   }
50364 
50365   // If this is a store of a scalar_to_vector to v1i1, just use a scalar store.
50366   // This will avoid a copy to k-register.
50367   if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&
50368       StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR &&
50369       StoredVal.getOperand(0).getValueType() == MVT::i8) {
50370     SDValue Val = StoredVal.getOperand(0);
50371     // We must store zeros to the unused bits.
50372     Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);
50373     return DAG.getStore(St->getChain(), dl, Val,
50374                         St->getBasePtr(), St->getPointerInfo(),
50375                         St->getOriginalAlign(),
50376                         St->getMemOperand()->getFlags());
50377   }
50378 
50379   // Widen v2i1/v4i1 stores to v8i1.
50380   if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
50381       Subtarget.hasAVX512()) {
50382     unsigned NumConcats = 8 / VT.getVectorNumElements();
50383     // We must store zeros to the unused bits.
50384     SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
50385     Ops[0] = StoredVal;
50386     StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
50387     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50388                         St->getPointerInfo(), St->getOriginalAlign(),
50389                         St->getMemOperand()->getFlags());
50390   }
50391 
50392   // Turn vXi1 stores of constants into a scalar store.
50393   if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
50394        VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
50395       ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
50396     // If its a v64i1 store without 64-bit support, we need two stores.
50397     if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {
50398       SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
50399                                       StoredVal->ops().slice(0, 32));
50400       Lo = combinevXi1ConstantToInteger(Lo, DAG);
50401       SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
50402                                       StoredVal->ops().slice(32, 32));
50403       Hi = combinevXi1ConstantToInteger(Hi, DAG);
50404 
50405       SDValue Ptr0 = St->getBasePtr();
50406       SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl);
50407 
50408       SDValue Ch0 =
50409           DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(),
50410                        St->getOriginalAlign(),
50411                        St->getMemOperand()->getFlags());
50412       SDValue Ch1 =
50413           DAG.getStore(St->getChain(), dl, Hi, Ptr1,
50414                        St->getPointerInfo().getWithOffset(4),
50415                        St->getOriginalAlign(),
50416                        St->getMemOperand()->getFlags());
50417       return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
50418     }
50419 
50420     StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG);
50421     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50422                         St->getPointerInfo(), St->getOriginalAlign(),
50423                         St->getMemOperand()->getFlags());
50424   }
50425 
50426   // If we are saving a 32-byte vector and 32-byte stores are slow, such as on
50427   // Sandy Bridge, perform two 16-byte stores.
50428   unsigned Fast;
50429   if (VT.is256BitVector() && StVT == VT &&
50430       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
50431                              *St->getMemOperand(), &Fast) &&
50432       !Fast) {
50433     unsigned NumElems = VT.getVectorNumElements();
50434     if (NumElems < 2)
50435       return SDValue();
50436 
50437     return splitVectorStore(St, DAG);
50438   }
50439 
50440   // Split under-aligned vector non-temporal stores.
50441   if (St->isNonTemporal() && StVT == VT &&
50442       St->getAlign().value() < VT.getStoreSize()) {
50443     // ZMM/YMM nt-stores - either it can be stored as a series of shorter
50444     // vectors or the legalizer can scalarize it to use MOVNTI.
50445     if (VT.is256BitVector() || VT.is512BitVector()) {
50446       unsigned NumElems = VT.getVectorNumElements();
50447       if (NumElems < 2)
50448         return SDValue();
50449       return splitVectorStore(St, DAG);
50450     }
50451 
50452     // XMM nt-stores - scalarize this to f64 nt-stores on SSE4A, else i32/i64
50453     // to use MOVNTI.
50454     if (VT.is128BitVector() && Subtarget.hasSSE2()) {
50455       MVT NTVT = Subtarget.hasSSE4A()
50456                      ? MVT::v2f64
50457                      : (TLI.isTypeLegal(MVT::i64) ? MVT::v2i64 : MVT::v4i32);
50458       return scalarizeVectorStore(St, NTVT, DAG);
50459     }
50460   }
50461 
50462   // Try to optimize v16i16->v16i8 truncating stores when BWI is not
50463   // supported, but avx512f is by extending to v16i32 and truncating.
50464   if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() &&
50465       St->getValue().getOpcode() == ISD::TRUNCATE &&
50466       St->getValue().getOperand(0).getValueType() == MVT::v16i16 &&
50467       TLI.isTruncStoreLegal(MVT::v16i32, MVT::v16i8) &&
50468       St->getValue().hasOneUse() && !DCI.isBeforeLegalizeOps()) {
50469     SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,
50470                               St->getValue().getOperand(0));
50471     return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(),
50472                              MVT::v16i8, St->getMemOperand());
50473   }
50474 
50475   // Try to fold a VTRUNCUS or VTRUNCS into a truncating store.
50476   if (!St->isTruncatingStore() &&
50477       (StoredVal.getOpcode() == X86ISD::VTRUNCUS ||
50478        StoredVal.getOpcode() == X86ISD::VTRUNCS) &&
50479       StoredVal.hasOneUse() &&
50480       TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) {
50481     bool IsSigned = StoredVal.getOpcode() == X86ISD::VTRUNCS;
50482     return EmitTruncSStore(IsSigned, St->getChain(),
50483                            dl, StoredVal.getOperand(0), St->getBasePtr(),
50484                            VT, St->getMemOperand(), DAG);
50485   }
50486 
50487   // Try to fold a extract_element(VTRUNC) pattern into a truncating store.
50488   if (!St->isTruncatingStore()) {
50489     auto IsExtractedElement = [](SDValue V) {
50490       if (V.getOpcode() == ISD::TRUNCATE && V.hasOneUse())
50491         V = V.getOperand(0);
50492       unsigned Opc = V.getOpcode();
50493       if ((Opc == ISD::EXTRACT_VECTOR_ELT || Opc == X86ISD::PEXTRW) &&
50494           isNullConstant(V.getOperand(1)) && V.hasOneUse() &&
50495           V.getOperand(0).hasOneUse())
50496         return V.getOperand(0);
50497       return SDValue();
50498     };
50499     if (SDValue Extract = IsExtractedElement(StoredVal)) {
50500       SDValue Trunc = peekThroughOneUseBitcasts(Extract);
50501       if (Trunc.getOpcode() == X86ISD::VTRUNC) {
50502         SDValue Src = Trunc.getOperand(0);
50503         MVT DstVT = Trunc.getSimpleValueType();
50504         MVT SrcVT = Src.getSimpleValueType();
50505         unsigned NumSrcElts = SrcVT.getVectorNumElements();
50506         unsigned NumTruncBits = DstVT.getScalarSizeInBits() * NumSrcElts;
50507         MVT TruncVT = MVT::getVectorVT(DstVT.getScalarType(), NumSrcElts);
50508         if (NumTruncBits == VT.getSizeInBits() &&
50509             TLI.isTruncStoreLegal(SrcVT, TruncVT)) {
50510           return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(),
50511                                    TruncVT, St->getMemOperand());
50512         }
50513       }
50514     }
50515   }
50516 
50517   // Optimize trunc store (of multiple scalars) to shuffle and store.
50518   // First, pack all of the elements in one place. Next, store to memory
50519   // in fewer chunks.
50520   if (St->isTruncatingStore() && VT.isVector()) {
50521     // Check if we can detect an AVG pattern from the truncation. If yes,
50522     // replace the trunc store by a normal store with the result of X86ISD::AVG
50523     // instruction.
50524     if (DCI.isBeforeLegalize() || TLI.isTypeLegal(St->getMemoryVT()))
50525       if (SDValue Avg = detectAVGPattern(St->getValue(), St->getMemoryVT(), DAG,
50526                                          Subtarget, dl))
50527         return DAG.getStore(St->getChain(), dl, Avg, St->getBasePtr(),
50528                             St->getPointerInfo(), St->getOriginalAlign(),
50529                             St->getMemOperand()->getFlags());
50530 
50531     if (TLI.isTruncStoreLegal(VT, StVT)) {
50532       if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
50533         return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
50534                                dl, Val, St->getBasePtr(),
50535                                St->getMemoryVT(), St->getMemOperand(), DAG);
50536       if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
50537                                           DAG, dl))
50538         return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
50539                                dl, Val, St->getBasePtr(),
50540                                St->getMemoryVT(), St->getMemOperand(), DAG);
50541     }
50542 
50543     return SDValue();
50544   }
50545 
50546   // Cast ptr32 and ptr64 pointers to the default address space before a store.
50547   unsigned AddrSpace = St->getAddressSpace();
50548   if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
50549       AddrSpace == X86AS::PTR32_UPTR) {
50550     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
50551     if (PtrVT != St->getBasePtr().getSimpleValueType()) {
50552       SDValue Cast =
50553           DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
50554       return DAG.getTruncStore(
50555           St->getChain(), dl, StoredVal, Cast, St->getPointerInfo(), StVT,
50556           St->getOriginalAlign(), St->getMemOperand()->getFlags(),
50557           St->getAAInfo());
50558     }
50559   }
50560 
50561   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
50562   // the FP state in cases where an emms may be missing.
50563   // A preferable solution to the general problem is to figure out the right
50564   // places to insert EMMS.  This qualifies as a quick hack.
50565 
50566   // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
50567   if (VT.getSizeInBits() != 64)
50568     return SDValue();
50569 
50570   const Function &F = DAG.getMachineFunction().getFunction();
50571   bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
50572   bool F64IsLegal =
50573       !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2();
50574 
50575   if (!F64IsLegal || Subtarget.is64Bit())
50576     return SDValue();
50577 
50578   if (VT == MVT::i64 && isa<LoadSDNode>(St->getValue()) &&
50579       cast<LoadSDNode>(St->getValue())->isSimple() &&
50580       St->getChain().hasOneUse() && St->isSimple()) {
50581     auto *Ld = cast<LoadSDNode>(St->getValue());
50582 
50583     if (!ISD::isNormalLoad(Ld))
50584       return SDValue();
50585 
50586     // Avoid the transformation if there are multiple uses of the loaded value.
50587     if (!Ld->hasNUsesOfValue(1, 0))
50588       return SDValue();
50589 
50590     SDLoc LdDL(Ld);
50591     SDLoc StDL(N);
50592     // Lower to a single movq load/store pair.
50593     SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
50594                                 Ld->getBasePtr(), Ld->getMemOperand());
50595 
50596     // Make sure new load is placed in same chain order.
50597     DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
50598     return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(),
50599                         St->getMemOperand());
50600   }
50601 
50602   // This is similar to the above case, but here we handle a scalar 64-bit
50603   // integer store that is extracted from a vector on a 32-bit target.
50604   // If we have SSE2, then we can treat it like a floating-point double
50605   // to get past legalization. The execution dependencies fixup pass will
50606   // choose the optimal machine instruction for the store if this really is
50607   // an integer or v2f32 rather than an f64.
50608   if (VT == MVT::i64 &&
50609       St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
50610     SDValue OldExtract = St->getOperand(1);
50611     SDValue ExtOp0 = OldExtract.getOperand(0);
50612     unsigned VecSize = ExtOp0.getValueSizeInBits();
50613     EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
50614     SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
50615     SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
50616                                      BitCast, OldExtract.getOperand(1));
50617     return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
50618                         St->getPointerInfo(), St->getOriginalAlign(),
50619                         St->getMemOperand()->getFlags());
50620   }
50621 
50622   return SDValue();
50623 }
50624 
50625 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
50626                                      TargetLowering::DAGCombinerInfo &DCI,
50627                                      const X86Subtarget &Subtarget) {
50628   auto *St = cast<MemIntrinsicSDNode>(N);
50629 
50630   SDValue StoredVal = N->getOperand(1);
50631   MVT VT = StoredVal.getSimpleValueType();
50632   EVT MemVT = St->getMemoryVT();
50633 
50634   // Figure out which elements we demand.
50635   unsigned StElts = MemVT.getSizeInBits() / VT.getScalarSizeInBits();
50636   APInt DemandedElts = APInt::getLowBitsSet(VT.getVectorNumElements(), StElts);
50637 
50638   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50639   if (TLI.SimplifyDemandedVectorElts(StoredVal, DemandedElts, DCI)) {
50640     if (N->getOpcode() != ISD::DELETED_NODE)
50641       DCI.AddToWorklist(N);
50642     return SDValue(N, 0);
50643   }
50644 
50645   return SDValue();
50646 }
50647 
50648 /// Return 'true' if this vector operation is "horizontal"
50649 /// and return the operands for the horizontal operation in LHS and RHS.  A
50650 /// horizontal operation performs the binary operation on successive elements
50651 /// of its first operand, then on successive elements of its second operand,
50652 /// returning the resulting values in a vector.  For example, if
50653 ///   A = < float a0, float a1, float a2, float a3 >
50654 /// and
50655 ///   B = < float b0, float b1, float b2, float b3 >
50656 /// then the result of doing a horizontal operation on A and B is
50657 ///   A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
50658 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
50659 /// A horizontal-op B, for some already available A and B, and if so then LHS is
50660 /// set to A, RHS to B, and the routine returns 'true'.
50661 static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS,
50662                               SelectionDAG &DAG, const X86Subtarget &Subtarget,
50663                               bool IsCommutative,
50664                               SmallVectorImpl<int> &PostShuffleMask) {
50665   // If either operand is undef, bail out. The binop should be simplified.
50666   if (LHS.isUndef() || RHS.isUndef())
50667     return false;
50668 
50669   // Look for the following pattern:
50670   //   A = < float a0, float a1, float a2, float a3 >
50671   //   B = < float b0, float b1, float b2, float b3 >
50672   // and
50673   //   LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
50674   //   RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
50675   // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
50676   // which is A horizontal-op B.
50677 
50678   MVT VT = LHS.getSimpleValueType();
50679   assert((VT.is128BitVector() || VT.is256BitVector()) &&
50680          "Unsupported vector type for horizontal add/sub");
50681   unsigned NumElts = VT.getVectorNumElements();
50682 
50683   auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
50684                         SmallVectorImpl<int> &ShuffleMask) {
50685     bool UseSubVector = false;
50686     if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
50687         Op.getOperand(0).getValueType().is256BitVector() &&
50688         llvm::isNullConstant(Op.getOperand(1))) {
50689       Op = Op.getOperand(0);
50690       UseSubVector = true;
50691     }
50692     SmallVector<SDValue, 2> SrcOps;
50693     SmallVector<int, 16> SrcMask, ScaledMask;
50694     SDValue BC = peekThroughBitcasts(Op);
50695     if (getTargetShuffleInputs(BC, SrcOps, SrcMask, DAG) &&
50696         !isAnyZero(SrcMask) && all_of(SrcOps, [BC](SDValue Op) {
50697           return Op.getValueSizeInBits() == BC.getValueSizeInBits();
50698         })) {
50699       resolveTargetShuffleInputsAndMask(SrcOps, SrcMask);
50700       if (!UseSubVector && SrcOps.size() <= 2 &&
50701           scaleShuffleElements(SrcMask, NumElts, ScaledMask)) {
50702         N0 = !SrcOps.empty() ? SrcOps[0] : SDValue();
50703         N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
50704         ShuffleMask.assign(ScaledMask.begin(), ScaledMask.end());
50705       }
50706       if (UseSubVector && SrcOps.size() == 1 &&
50707           scaleShuffleElements(SrcMask, 2 * NumElts, ScaledMask)) {
50708         std::tie(N0, N1) = DAG.SplitVector(SrcOps[0], SDLoc(Op));
50709         ArrayRef<int> Mask = ArrayRef<int>(ScaledMask).slice(0, NumElts);
50710         ShuffleMask.assign(Mask.begin(), Mask.end());
50711       }
50712     }
50713   };
50714 
50715   // View LHS in the form
50716   //   LHS = VECTOR_SHUFFLE A, B, LMask
50717   // If LHS is not a shuffle, then pretend it is the identity shuffle:
50718   //   LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
50719   // NOTE: A default initialized SDValue represents an UNDEF of type VT.
50720   SDValue A, B;
50721   SmallVector<int, 16> LMask;
50722   GetShuffle(LHS, A, B, LMask);
50723 
50724   // Likewise, view RHS in the form
50725   //   RHS = VECTOR_SHUFFLE C, D, RMask
50726   SDValue C, D;
50727   SmallVector<int, 16> RMask;
50728   GetShuffle(RHS, C, D, RMask);
50729 
50730   // At least one of the operands should be a vector shuffle.
50731   unsigned NumShuffles = (LMask.empty() ? 0 : 1) + (RMask.empty() ? 0 : 1);
50732   if (NumShuffles == 0)
50733     return false;
50734 
50735   if (LMask.empty()) {
50736     A = LHS;
50737     for (unsigned i = 0; i != NumElts; ++i)
50738       LMask.push_back(i);
50739   }
50740 
50741   if (RMask.empty()) {
50742     C = RHS;
50743     for (unsigned i = 0; i != NumElts; ++i)
50744       RMask.push_back(i);
50745   }
50746 
50747   // If we have an unary mask, ensure the other op is set to null.
50748   if (isUndefOrInRange(LMask, 0, NumElts))
50749     B = SDValue();
50750   else if (isUndefOrInRange(LMask, NumElts, NumElts * 2))
50751     A = SDValue();
50752 
50753   if (isUndefOrInRange(RMask, 0, NumElts))
50754     D = SDValue();
50755   else if (isUndefOrInRange(RMask, NumElts, NumElts * 2))
50756     C = SDValue();
50757 
50758   // If A and B occur in reverse order in RHS, then canonicalize by commuting
50759   // RHS operands and shuffle mask.
50760   if (A != C) {
50761     std::swap(C, D);
50762     ShuffleVectorSDNode::commuteMask(RMask);
50763   }
50764   // Check that the shuffles are both shuffling the same vectors.
50765   if (!(A == C && B == D))
50766     return false;
50767 
50768   PostShuffleMask.clear();
50769   PostShuffleMask.append(NumElts, SM_SentinelUndef);
50770 
50771   // LHS and RHS are now:
50772   //   LHS = shuffle A, B, LMask
50773   //   RHS = shuffle A, B, RMask
50774   // Check that the masks correspond to performing a horizontal operation.
50775   // AVX defines horizontal add/sub to operate independently on 128-bit lanes,
50776   // so we just repeat the inner loop if this is a 256-bit op.
50777   unsigned Num128BitChunks = VT.getSizeInBits() / 128;
50778   unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks;
50779   unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
50780   assert((NumEltsPer128BitChunk % 2 == 0) &&
50781          "Vector type should have an even number of elements in each lane");
50782   for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) {
50783     for (unsigned i = 0; i != NumEltsPer128BitChunk; ++i) {
50784       // Ignore undefined components.
50785       int LIdx = LMask[i + j], RIdx = RMask[i + j];
50786       if (LIdx < 0 || RIdx < 0 ||
50787           (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
50788           (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
50789         continue;
50790 
50791       // Check that successive odd/even elements are being operated on. If not,
50792       // this is not a horizontal operation.
50793       if (!((RIdx & 1) == 1 && (LIdx + 1) == RIdx) &&
50794           !((LIdx & 1) == 1 && (RIdx + 1) == LIdx && IsCommutative))
50795         return false;
50796 
50797       // Compute the post-shuffle mask index based on where the element
50798       // is stored in the HOP result, and where it needs to be moved to.
50799       int Base = LIdx & ~1u;
50800       int Index = ((Base % NumEltsPer128BitChunk) / 2) +
50801                   ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1));
50802 
50803       // The  low half of the 128-bit result must choose from A.
50804       // The high half of the 128-bit result must choose from B,
50805       // unless B is undef. In that case, we are always choosing from A.
50806       if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk))
50807         Index += NumEltsPer64BitChunk;
50808       PostShuffleMask[i + j] = Index;
50809     }
50810   }
50811 
50812   SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
50813   SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
50814 
50815   bool IsIdentityPostShuffle =
50816       isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0);
50817   if (IsIdentityPostShuffle)
50818     PostShuffleMask.clear();
50819 
50820   // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split).
50821   if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() &&
50822       isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask))
50823     return false;
50824 
50825   // If the source nodes are already used in HorizOps then always accept this.
50826   // Shuffle folding should merge these back together.
50827   bool FoundHorizLHS = llvm::any_of(NewLHS->uses(), [&](SDNode *User) {
50828     return User->getOpcode() == HOpcode && User->getValueType(0) == VT;
50829   });
50830   bool FoundHorizRHS = llvm::any_of(NewRHS->uses(), [&](SDNode *User) {
50831     return User->getOpcode() == HOpcode && User->getValueType(0) == VT;
50832   });
50833   bool ForceHorizOp = FoundHorizLHS && FoundHorizRHS;
50834 
50835   // Assume a SingleSource HOP if we only shuffle one input and don't need to
50836   // shuffle the result.
50837   if (!ForceHorizOp &&
50838       !shouldUseHorizontalOp(NewLHS == NewRHS &&
50839                                  (NumShuffles < 2 || !IsIdentityPostShuffle),
50840                              DAG, Subtarget))
50841     return false;
50842 
50843   LHS = DAG.getBitcast(VT, NewLHS);
50844   RHS = DAG.getBitcast(VT, NewRHS);
50845   return true;
50846 }
50847 
50848 // Try to synthesize horizontal (f)hadd/hsub from (f)adds/subs of shuffles.
50849 static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
50850                                          const X86Subtarget &Subtarget) {
50851   EVT VT = N->getValueType(0);
50852   unsigned Opcode = N->getOpcode();
50853   bool IsAdd = (Opcode == ISD::FADD) || (Opcode == ISD::ADD);
50854   SmallVector<int, 8> PostShuffleMask;
50855 
50856   switch (Opcode) {
50857   case ISD::FADD:
50858   case ISD::FSUB:
50859     if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
50860         (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
50861       SDValue LHS = N->getOperand(0);
50862       SDValue RHS = N->getOperand(1);
50863       auto HorizOpcode = IsAdd ? X86ISD::FHADD : X86ISD::FHSUB;
50864       if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
50865                             PostShuffleMask)) {
50866         SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
50867         if (!PostShuffleMask.empty())
50868           HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
50869                                             DAG.getUNDEF(VT), PostShuffleMask);
50870         return HorizBinOp;
50871       }
50872     }
50873     break;
50874   case ISD::ADD:
50875   case ISD::SUB:
50876     if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
50877                                  VT == MVT::v16i16 || VT == MVT::v8i32)) {
50878       SDValue LHS = N->getOperand(0);
50879       SDValue RHS = N->getOperand(1);
50880       auto HorizOpcode = IsAdd ? X86ISD::HADD : X86ISD::HSUB;
50881       if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
50882                             PostShuffleMask)) {
50883         auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL,
50884                                         ArrayRef<SDValue> Ops) {
50885           return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops);
50886         };
50887         SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
50888                                               {LHS, RHS}, HOpBuilder);
50889         if (!PostShuffleMask.empty())
50890           HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
50891                                             DAG.getUNDEF(VT), PostShuffleMask);
50892         return HorizBinOp;
50893       }
50894     }
50895     break;
50896   }
50897 
50898   return SDValue();
50899 }
50900 
50901 //  Try to combine the following nodes
50902 //  t29: i64 = X86ISD::Wrapper TargetConstantPool:i64
50903 //    <i32 -2147483648[float -0.000000e+00]> 0
50904 //  t27: v16i32[v16f32],ch = X86ISD::VBROADCAST_LOAD
50905 //    <(load 4 from constant-pool)> t0, t29
50906 //  [t30: v16i32 = bitcast t27]
50907 //  t6: v16i32 = xor t7, t27[t30]
50908 //  t11: v16f32 = bitcast t6
50909 //  t21: v16f32 = X86ISD::VFMULC[X86ISD::VCFMULC] t11, t8
50910 //  into X86ISD::VFCMULC[X86ISD::VFMULC] if possible:
50911 //  t22: v16f32 = bitcast t7
50912 //  t23: v16f32 = X86ISD::VFCMULC[X86ISD::VFMULC] t8, t22
50913 //  t24: v32f16 = bitcast t23
50914 static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
50915                                   const X86Subtarget &Subtarget) {
50916   EVT VT = N->getValueType(0);
50917   SDValue LHS = N->getOperand(0);
50918   SDValue RHS = N->getOperand(1);
50919   int CombineOpcode =
50920       N->getOpcode() == X86ISD::VFCMULC ? X86ISD::VFMULC : X86ISD::VFCMULC;
50921   auto combineConjugation = [&](SDValue &r) {
50922     if (LHS->getOpcode() == ISD::BITCAST && RHS.hasOneUse()) {
50923       SDValue XOR = LHS.getOperand(0);
50924       if (XOR->getOpcode() == ISD::XOR && XOR.hasOneUse()) {
50925         KnownBits XORRHS = DAG.computeKnownBits(XOR.getOperand(1));
50926         if (XORRHS.isConstant()) {
50927           APInt ConjugationInt32 = APInt(32, 0x80000000, true);
50928           APInt ConjugationInt64 = APInt(64, 0x8000000080000000ULL, true);
50929           if ((XORRHS.getBitWidth() == 32 &&
50930                XORRHS.getConstant() == ConjugationInt32) ||
50931               (XORRHS.getBitWidth() == 64 &&
50932                XORRHS.getConstant() == ConjugationInt64)) {
50933             SelectionDAG::FlagInserter FlagsInserter(DAG, N);
50934             SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0));
50935             SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F);
50936             r = DAG.getBitcast(VT, FCMulC);
50937             return true;
50938           }
50939         }
50940       }
50941     }
50942     return false;
50943   };
50944   SDValue Res;
50945   if (combineConjugation(Res))
50946     return Res;
50947   std::swap(LHS, RHS);
50948   if (combineConjugation(Res))
50949     return Res;
50950   return Res;
50951 }
50952 
50953 //  Try to combine the following nodes:
50954 //  FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
50955 static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
50956                                 const X86Subtarget &Subtarget) {
50957   auto AllowContract = [&DAG](const SDNodeFlags &Flags) {
50958     return DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
50959            Flags.hasAllowContract();
50960   };
50961 
50962   auto HasNoSignedZero = [&DAG](const SDNodeFlags &Flags) {
50963     return DAG.getTarget().Options.NoSignedZerosFPMath ||
50964            Flags.hasNoSignedZeros();
50965   };
50966   auto IsVectorAllNegativeZero = [&DAG](SDValue Op) {
50967     APInt AI = APInt(32, 0x80008000, true);
50968     KnownBits Bits = DAG.computeKnownBits(Op);
50969     return Bits.getBitWidth() == 32 && Bits.isConstant() &&
50970            Bits.getConstant() == AI;
50971   };
50972 
50973   if (N->getOpcode() != ISD::FADD || !Subtarget.hasFP16() ||
50974       !AllowContract(N->getFlags()))
50975     return SDValue();
50976 
50977   EVT VT = N->getValueType(0);
50978   if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16)
50979     return SDValue();
50980 
50981   SDValue LHS = N->getOperand(0);
50982   SDValue RHS = N->getOperand(1);
50983   bool IsConj;
50984   SDValue FAddOp1, MulOp0, MulOp1;
50985   auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &AllowContract,
50986                        &IsVectorAllNegativeZero,
50987                        &HasNoSignedZero](SDValue N) -> bool {
50988     if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
50989       return false;
50990     SDValue Op0 = N.getOperand(0);
50991     unsigned Opcode = Op0.getOpcode();
50992     if (Op0.hasOneUse() && AllowContract(Op0->getFlags())) {
50993       if ((Opcode == X86ISD::VFMULC || Opcode == X86ISD::VFCMULC)) {
50994         MulOp0 = Op0.getOperand(0);
50995         MulOp1 = Op0.getOperand(1);
50996         IsConj = Opcode == X86ISD::VFCMULC;
50997         return true;
50998       }
50999       if ((Opcode == X86ISD::VFMADDC || Opcode == X86ISD::VFCMADDC) &&
51000           ((ISD::isBuildVectorAllZeros(Op0->getOperand(2).getNode()) &&
51001             HasNoSignedZero(Op0->getFlags())) ||
51002            IsVectorAllNegativeZero(Op0->getOperand(2)))) {
51003         MulOp0 = Op0.getOperand(0);
51004         MulOp1 = Op0.getOperand(1);
51005         IsConj = Opcode == X86ISD::VFCMADDC;
51006         return true;
51007       }
51008     }
51009     return false;
51010   };
51011 
51012   if (GetCFmulFrom(LHS))
51013     FAddOp1 = RHS;
51014   else if (GetCFmulFrom(RHS))
51015     FAddOp1 = LHS;
51016   else
51017     return SDValue();
51018 
51019   MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
51020   FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
51021   unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
51022   // FIXME: How do we handle when fast math flags of FADD are different from
51023   // CFMUL's?
51024   SDValue CFmul =
51025       DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags());
51026   return DAG.getBitcast(VT, CFmul);
51027 }
51028 
51029 /// Do target-specific dag combines on floating-point adds/subs.
51030 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
51031                                const X86Subtarget &Subtarget) {
51032   if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget))
51033     return HOp;
51034 
51035   if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
51036     return COp;
51037 
51038   return SDValue();
51039 }
51040 
51041 /// Attempt to pre-truncate inputs to arithmetic ops if it will simplify
51042 /// the codegen.
51043 /// e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) )
51044 /// TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove
51045 ///       anything that is guaranteed to be transformed by DAGCombiner.
51046 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
51047                                           const X86Subtarget &Subtarget,
51048                                           const SDLoc &DL) {
51049   assert(N->getOpcode() == ISD::TRUNCATE && "Wrong opcode");
51050   SDValue Src = N->getOperand(0);
51051   unsigned SrcOpcode = Src.getOpcode();
51052   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51053 
51054   EVT VT = N->getValueType(0);
51055   EVT SrcVT = Src.getValueType();
51056 
51057   auto IsFreeTruncation = [VT](SDValue Op) {
51058     unsigned TruncSizeInBits = VT.getScalarSizeInBits();
51059 
51060     // See if this has been extended from a smaller/equal size to
51061     // the truncation size, allowing a truncation to combine with the extend.
51062     unsigned Opcode = Op.getOpcode();
51063     if ((Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND ||
51064          Opcode == ISD::ZERO_EXTEND) &&
51065         Op.getOperand(0).getScalarValueSizeInBits() <= TruncSizeInBits)
51066       return true;
51067 
51068     // See if this is a single use constant which can be constant folded.
51069     // NOTE: We don't peek throught bitcasts here because there is currently
51070     // no support for constant folding truncate+bitcast+vector_of_constants. So
51071     // we'll just send up with a truncate on both operands which will
51072     // get turned back into (truncate (binop)) causing an infinite loop.
51073     return ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
51074   };
51075 
51076   auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
51077     SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
51078     SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
51079     return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
51080   };
51081 
51082   // Don't combine if the operation has other uses.
51083   if (!Src.hasOneUse())
51084     return SDValue();
51085 
51086   // Only support vector truncation for now.
51087   // TODO: i64 scalar math would benefit as well.
51088   if (!VT.isVector())
51089     return SDValue();
51090 
51091   // In most cases its only worth pre-truncating if we're only facing the cost
51092   // of one truncation.
51093   // i.e. if one of the inputs will constant fold or the input is repeated.
51094   switch (SrcOpcode) {
51095   case ISD::MUL:
51096     // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its
51097     // better to truncate if we have the chance.
51098     if (SrcVT.getScalarType() == MVT::i64 &&
51099         TLI.isOperationLegal(SrcOpcode, VT) &&
51100         !TLI.isOperationLegal(SrcOpcode, SrcVT))
51101       return TruncateArithmetic(Src.getOperand(0), Src.getOperand(1));
51102     [[fallthrough]];
51103   case ISD::AND:
51104   case ISD::XOR:
51105   case ISD::OR:
51106   case ISD::ADD:
51107   case ISD::SUB: {
51108     SDValue Op0 = Src.getOperand(0);
51109     SDValue Op1 = Src.getOperand(1);
51110     if (TLI.isOperationLegal(SrcOpcode, VT) &&
51111         (Op0 == Op1 || IsFreeTruncation(Op0) || IsFreeTruncation(Op1)))
51112       return TruncateArithmetic(Op0, Op1);
51113     break;
51114   }
51115   }
51116 
51117   return SDValue();
51118 }
51119 
51120 // Try to form a MULHU or MULHS node by looking for
51121 // (trunc (srl (mul ext, ext), 16))
51122 // TODO: This is X86 specific because we want to be able to handle wide types
51123 // before type legalization. But we can only do it if the vector will be
51124 // legalized via widening/splitting. Type legalization can't handle promotion
51125 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
51126 // combiner.
51127 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
51128                             SelectionDAG &DAG, const X86Subtarget &Subtarget) {
51129   // First instruction should be a right shift of a multiply.
51130   if (Src.getOpcode() != ISD::SRL ||
51131       Src.getOperand(0).getOpcode() != ISD::MUL)
51132     return SDValue();
51133 
51134   if (!Subtarget.hasSSE2())
51135     return SDValue();
51136 
51137   // Only handle vXi16 types that are at least 128-bits unless they will be
51138   // widened.
51139   if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)
51140     return SDValue();
51141 
51142   // Input type should be at least vXi32.
51143   EVT InVT = Src.getValueType();
51144   if (InVT.getVectorElementType().getSizeInBits() < 32)
51145     return SDValue();
51146 
51147   // Need a shift by 16.
51148   APInt ShiftAmt;
51149   if (!ISD::isConstantSplatVector(Src.getOperand(1).getNode(), ShiftAmt) ||
51150       ShiftAmt != 16)
51151     return SDValue();
51152 
51153   SDValue LHS = Src.getOperand(0).getOperand(0);
51154   SDValue RHS = Src.getOperand(0).getOperand(1);
51155 
51156   // Count leading sign/zero bits on both inputs - if there are enough then
51157   // truncation back to vXi16 will be cheap - either as a pack/shuffle
51158   // sequence or using AVX512 truncations. If the inputs are sext/zext then the
51159   // truncations may actually be free by peeking through to the ext source.
51160   auto IsSext = [&DAG](SDValue V) {
51161     return DAG.ComputeMaxSignificantBits(V) <= 16;
51162   };
51163   auto IsZext = [&DAG](SDValue V) {
51164     return DAG.computeKnownBits(V).countMaxActiveBits() <= 16;
51165   };
51166 
51167   bool IsSigned = IsSext(LHS) && IsSext(RHS);
51168   bool IsUnsigned = IsZext(LHS) && IsZext(RHS);
51169   if (!IsSigned && !IsUnsigned)
51170     return SDValue();
51171 
51172   // Check if both inputs are extensions, which will be removed by truncation.
51173   bool IsTruncateFree = (LHS.getOpcode() == ISD::SIGN_EXTEND ||
51174                          LHS.getOpcode() == ISD::ZERO_EXTEND) &&
51175                         (RHS.getOpcode() == ISD::SIGN_EXTEND ||
51176                          RHS.getOpcode() == ISD::ZERO_EXTEND) &&
51177                         LHS.getOperand(0).getScalarValueSizeInBits() <= 16 &&
51178                         RHS.getOperand(0).getScalarValueSizeInBits() <= 16;
51179 
51180   // For AVX2+ targets, with the upper bits known zero, we can perform MULHU on
51181   // the (bitcasted) inputs directly, and then cheaply pack/truncate the result
51182   // (upper elts will be zero). Don't attempt this with just AVX512F as MULHU
51183   // will have to split anyway.
51184   unsigned InSizeInBits = InVT.getSizeInBits();
51185   if (IsUnsigned && !IsTruncateFree && Subtarget.hasInt256() &&
51186       !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) &&
51187       (InSizeInBits % 16) == 0) {
51188     EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
51189                                 InVT.getSizeInBits() / 16);
51190     SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS),
51191                               DAG.getBitcast(BCVT, RHS));
51192     return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res));
51193   }
51194 
51195   // Truncate back to source type.
51196   LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS);
51197   RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS);
51198 
51199   unsigned Opc = IsSigned ? ISD::MULHS : ISD::MULHU;
51200   return DAG.getNode(Opc, DL, VT, LHS, RHS);
51201 }
51202 
51203 // Attempt to match PMADDUBSW, which multiplies corresponding unsigned bytes
51204 // from one vector with signed bytes from another vector, adds together
51205 // adjacent pairs of 16-bit products, and saturates the result before
51206 // truncating to 16-bits.
51207 //
51208 // Which looks something like this:
51209 // (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
51210 //                 (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
51211 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
51212                                const X86Subtarget &Subtarget,
51213                                const SDLoc &DL) {
51214   if (!VT.isVector() || !Subtarget.hasSSSE3())
51215     return SDValue();
51216 
51217   unsigned NumElems = VT.getVectorNumElements();
51218   EVT ScalarVT = VT.getVectorElementType();
51219   if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems))
51220     return SDValue();
51221 
51222   SDValue SSatVal = detectSSatPattern(In, VT);
51223   if (!SSatVal || SSatVal.getOpcode() != ISD::ADD)
51224     return SDValue();
51225 
51226   // Ok this is a signed saturation of an ADD. See if this ADD is adding pairs
51227   // of multiplies from even/odd elements.
51228   SDValue N0 = SSatVal.getOperand(0);
51229   SDValue N1 = SSatVal.getOperand(1);
51230 
51231   if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
51232     return SDValue();
51233 
51234   SDValue N00 = N0.getOperand(0);
51235   SDValue N01 = N0.getOperand(1);
51236   SDValue N10 = N1.getOperand(0);
51237   SDValue N11 = N1.getOperand(1);
51238 
51239   // TODO: Handle constant vectors and use knownbits/computenumsignbits?
51240   // Canonicalize zero_extend to LHS.
51241   if (N01.getOpcode() == ISD::ZERO_EXTEND)
51242     std::swap(N00, N01);
51243   if (N11.getOpcode() == ISD::ZERO_EXTEND)
51244     std::swap(N10, N11);
51245 
51246   // Ensure we have a zero_extend and a sign_extend.
51247   if (N00.getOpcode() != ISD::ZERO_EXTEND ||
51248       N01.getOpcode() != ISD::SIGN_EXTEND ||
51249       N10.getOpcode() != ISD::ZERO_EXTEND ||
51250       N11.getOpcode() != ISD::SIGN_EXTEND)
51251     return SDValue();
51252 
51253   // Peek through the extends.
51254   N00 = N00.getOperand(0);
51255   N01 = N01.getOperand(0);
51256   N10 = N10.getOperand(0);
51257   N11 = N11.getOperand(0);
51258 
51259   // Ensure the extend is from vXi8.
51260   if (N00.getValueType().getVectorElementType() != MVT::i8 ||
51261       N01.getValueType().getVectorElementType() != MVT::i8 ||
51262       N10.getValueType().getVectorElementType() != MVT::i8 ||
51263       N11.getValueType().getVectorElementType() != MVT::i8)
51264     return SDValue();
51265 
51266   // All inputs should be build_vectors.
51267   if (N00.getOpcode() != ISD::BUILD_VECTOR ||
51268       N01.getOpcode() != ISD::BUILD_VECTOR ||
51269       N10.getOpcode() != ISD::BUILD_VECTOR ||
51270       N11.getOpcode() != ISD::BUILD_VECTOR)
51271     return SDValue();
51272 
51273   // N00/N10 are zero extended. N01/N11 are sign extended.
51274 
51275   // For each element, we need to ensure we have an odd element from one vector
51276   // multiplied by the odd element of another vector and the even element from
51277   // one of the same vectors being multiplied by the even element from the
51278   // other vector. So we need to make sure for each element i, this operator
51279   // is being performed:
51280   //  A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
51281   SDValue ZExtIn, SExtIn;
51282   for (unsigned i = 0; i != NumElems; ++i) {
51283     SDValue N00Elt = N00.getOperand(i);
51284     SDValue N01Elt = N01.getOperand(i);
51285     SDValue N10Elt = N10.getOperand(i);
51286     SDValue N11Elt = N11.getOperand(i);
51287     // TODO: Be more tolerant to undefs.
51288     if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51289         N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51290         N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51291         N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
51292       return SDValue();
51293     auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
51294     auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
51295     auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
51296     auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
51297     if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
51298       return SDValue();
51299     unsigned IdxN00 = ConstN00Elt->getZExtValue();
51300     unsigned IdxN01 = ConstN01Elt->getZExtValue();
51301     unsigned IdxN10 = ConstN10Elt->getZExtValue();
51302     unsigned IdxN11 = ConstN11Elt->getZExtValue();
51303     // Add is commutative so indices can be reordered.
51304     if (IdxN00 > IdxN10) {
51305       std::swap(IdxN00, IdxN10);
51306       std::swap(IdxN01, IdxN11);
51307     }
51308     // N0 indices be the even element. N1 indices must be the next odd element.
51309     if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
51310         IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
51311       return SDValue();
51312     SDValue N00In = N00Elt.getOperand(0);
51313     SDValue N01In = N01Elt.getOperand(0);
51314     SDValue N10In = N10Elt.getOperand(0);
51315     SDValue N11In = N11Elt.getOperand(0);
51316     // First time we find an input capture it.
51317     if (!ZExtIn) {
51318       ZExtIn = N00In;
51319       SExtIn = N01In;
51320     }
51321     if (ZExtIn != N00In || SExtIn != N01In ||
51322         ZExtIn != N10In || SExtIn != N11In)
51323       return SDValue();
51324   }
51325 
51326   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
51327                          ArrayRef<SDValue> Ops) {
51328     // Shrink by adding truncate nodes and let DAGCombine fold with the
51329     // sources.
51330     EVT InVT = Ops[0].getValueType();
51331     assert(InVT.getScalarType() == MVT::i8 &&
51332            "Unexpected scalar element type");
51333     assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
51334     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
51335                                  InVT.getVectorNumElements() / 2);
51336     return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
51337   };
51338   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
51339                           PMADDBuilder);
51340 }
51341 
51342 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
51343                                const X86Subtarget &Subtarget) {
51344   EVT VT = N->getValueType(0);
51345   SDValue Src = N->getOperand(0);
51346   SDLoc DL(N);
51347 
51348   // Attempt to pre-truncate inputs to arithmetic ops instead.
51349   if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
51350     return V;
51351 
51352   // Try to detect AVG pattern first.
51353   if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
51354     return Avg;
51355 
51356   // Try to detect PMADD
51357   if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
51358     return PMAdd;
51359 
51360   // Try to combine truncation with signed/unsigned saturation.
51361   if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
51362     return Val;
51363 
51364   // Try to combine PMULHUW/PMULHW for vXi16.
51365   if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
51366     return V;
51367 
51368   // The bitcast source is a direct mmx result.
51369   // Detect bitcasts between i32 to x86mmx
51370   if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {
51371     SDValue BCSrc = Src.getOperand(0);
51372     if (BCSrc.getValueType() == MVT::x86mmx)
51373       return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
51374   }
51375 
51376   return SDValue();
51377 }
51378 
51379 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
51380                              TargetLowering::DAGCombinerInfo &DCI) {
51381   EVT VT = N->getValueType(0);
51382   SDValue In = N->getOperand(0);
51383   SDLoc DL(N);
51384 
51385   if (SDValue SSatVal = detectSSatPattern(In, VT))
51386     return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
51387   if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL))
51388     return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
51389 
51390   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51391   APInt DemandedMask(APInt::getAllOnes(VT.getScalarSizeInBits()));
51392   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
51393     return SDValue(N, 0);
51394 
51395   return SDValue();
51396 }
51397 
51398 /// Returns the negated value if the node \p N flips sign of FP value.
51399 ///
51400 /// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
51401 /// or FSUB(0, x)
51402 /// AVX512F does not have FXOR, so FNEG is lowered as
51403 /// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))).
51404 /// In this case we go though all bitcasts.
51405 /// This also recognizes splat of a negated value and returns the splat of that
51406 /// value.
51407 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
51408   if (N->getOpcode() == ISD::FNEG)
51409     return N->getOperand(0);
51410 
51411   // Don't recurse exponentially.
51412   if (Depth > SelectionDAG::MaxRecursionDepth)
51413     return SDValue();
51414 
51415   unsigned ScalarSize = N->getValueType(0).getScalarSizeInBits();
51416 
51417   SDValue Op = peekThroughBitcasts(SDValue(N, 0));
51418   EVT VT = Op->getValueType(0);
51419 
51420   // Make sure the element size doesn't change.
51421   if (VT.getScalarSizeInBits() != ScalarSize)
51422     return SDValue();
51423 
51424   unsigned Opc = Op.getOpcode();
51425   switch (Opc) {
51426   case ISD::VECTOR_SHUFFLE: {
51427     // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate
51428     // of this is VECTOR_SHUFFLE(-VEC1, UNDEF).  The mask can be anything here.
51429     if (!Op.getOperand(1).isUndef())
51430       return SDValue();
51431     if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
51432       if (NegOp0.getValueType() == VT) // FIXME: Can we do better?
51433         return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT),
51434                                     cast<ShuffleVectorSDNode>(Op)->getMask());
51435     break;
51436   }
51437   case ISD::INSERT_VECTOR_ELT: {
51438     // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF,
51439     // -V, INDEX).
51440     SDValue InsVector = Op.getOperand(0);
51441     SDValue InsVal = Op.getOperand(1);
51442     if (!InsVector.isUndef())
51443       return SDValue();
51444     if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
51445       if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME
51446         return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,
51447                            NegInsVal, Op.getOperand(2));
51448     break;
51449   }
51450   case ISD::FSUB:
51451   case ISD::XOR:
51452   case X86ISD::FXOR: {
51453     SDValue Op1 = Op.getOperand(1);
51454     SDValue Op0 = Op.getOperand(0);
51455 
51456     // For XOR and FXOR, we want to check if constant
51457     // bits of Op1 are sign bit masks. For FSUB, we
51458     // have to check if constant bits of Op0 are sign
51459     // bit masks and hence we swap the operands.
51460     if (Opc == ISD::FSUB)
51461       std::swap(Op0, Op1);
51462 
51463     APInt UndefElts;
51464     SmallVector<APInt, 16> EltBits;
51465     // Extract constant bits and see if they are all
51466     // sign bit masks. Ignore the undef elements.
51467     if (getTargetConstantBitsFromNode(Op1, ScalarSize, UndefElts, EltBits,
51468                                       /* AllowWholeUndefs */ true,
51469                                       /* AllowPartialUndefs */ false)) {
51470       for (unsigned I = 0, E = EltBits.size(); I < E; I++)
51471         if (!UndefElts[I] && !EltBits[I].isSignMask())
51472           return SDValue();
51473 
51474       // Only allow bitcast from correctly-sized constant.
51475       Op0 = peekThroughBitcasts(Op0);
51476       if (Op0.getScalarValueSizeInBits() == ScalarSize)
51477         return Op0;
51478     }
51479     break;
51480   } // case
51481   } // switch
51482 
51483   return SDValue();
51484 }
51485 
51486 static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
51487                                 bool NegRes) {
51488   if (NegMul) {
51489     switch (Opcode) {
51490     default: llvm_unreachable("Unexpected opcode");
51491     case ISD::FMA:              Opcode = X86ISD::FNMADD;        break;
51492     case ISD::STRICT_FMA:       Opcode = X86ISD::STRICT_FNMADD; break;
51493     case X86ISD::FMADD_RND:     Opcode = X86ISD::FNMADD_RND;    break;
51494     case X86ISD::FMSUB:         Opcode = X86ISD::FNMSUB;        break;
51495     case X86ISD::STRICT_FMSUB:  Opcode = X86ISD::STRICT_FNMSUB; break;
51496     case X86ISD::FMSUB_RND:     Opcode = X86ISD::FNMSUB_RND;    break;
51497     case X86ISD::FNMADD:        Opcode = ISD::FMA;              break;
51498     case X86ISD::STRICT_FNMADD: Opcode = ISD::STRICT_FMA;       break;
51499     case X86ISD::FNMADD_RND:    Opcode = X86ISD::FMADD_RND;     break;
51500     case X86ISD::FNMSUB:        Opcode = X86ISD::FMSUB;         break;
51501     case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FMSUB;  break;
51502     case X86ISD::FNMSUB_RND:    Opcode = X86ISD::FMSUB_RND;     break;
51503     }
51504   }
51505 
51506   if (NegAcc) {
51507     switch (Opcode) {
51508     default: llvm_unreachable("Unexpected opcode");
51509     case ISD::FMA:              Opcode = X86ISD::FMSUB;         break;
51510     case ISD::STRICT_FMA:       Opcode = X86ISD::STRICT_FMSUB;  break;
51511     case X86ISD::FMADD_RND:     Opcode = X86ISD::FMSUB_RND;     break;
51512     case X86ISD::FMSUB:         Opcode = ISD::FMA;              break;
51513     case X86ISD::STRICT_FMSUB:  Opcode = ISD::STRICT_FMA;       break;
51514     case X86ISD::FMSUB_RND:     Opcode = X86ISD::FMADD_RND;     break;
51515     case X86ISD::FNMADD:        Opcode = X86ISD::FNMSUB;        break;
51516     case X86ISD::STRICT_FNMADD: Opcode = X86ISD::STRICT_FNMSUB; break;
51517     case X86ISD::FNMADD_RND:    Opcode = X86ISD::FNMSUB_RND;    break;
51518     case X86ISD::FNMSUB:        Opcode = X86ISD::FNMADD;        break;
51519     case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FNMADD; break;
51520     case X86ISD::FNMSUB_RND:    Opcode = X86ISD::FNMADD_RND;    break;
51521     case X86ISD::FMADDSUB:      Opcode = X86ISD::FMSUBADD;      break;
51522     case X86ISD::FMADDSUB_RND:  Opcode = X86ISD::FMSUBADD_RND;  break;
51523     case X86ISD::FMSUBADD:      Opcode = X86ISD::FMADDSUB;      break;
51524     case X86ISD::FMSUBADD_RND:  Opcode = X86ISD::FMADDSUB_RND;  break;
51525     }
51526   }
51527 
51528   if (NegRes) {
51529     switch (Opcode) {
51530     // For accuracy reason, we never combine fneg and fma under strict FP.
51531     default: llvm_unreachable("Unexpected opcode");
51532     case ISD::FMA:             Opcode = X86ISD::FNMSUB;       break;
51533     case X86ISD::FMADD_RND:    Opcode = X86ISD::FNMSUB_RND;   break;
51534     case X86ISD::FMSUB:        Opcode = X86ISD::FNMADD;       break;
51535     case X86ISD::FMSUB_RND:    Opcode = X86ISD::FNMADD_RND;   break;
51536     case X86ISD::FNMADD:       Opcode = X86ISD::FMSUB;        break;
51537     case X86ISD::FNMADD_RND:   Opcode = X86ISD::FMSUB_RND;    break;
51538     case X86ISD::FNMSUB:       Opcode = ISD::FMA;             break;
51539     case X86ISD::FNMSUB_RND:   Opcode = X86ISD::FMADD_RND;    break;
51540     }
51541   }
51542 
51543   return Opcode;
51544 }
51545 
51546 /// Do target-specific dag combines on floating point negations.
51547 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
51548                            TargetLowering::DAGCombinerInfo &DCI,
51549                            const X86Subtarget &Subtarget) {
51550   EVT OrigVT = N->getValueType(0);
51551   SDValue Arg = isFNEG(DAG, N);
51552   if (!Arg)
51553     return SDValue();
51554 
51555   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51556   EVT VT = Arg.getValueType();
51557   EVT SVT = VT.getScalarType();
51558   SDLoc DL(N);
51559 
51560   // Let legalize expand this if it isn't a legal type yet.
51561   if (!TLI.isTypeLegal(VT))
51562     return SDValue();
51563 
51564   // If we're negating a FMUL node on a target with FMA, then we can avoid the
51565   // use of a constant by performing (-0 - A*B) instead.
51566   // FIXME: Check rounding control flags as well once it becomes available.
51567   if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&
51568       Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
51569     SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
51570     SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
51571                                   Arg.getOperand(1), Zero);
51572     return DAG.getBitcast(OrigVT, NewNode);
51573   }
51574 
51575   bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
51576   bool LegalOperations = !DCI.isBeforeLegalizeOps();
51577   if (SDValue NegArg =
51578           TLI.getNegatedExpression(Arg, DAG, LegalOperations, CodeSize))
51579     return DAG.getBitcast(OrigVT, NegArg);
51580 
51581   return SDValue();
51582 }
51583 
51584 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
51585                                                 bool LegalOperations,
51586                                                 bool ForCodeSize,
51587                                                 NegatibleCost &Cost,
51588                                                 unsigned Depth) const {
51589   // fneg patterns are removable even if they have multiple uses.
51590   if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
51591     Cost = NegatibleCost::Cheaper;
51592     return DAG.getBitcast(Op.getValueType(), Arg);
51593   }
51594 
51595   EVT VT = Op.getValueType();
51596   EVT SVT = VT.getScalarType();
51597   unsigned Opc = Op.getOpcode();
51598   SDNodeFlags Flags = Op.getNode()->getFlags();
51599   switch (Opc) {
51600   case ISD::FMA:
51601   case X86ISD::FMSUB:
51602   case X86ISD::FNMADD:
51603   case X86ISD::FNMSUB:
51604   case X86ISD::FMADD_RND:
51605   case X86ISD::FMSUB_RND:
51606   case X86ISD::FNMADD_RND:
51607   case X86ISD::FNMSUB_RND: {
51608     if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
51609         !(SVT == MVT::f32 || SVT == MVT::f64) ||
51610         !isOperationLegal(ISD::FMA, VT))
51611       break;
51612 
51613     // Don't fold (fneg (fma (fneg x), y, (fneg z))) to (fma x, y, z)
51614     // if it may have signed zeros.
51615     if (!Flags.hasNoSignedZeros())
51616       break;
51617 
51618     // This is always negatible for free but we might be able to remove some
51619     // extra operand negations as well.
51620     SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
51621     for (int i = 0; i != 3; ++i)
51622       NewOps[i] = getCheaperNegatedExpression(
51623           Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1);
51624 
51625     bool NegA = !!NewOps[0];
51626     bool NegB = !!NewOps[1];
51627     bool NegC = !!NewOps[2];
51628     unsigned NewOpc = negateFMAOpcode(Opc, NegA != NegB, NegC, true);
51629 
51630     Cost = (NegA || NegB || NegC) ? NegatibleCost::Cheaper
51631                                   : NegatibleCost::Neutral;
51632 
51633     // Fill in the non-negated ops with the original values.
51634     for (int i = 0, e = Op.getNumOperands(); i != e; ++i)
51635       if (!NewOps[i])
51636         NewOps[i] = Op.getOperand(i);
51637     return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
51638   }
51639   case X86ISD::FRCP:
51640     if (SDValue NegOp0 =
51641             getNegatedExpression(Op.getOperand(0), DAG, LegalOperations,
51642                                  ForCodeSize, Cost, Depth + 1))
51643       return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0);
51644     break;
51645   }
51646 
51647   return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
51648                                               ForCodeSize, Cost, Depth);
51649 }
51650 
51651 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
51652                                  const X86Subtarget &Subtarget) {
51653   MVT VT = N->getSimpleValueType(0);
51654   // If we have integer vector types available, use the integer opcodes.
51655   if (!VT.isVector() || !Subtarget.hasSSE2())
51656     return SDValue();
51657 
51658   SDLoc dl(N);
51659 
51660   unsigned IntBits = VT.getScalarSizeInBits();
51661   MVT IntSVT = MVT::getIntegerVT(IntBits);
51662   MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
51663 
51664   SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
51665   SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
51666   unsigned IntOpcode;
51667   switch (N->getOpcode()) {
51668   default: llvm_unreachable("Unexpected FP logic op");
51669   case X86ISD::FOR:   IntOpcode = ISD::OR; break;
51670   case X86ISD::FXOR:  IntOpcode = ISD::XOR; break;
51671   case X86ISD::FAND:  IntOpcode = ISD::AND; break;
51672   case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
51673   }
51674   SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
51675   return DAG.getBitcast(VT, IntOp);
51676 }
51677 
51678 
51679 /// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
51680 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
51681   if (N->getOpcode() != ISD::XOR)
51682     return SDValue();
51683 
51684   SDValue LHS = N->getOperand(0);
51685   if (!isOneConstant(N->getOperand(1)) || LHS->getOpcode() != X86ISD::SETCC)
51686     return SDValue();
51687 
51688   X86::CondCode NewCC = X86::GetOppositeBranchCondition(
51689       X86::CondCode(LHS->getConstantOperandVal(0)));
51690   SDLoc DL(N);
51691   return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
51692 }
51693 
51694 static SDValue combineXorSubCTLZ(SDNode *N, SelectionDAG &DAG,
51695                                  const X86Subtarget &Subtarget) {
51696   assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::SUB) &&
51697          "Invalid opcode for combing with CTLZ");
51698   if (Subtarget.hasFastLZCNT())
51699     return SDValue();
51700 
51701   EVT VT = N->getValueType(0);
51702   if (VT != MVT::i8 && VT != MVT::i16 && VT != MVT::i32 &&
51703       (VT != MVT::i64 || !Subtarget.is64Bit()))
51704     return SDValue();
51705 
51706   SDValue N0 = N->getOperand(0);
51707   SDValue N1 = N->getOperand(1);
51708 
51709   if (N0.getOpcode() != ISD::CTLZ_ZERO_UNDEF &&
51710       N1.getOpcode() != ISD::CTLZ_ZERO_UNDEF)
51711     return SDValue();
51712 
51713   SDValue OpCTLZ;
51714   SDValue OpSizeTM1;
51715 
51716   if (N1.getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
51717     OpCTLZ = N1;
51718     OpSizeTM1 = N0;
51719   } else if (N->getOpcode() == ISD::SUB) {
51720     return SDValue();
51721   } else {
51722     OpCTLZ = N0;
51723     OpSizeTM1 = N1;
51724   }
51725 
51726   if (!OpCTLZ.hasOneUse())
51727     return SDValue();
51728   auto *C = dyn_cast<ConstantSDNode>(OpSizeTM1);
51729   if (!C)
51730     return SDValue();
51731 
51732   if (C->getZExtValue() != uint64_t(OpCTLZ.getValueSizeInBits() - 1))
51733     return SDValue();
51734   SDLoc DL(N);
51735   EVT OpVT = VT;
51736   SDValue Op = OpCTLZ.getOperand(0);
51737   if (VT == MVT::i8) {
51738     // Zero extend to i32 since there is not an i8 bsr.
51739     OpVT = MVT::i32;
51740     Op = DAG.getNode(ISD::ZERO_EXTEND, DL, OpVT, Op);
51741   }
51742 
51743   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
51744   Op = DAG.getNode(X86ISD::BSR, DL, VTs, Op);
51745   if (VT == MVT::i8)
51746     Op = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Op);
51747 
51748   return Op;
51749 }
51750 
51751 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
51752                           TargetLowering::DAGCombinerInfo &DCI,
51753                           const X86Subtarget &Subtarget) {
51754   SDValue N0 = N->getOperand(0);
51755   SDValue N1 = N->getOperand(1);
51756   EVT VT = N->getValueType(0);
51757 
51758   // If this is SSE1 only convert to FXOR to avoid scalarization.
51759   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
51760     return DAG.getBitcast(MVT::v4i32,
51761                           DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32,
51762                                       DAG.getBitcast(MVT::v4f32, N0),
51763                                       DAG.getBitcast(MVT::v4f32, N1)));
51764   }
51765 
51766   if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
51767     return Cmp;
51768 
51769   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
51770     return R;
51771 
51772   if (SDValue R = combineBitOpWithShift(N, DAG))
51773     return R;
51774 
51775   if (SDValue R = combineBitOpWithPACK(N, DAG))
51776     return R;
51777 
51778   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
51779     return FPLogic;
51780 
51781   if (SDValue R = combineXorSubCTLZ(N, DAG, Subtarget))
51782     return R;
51783 
51784   if (DCI.isBeforeLegalizeOps())
51785     return SDValue();
51786 
51787   if (SDValue SetCC = foldXor1SetCC(N, DAG))
51788     return SetCC;
51789 
51790   if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
51791     return R;
51792 
51793   if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
51794     return RV;
51795 
51796   // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs.
51797   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51798   if (llvm::isAllOnesConstant(N1) && N0.getOpcode() == ISD::BITCAST &&
51799       N0.getOperand(0).getValueType().isVector() &&
51800       N0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
51801       TLI.isTypeLegal(N0.getOperand(0).getValueType()) && N0.hasOneUse()) {
51802     return DAG.getBitcast(VT, DAG.getNOT(SDLoc(N), N0.getOperand(0),
51803                                          N0.getOperand(0).getValueType()));
51804   }
51805 
51806   // Handle AVX512 mask widening.
51807   // Fold not(insert_subvector(undef,sub)) -> insert_subvector(undef,not(sub))
51808   if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() &&
51809       VT.getVectorElementType() == MVT::i1 &&
51810       N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.getOperand(0).isUndef() &&
51811       TLI.isTypeLegal(N0.getOperand(1).getValueType())) {
51812     return DAG.getNode(
51813         ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
51814         DAG.getNOT(SDLoc(N), N0.getOperand(1), N0.getOperand(1).getValueType()),
51815         N0.getOperand(2));
51816   }
51817 
51818   // Fold xor(zext(xor(x,c1)),c2) -> xor(zext(x),xor(zext(c1),c2))
51819   // Fold xor(truncate(xor(x,c1)),c2) -> xor(truncate(x),xor(truncate(c1),c2))
51820   // TODO: Under what circumstances could this be performed in DAGCombine?
51821   if ((N0.getOpcode() == ISD::TRUNCATE || N0.getOpcode() == ISD::ZERO_EXTEND) &&
51822       N0.getOperand(0).getOpcode() == N->getOpcode()) {
51823     SDValue TruncExtSrc = N0.getOperand(0);
51824     auto *N1C = dyn_cast<ConstantSDNode>(N1);
51825     auto *N001C = dyn_cast<ConstantSDNode>(TruncExtSrc.getOperand(1));
51826     if (N1C && !N1C->isOpaque() && N001C && !N001C->isOpaque()) {
51827       SDLoc DL(N);
51828       SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT);
51829       SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT);
51830       return DAG.getNode(ISD::XOR, DL, VT, LHS,
51831                          DAG.getNode(ISD::XOR, DL, VT, RHS, N1));
51832     }
51833   }
51834 
51835   if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
51836     return R;
51837 
51838   return combineFneg(N, DAG, DCI, Subtarget);
51839 }
51840 
51841 static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG,
51842                                  TargetLowering::DAGCombinerInfo &DCI,
51843                                  const X86Subtarget &Subtarget) {
51844   SDValue N0 = N->getOperand(0);
51845   EVT VT = N->getValueType(0);
51846 
51847   // Convert a (iX bitreverse(bitcast(vXi1 X))) -> (iX bitcast(shuffle(X)))
51848   if (VT.isInteger() && N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) {
51849     SDValue Src = N0.getOperand(0);
51850     EVT SrcVT = Src.getValueType();
51851     if (SrcVT.isVector() && SrcVT.getScalarType() == MVT::i1 &&
51852         (DCI.isBeforeLegalize() ||
51853          DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) &&
51854         Subtarget.hasSSSE3()) {
51855       unsigned NumElts = SrcVT.getVectorNumElements();
51856       SmallVector<int, 32> ReverseMask(NumElts);
51857       for (unsigned I = 0; I != NumElts; ++I)
51858         ReverseMask[I] = (NumElts - 1) - I;
51859       SDValue Rev =
51860           DAG.getVectorShuffle(SrcVT, SDLoc(N), Src, Src, ReverseMask);
51861       return DAG.getBitcast(VT, Rev);
51862     }
51863   }
51864 
51865   return SDValue();
51866 }
51867 
51868 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
51869                             TargetLowering::DAGCombinerInfo &DCI,
51870                             const X86Subtarget &Subtarget) {
51871   EVT VT = N->getValueType(0);
51872   unsigned NumBits = VT.getSizeInBits();
51873 
51874   // TODO - Constant Folding.
51875 
51876   // Simplify the inputs.
51877   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51878   APInt DemandedMask(APInt::getAllOnes(NumBits));
51879   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
51880     return SDValue(N, 0);
51881 
51882   return SDValue();
51883 }
51884 
51885 static bool isNullFPScalarOrVectorConst(SDValue V) {
51886   return isNullFPConstant(V) || ISD::isBuildVectorAllZeros(V.getNode());
51887 }
51888 
51889 /// If a value is a scalar FP zero or a vector FP zero (potentially including
51890 /// undefined elements), return a zero constant that may be used to fold away
51891 /// that value. In the case of a vector, the returned constant will not contain
51892 /// undefined elements even if the input parameter does. This makes it suitable
51893 /// to be used as a replacement operand with operations (eg, bitwise-and) where
51894 /// an undef should not propagate.
51895 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
51896                                         const X86Subtarget &Subtarget) {
51897   if (!isNullFPScalarOrVectorConst(V))
51898     return SDValue();
51899 
51900   if (V.getValueType().isVector())
51901     return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V));
51902 
51903   return V;
51904 }
51905 
51906 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
51907                                       const X86Subtarget &Subtarget) {
51908   SDValue N0 = N->getOperand(0);
51909   SDValue N1 = N->getOperand(1);
51910   EVT VT = N->getValueType(0);
51911   SDLoc DL(N);
51912 
51913   // Vector types are handled in combineANDXORWithAllOnesIntoANDNP().
51914   if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||
51915         (VT == MVT::f64 && Subtarget.hasSSE2()) ||
51916         (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))
51917     return SDValue();
51918 
51919   auto isAllOnesConstantFP = [](SDValue V) {
51920     if (V.getSimpleValueType().isVector())
51921       return ISD::isBuildVectorAllOnes(V.getNode());
51922     auto *C = dyn_cast<ConstantFPSDNode>(V);
51923     return C && C->getConstantFPValue()->isAllOnesValue();
51924   };
51925 
51926   // fand (fxor X, -1), Y --> fandn X, Y
51927   if (N0.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N0.getOperand(1)))
51928     return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
51929 
51930   // fand X, (fxor Y, -1) --> fandn Y, X
51931   if (N1.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N1.getOperand(1)))
51932     return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
51933 
51934   return SDValue();
51935 }
51936 
51937 /// Do target-specific dag combines on X86ISD::FAND nodes.
51938 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
51939                            const X86Subtarget &Subtarget) {
51940   // FAND(0.0, x) -> 0.0
51941   if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
51942     return V;
51943 
51944   // FAND(x, 0.0) -> 0.0
51945   if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
51946     return V;
51947 
51948   if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
51949     return V;
51950 
51951   return lowerX86FPLogicOp(N, DAG, Subtarget);
51952 }
51953 
51954 /// Do target-specific dag combines on X86ISD::FANDN nodes.
51955 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
51956                             const X86Subtarget &Subtarget) {
51957   // FANDN(0.0, x) -> x
51958   if (isNullFPScalarOrVectorConst(N->getOperand(0)))
51959     return N->getOperand(1);
51960 
51961   // FANDN(x, 0.0) -> 0.0
51962   if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
51963     return V;
51964 
51965   return lowerX86FPLogicOp(N, DAG, Subtarget);
51966 }
51967 
51968 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
51969 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
51970                           TargetLowering::DAGCombinerInfo &DCI,
51971                           const X86Subtarget &Subtarget) {
51972   assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
51973 
51974   // F[X]OR(0.0, x) -> x
51975   if (isNullFPScalarOrVectorConst(N->getOperand(0)))
51976     return N->getOperand(1);
51977 
51978   // F[X]OR(x, 0.0) -> x
51979   if (isNullFPScalarOrVectorConst(N->getOperand(1)))
51980     return N->getOperand(0);
51981 
51982   if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
51983     return NewVal;
51984 
51985   return lowerX86FPLogicOp(N, DAG, Subtarget);
51986 }
51987 
51988 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
51989 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
51990   assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
51991 
51992   // FMIN/FMAX are commutative if no NaNs and no negative zeros are allowed.
51993   if (!DAG.getTarget().Options.NoNaNsFPMath ||
51994       !DAG.getTarget().Options.NoSignedZerosFPMath)
51995     return SDValue();
51996 
51997   // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
51998   // into FMINC and FMAXC, which are Commutative operations.
51999   unsigned NewOp = 0;
52000   switch (N->getOpcode()) {
52001     default: llvm_unreachable("unknown opcode");
52002     case X86ISD::FMIN:  NewOp = X86ISD::FMINC; break;
52003     case X86ISD::FMAX:  NewOp = X86ISD::FMAXC; break;
52004   }
52005 
52006   return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
52007                      N->getOperand(0), N->getOperand(1));
52008 }
52009 
52010 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
52011                                      const X86Subtarget &Subtarget) {
52012   EVT VT = N->getValueType(0);
52013   if (Subtarget.useSoftFloat() || isSoftF16(VT, Subtarget))
52014     return SDValue();
52015 
52016   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52017 
52018   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
52019         (Subtarget.hasSSE2() && VT == MVT::f64) ||
52020         (Subtarget.hasFP16() && VT == MVT::f16) ||
52021         (VT.isVector() && TLI.isTypeLegal(VT))))
52022     return SDValue();
52023 
52024   SDValue Op0 = N->getOperand(0);
52025   SDValue Op1 = N->getOperand(1);
52026   SDLoc DL(N);
52027   auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
52028 
52029   // If we don't have to respect NaN inputs, this is a direct translation to x86
52030   // min/max instructions.
52031   if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs())
52032     return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
52033 
52034   // If one of the operands is known non-NaN use the native min/max instructions
52035   // with the non-NaN input as second operand.
52036   if (DAG.isKnownNeverNaN(Op1))
52037     return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
52038   if (DAG.isKnownNeverNaN(Op0))
52039     return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
52040 
52041   // If we have to respect NaN inputs, this takes at least 3 instructions.
52042   // Favor a library call when operating on a scalar and minimizing code size.
52043   if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
52044     return SDValue();
52045 
52046   EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
52047                                          VT);
52048 
52049   // There are 4 possibilities involving NaN inputs, and these are the required
52050   // outputs:
52051   //                   Op1
52052   //               Num     NaN
52053   //            ----------------
52054   //       Num  |  Max  |  Op0 |
52055   // Op0        ----------------
52056   //       NaN  |  Op1  |  NaN |
52057   //            ----------------
52058   //
52059   // The SSE FP max/min instructions were not designed for this case, but rather
52060   // to implement:
52061   //   Min = Op1 < Op0 ? Op1 : Op0
52062   //   Max = Op1 > Op0 ? Op1 : Op0
52063   //
52064   // So they always return Op0 if either input is a NaN. However, we can still
52065   // use those instructions for fmaxnum by selecting away a NaN input.
52066 
52067   // If either operand is NaN, the 2nd source operand (Op0) is passed through.
52068   SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
52069   SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
52070 
52071   // If Op0 is a NaN, select Op1. Otherwise, select the max. If both operands
52072   // are NaN, the NaN value of Op1 is the result.
52073   return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
52074 }
52075 
52076 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
52077                                    TargetLowering::DAGCombinerInfo &DCI) {
52078   EVT VT = N->getValueType(0);
52079   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52080 
52081   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
52082   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
52083     return SDValue(N, 0);
52084 
52085   // Convert a full vector load into vzload when not all bits are needed.
52086   SDValue In = N->getOperand(0);
52087   MVT InVT = In.getSimpleValueType();
52088   if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
52089       ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
52090     assert(InVT.is128BitVector() && "Expected 128-bit input vector");
52091     LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0));
52092     unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
52093     MVT MemVT = MVT::getIntegerVT(NumBits);
52094     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
52095     if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
52096       SDLoc dl(N);
52097       SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
52098                                     DAG.getBitcast(InVT, VZLoad));
52099       DCI.CombineTo(N, Convert);
52100       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52101       DCI.recursivelyDeleteUnusedNodes(LN);
52102       return SDValue(N, 0);
52103     }
52104   }
52105 
52106   return SDValue();
52107 }
52108 
52109 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
52110                                      TargetLowering::DAGCombinerInfo &DCI) {
52111   bool IsStrict = N->isTargetStrictFPOpcode();
52112   EVT VT = N->getValueType(0);
52113 
52114   // Convert a full vector load into vzload when not all bits are needed.
52115   SDValue In = N->getOperand(IsStrict ? 1 : 0);
52116   MVT InVT = In.getSimpleValueType();
52117   if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
52118       ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
52119     assert(InVT.is128BitVector() && "Expected 128-bit input vector");
52120     LoadSDNode *LN = cast<LoadSDNode>(In);
52121     unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
52122     MVT MemVT = MVT::getFloatingPointVT(NumBits);
52123     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
52124     if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
52125       SDLoc dl(N);
52126       if (IsStrict) {
52127         SDValue Convert =
52128             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
52129                         {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)});
52130         DCI.CombineTo(N, Convert, Convert.getValue(1));
52131       } else {
52132         SDValue Convert =
52133             DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
52134         DCI.CombineTo(N, Convert);
52135       }
52136       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52137       DCI.recursivelyDeleteUnusedNodes(LN);
52138       return SDValue(N, 0);
52139     }
52140   }
52141 
52142   return SDValue();
52143 }
52144 
52145 /// Do target-specific dag combines on X86ISD::ANDNP nodes.
52146 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
52147                             TargetLowering::DAGCombinerInfo &DCI,
52148                             const X86Subtarget &Subtarget) {
52149   SDValue N0 = N->getOperand(0);
52150   SDValue N1 = N->getOperand(1);
52151   MVT VT = N->getSimpleValueType(0);
52152   int NumElts = VT.getVectorNumElements();
52153   unsigned EltSizeInBits = VT.getScalarSizeInBits();
52154   SDLoc DL(N);
52155 
52156   // ANDNP(undef, x) -> 0
52157   // ANDNP(x, undef) -> 0
52158   if (N0.isUndef() || N1.isUndef())
52159     return DAG.getConstant(0, DL, VT);
52160 
52161   // ANDNP(0, x) -> x
52162   if (ISD::isBuildVectorAllZeros(N0.getNode()))
52163     return N1;
52164 
52165   // ANDNP(x, 0) -> 0
52166   if (ISD::isBuildVectorAllZeros(N1.getNode()))
52167     return DAG.getConstant(0, DL, VT);
52168 
52169   // ANDNP(x, -1) -> NOT(x) -> XOR(x, -1)
52170   if (ISD::isBuildVectorAllOnes(N1.getNode()))
52171     return DAG.getNOT(DL, N0, VT);
52172 
52173   // Turn ANDNP back to AND if input is inverted.
52174   if (SDValue Not = IsNOT(N0, DAG))
52175     return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
52176 
52177   // Fold for better commutatvity:
52178   // ANDNP(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
52179   if (N1->hasOneUse())
52180     if (SDValue Not = IsNOT(N1, DAG))
52181       return DAG.getNOT(
52182           DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT);
52183 
52184   // Constant Folding
52185   APInt Undefs0, Undefs1;
52186   SmallVector<APInt> EltBits0, EltBits1;
52187   if (getTargetConstantBitsFromNode(N0, EltSizeInBits, Undefs0, EltBits0)) {
52188     if (getTargetConstantBitsFromNode(N1, EltSizeInBits, Undefs1, EltBits1)) {
52189       SmallVector<APInt> ResultBits;
52190       for (int I = 0; I != NumElts; ++I)
52191         ResultBits.push_back(~EltBits0[I] & EltBits1[I]);
52192       return getConstVector(ResultBits, VT, DAG, DL);
52193     }
52194 
52195     // Constant fold NOT(N0) to allow us to use AND.
52196     // Ensure this is only performed if we can confirm that the bitcasted source
52197     // has oneuse to prevent an infinite loop with canonicalizeBitSelect.
52198     if (N0->hasOneUse()) {
52199       SDValue BC0 = peekThroughOneUseBitcasts(N0);
52200       if (BC0.getOpcode() != ISD::BITCAST) {
52201         for (APInt &Elt : EltBits0)
52202           Elt = ~Elt;
52203         SDValue Not = getConstVector(EltBits0, VT, DAG, DL);
52204         return DAG.getNode(ISD::AND, DL, VT, Not, N1);
52205       }
52206     }
52207   }
52208 
52209   // Attempt to recursively combine a bitmask ANDNP with shuffles.
52210   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
52211     SDValue Op(N, 0);
52212     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
52213       return Res;
52214 
52215     // If either operand is a constant mask, then only the elements that aren't
52216     // zero are actually demanded by the other operand.
52217     auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
52218       APInt UndefElts;
52219       SmallVector<APInt> EltBits;
52220       APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
52221       APInt DemandedElts = APInt::getAllOnes(NumElts);
52222       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
52223                                         EltBits)) {
52224         DemandedBits.clearAllBits();
52225         DemandedElts.clearAllBits();
52226         for (int I = 0; I != NumElts; ++I) {
52227           if (UndefElts[I]) {
52228             // We can't assume an undef src element gives an undef dst - the
52229             // other src might be zero.
52230             DemandedBits.setAllBits();
52231             DemandedElts.setBit(I);
52232           } else if ((Invert && !EltBits[I].isAllOnes()) ||
52233                      (!Invert && !EltBits[I].isZero())) {
52234             DemandedBits |= Invert ? ~EltBits[I] : EltBits[I];
52235             DemandedElts.setBit(I);
52236           }
52237         }
52238       }
52239       return std::make_pair(DemandedBits, DemandedElts);
52240     };
52241     APInt Bits0, Elts0;
52242     APInt Bits1, Elts1;
52243     std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
52244     std::tie(Bits1, Elts1) = GetDemandedMasks(N0, true);
52245 
52246     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52247     if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
52248         TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
52249         TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
52250         TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
52251       if (N->getOpcode() != ISD::DELETED_NODE)
52252         DCI.AddToWorklist(N);
52253       return SDValue(N, 0);
52254     }
52255   }
52256 
52257   return SDValue();
52258 }
52259 
52260 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
52261                          TargetLowering::DAGCombinerInfo &DCI) {
52262   SDValue N1 = N->getOperand(1);
52263 
52264   // BT ignores high bits in the bit index operand.
52265   unsigned BitWidth = N1.getValueSizeInBits();
52266   APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
52267   if (DAG.getTargetLoweringInfo().SimplifyDemandedBits(N1, DemandedMask, DCI)) {
52268     if (N->getOpcode() != ISD::DELETED_NODE)
52269       DCI.AddToWorklist(N);
52270     return SDValue(N, 0);
52271   }
52272 
52273   return SDValue();
52274 }
52275 
52276 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
52277                                TargetLowering::DAGCombinerInfo &DCI) {
52278   bool IsStrict = N->getOpcode() == X86ISD::STRICT_CVTPH2PS;
52279   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
52280 
52281   if (N->getValueType(0) == MVT::v4f32 && Src.getValueType() == MVT::v8i16) {
52282     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52283     APInt DemandedElts = APInt::getLowBitsSet(8, 4);
52284     if (TLI.SimplifyDemandedVectorElts(Src, DemandedElts, DCI)) {
52285       if (N->getOpcode() != ISD::DELETED_NODE)
52286         DCI.AddToWorklist(N);
52287       return SDValue(N, 0);
52288     }
52289 
52290     // Convert a full vector load into vzload when not all bits are needed.
52291     if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
52292       LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(IsStrict ? 1 : 0));
52293       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
52294         SDLoc dl(N);
52295         if (IsStrict) {
52296           SDValue Convert = DAG.getNode(
52297               N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
52298               {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)});
52299           DCI.CombineTo(N, Convert, Convert.getValue(1));
52300         } else {
52301           SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
52302                                         DAG.getBitcast(MVT::v8i16, VZLoad));
52303           DCI.CombineTo(N, Convert);
52304         }
52305 
52306         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52307         DCI.recursivelyDeleteUnusedNodes(LN);
52308         return SDValue(N, 0);
52309       }
52310     }
52311   }
52312 
52313   return SDValue();
52314 }
52315 
52316 // Try to combine sext_in_reg of a cmov of constants by extending the constants.
52317 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
52318   assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
52319 
52320   EVT DstVT = N->getValueType(0);
52321 
52322   SDValue N0 = N->getOperand(0);
52323   SDValue N1 = N->getOperand(1);
52324   EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
52325 
52326   if (ExtraVT != MVT::i8 && ExtraVT != MVT::i16)
52327     return SDValue();
52328 
52329   // Look through single use any_extends / truncs.
52330   SDValue IntermediateBitwidthOp;
52331   if ((N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::TRUNCATE) &&
52332       N0.hasOneUse()) {
52333     IntermediateBitwidthOp = N0;
52334     N0 = N0.getOperand(0);
52335   }
52336 
52337   // See if we have a single use cmov.
52338   if (N0.getOpcode() != X86ISD::CMOV || !N0.hasOneUse())
52339     return SDValue();
52340 
52341   SDValue CMovOp0 = N0.getOperand(0);
52342   SDValue CMovOp1 = N0.getOperand(1);
52343 
52344   // Make sure both operands are constants.
52345   if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
52346       !isa<ConstantSDNode>(CMovOp1.getNode()))
52347     return SDValue();
52348 
52349   SDLoc DL(N);
52350 
52351   // If we looked through an any_extend/trunc above, add one to the constants.
52352   if (IntermediateBitwidthOp) {
52353     unsigned IntermediateOpc = IntermediateBitwidthOp.getOpcode();
52354     CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
52355     CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
52356   }
52357 
52358   CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
52359   CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
52360 
52361   EVT CMovVT = DstVT;
52362   // We do not want i16 CMOV's. Promote to i32 and truncate afterwards.
52363   if (DstVT == MVT::i16) {
52364     CMovVT = MVT::i32;
52365     CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
52366     CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
52367   }
52368 
52369   SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
52370                              N0.getOperand(2), N0.getOperand(3));
52371 
52372   if (CMovVT != DstVT)
52373     CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
52374 
52375   return CMov;
52376 }
52377 
52378 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
52379                                       const X86Subtarget &Subtarget) {
52380   assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
52381 
52382   if (SDValue V = combineSextInRegCmov(N, DAG))
52383     return V;
52384 
52385   EVT VT = N->getValueType(0);
52386   SDValue N0 = N->getOperand(0);
52387   SDValue N1 = N->getOperand(1);
52388   EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
52389   SDLoc dl(N);
52390 
52391   // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
52392   // both SSE and AVX2 since there is no sign-extended shift right
52393   // operation on a vector with 64-bit elements.
52394   //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
52395   // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
52396   if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
52397                            N0.getOpcode() == ISD::SIGN_EXTEND)) {
52398     SDValue N00 = N0.getOperand(0);
52399 
52400     // EXTLOAD has a better solution on AVX2,
52401     // it may be replaced with X86ISD::VSEXT node.
52402     if (N00.getOpcode() == ISD::LOAD && Subtarget.hasInt256())
52403       if (!ISD::isNormalLoad(N00.getNode()))
52404         return SDValue();
52405 
52406     // Attempt to promote any comparison mask ops before moving the
52407     // SIGN_EXTEND_INREG in the way.
52408     if (SDValue Promote = PromoteMaskArithmetic(N0.getNode(), DAG, Subtarget))
52409       return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1);
52410 
52411     if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
52412       SDValue Tmp =
52413           DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1);
52414       return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
52415     }
52416   }
52417   return SDValue();
52418 }
52419 
52420 /// sext(add_nsw(x, C)) --> add(sext(x), C_sext)
52421 /// zext(add_nuw(x, C)) --> add(zext(x), C_zext)
52422 /// Promoting a sign/zero extension ahead of a no overflow 'add' exposes
52423 /// opportunities to combine math ops, use an LEA, or use a complex addressing
52424 /// mode. This can eliminate extend, add, and shift instructions.
52425 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
52426                                    const X86Subtarget &Subtarget) {
52427   if (Ext->getOpcode() != ISD::SIGN_EXTEND &&
52428       Ext->getOpcode() != ISD::ZERO_EXTEND)
52429     return SDValue();
52430 
52431   // TODO: This should be valid for other integer types.
52432   EVT VT = Ext->getValueType(0);
52433   if (VT != MVT::i64)
52434     return SDValue();
52435 
52436   SDValue Add = Ext->getOperand(0);
52437   if (Add.getOpcode() != ISD::ADD)
52438     return SDValue();
52439 
52440   SDValue AddOp0 = Add.getOperand(0);
52441   SDValue AddOp1 = Add.getOperand(1);
52442   bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND;
52443   bool NSW = Add->getFlags().hasNoSignedWrap();
52444   bool NUW = Add->getFlags().hasNoUnsignedWrap();
52445   NSW = NSW || (Sext && DAG.willNotOverflowAdd(true, AddOp0, AddOp1));
52446   NUW = NUW || (!Sext && DAG.willNotOverflowAdd(false, AddOp0, AddOp1));
52447 
52448   // We need an 'add nsw' feeding into the 'sext' or 'add nuw' feeding
52449   // into the 'zext'
52450   if ((Sext && !NSW) || (!Sext && !NUW))
52451     return SDValue();
52452 
52453   // Having a constant operand to the 'add' ensures that we are not increasing
52454   // the instruction count because the constant is extended for free below.
52455   // A constant operand can also become the displacement field of an LEA.
52456   auto *AddOp1C = dyn_cast<ConstantSDNode>(AddOp1);
52457   if (!AddOp1C)
52458     return SDValue();
52459 
52460   // Don't make the 'add' bigger if there's no hope of combining it with some
52461   // other 'add' or 'shl' instruction.
52462   // TODO: It may be profitable to generate simpler LEA instructions in place
52463   // of single 'add' instructions, but the cost model for selecting an LEA
52464   // currently has a high threshold.
52465   bool HasLEAPotential = false;
52466   for (auto *User : Ext->uses()) {
52467     if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) {
52468       HasLEAPotential = true;
52469       break;
52470     }
52471   }
52472   if (!HasLEAPotential)
52473     return SDValue();
52474 
52475   // Everything looks good, so pull the '{s|z}ext' ahead of the 'add'.
52476   int64_t AddC = Sext ? AddOp1C->getSExtValue() : AddOp1C->getZExtValue();
52477   SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
52478   SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT);
52479 
52480   // The wider add is guaranteed to not wrap because both operands are
52481   // sign-extended.
52482   SDNodeFlags Flags;
52483   Flags.setNoSignedWrap(NSW);
52484   Flags.setNoUnsignedWrap(NUW);
52485   return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
52486 }
52487 
52488 // If we face {ANY,SIGN,ZERO}_EXTEND that is applied to a CMOV with constant
52489 // operands and the result of CMOV is not used anywhere else - promote CMOV
52490 // itself instead of promoting its result. This could be beneficial, because:
52491 //     1) X86TargetLowering::EmitLoweredSelect later can do merging of two
52492 //        (or more) pseudo-CMOVs only when they go one-after-another and
52493 //        getting rid of result extension code after CMOV will help that.
52494 //     2) Promotion of constant CMOV arguments is free, hence the
52495 //        {ANY,SIGN,ZERO}_EXTEND will just be deleted.
52496 //     3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
52497 //        promotion is also good in terms of code-size.
52498 //        (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
52499 //         promotion).
52500 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
52501   SDValue CMovN = Extend->getOperand(0);
52502   if (CMovN.getOpcode() != X86ISD::CMOV || !CMovN.hasOneUse())
52503     return SDValue();
52504 
52505   EVT TargetVT = Extend->getValueType(0);
52506   unsigned ExtendOpcode = Extend->getOpcode();
52507   SDLoc DL(Extend);
52508 
52509   EVT VT = CMovN.getValueType();
52510   SDValue CMovOp0 = CMovN.getOperand(0);
52511   SDValue CMovOp1 = CMovN.getOperand(1);
52512 
52513   if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
52514       !isa<ConstantSDNode>(CMovOp1.getNode()))
52515     return SDValue();
52516 
52517   // Only extend to i32 or i64.
52518   if (TargetVT != MVT::i32 && TargetVT != MVT::i64)
52519     return SDValue();
52520 
52521   // Only extend from i16 unless its a sign_extend from i32. Zext/aext from i32
52522   // are free.
52523   if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))
52524     return SDValue();
52525 
52526   // If this a zero extend to i64, we should only extend to i32 and use a free
52527   // zero extend to finish.
52528   EVT ExtendVT = TargetVT;
52529   if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND)
52530     ExtendVT = MVT::i32;
52531 
52532   CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
52533   CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
52534 
52535   SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
52536                             CMovN.getOperand(2), CMovN.getOperand(3));
52537 
52538   // Finish extending if needed.
52539   if (ExtendVT != TargetVT)
52540     Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
52541 
52542   return Res;
52543 }
52544 
52545 // Attempt to combine a (sext/zext (setcc)) to a setcc with a xmm/ymm/zmm
52546 // result type.
52547 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
52548                                const X86Subtarget &Subtarget) {
52549   SDValue N0 = N->getOperand(0);
52550   EVT VT = N->getValueType(0);
52551   SDLoc dl(N);
52552 
52553   // Only do this combine with AVX512 for vector extends.
52554   if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC)
52555     return SDValue();
52556 
52557   // Only combine legal element types.
52558   EVT SVT = VT.getVectorElementType();
52559   if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
52560       SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
52561     return SDValue();
52562 
52563   // We don't have CMPP Instruction for vxf16
52564   if (N0.getOperand(0).getValueType().getVectorElementType() == MVT::f16)
52565     return SDValue();
52566   // We can only do this if the vector size in 256 bits or less.
52567   unsigned Size = VT.getSizeInBits();
52568   if (Size > 256 && Subtarget.useAVX512Regs())
52569     return SDValue();
52570 
52571   // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
52572   // that's the only integer compares with we have.
52573   ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
52574   if (ISD::isUnsignedIntSetCC(CC))
52575     return SDValue();
52576 
52577   // Only do this combine if the extension will be fully consumed by the setcc.
52578   EVT N00VT = N0.getOperand(0).getValueType();
52579   EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
52580   if (Size != MatchingVecType.getSizeInBits())
52581     return SDValue();
52582 
52583   SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
52584 
52585   if (N->getOpcode() == ISD::ZERO_EXTEND)
52586     Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType());
52587 
52588   return Res;
52589 }
52590 
52591 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
52592                            TargetLowering::DAGCombinerInfo &DCI,
52593                            const X86Subtarget &Subtarget) {
52594   SDValue N0 = N->getOperand(0);
52595   EVT VT = N->getValueType(0);
52596   SDLoc DL(N);
52597 
52598   // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
52599   if (!DCI.isBeforeLegalizeOps() &&
52600       N0.getOpcode() == X86ISD::SETCC_CARRY) {
52601     SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
52602                                  N0->getOperand(1));
52603     bool ReplaceOtherUses = !N0.hasOneUse();
52604     DCI.CombineTo(N, Setcc);
52605     // Replace other uses with a truncate of the widened setcc_carry.
52606     if (ReplaceOtherUses) {
52607       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
52608                                   N0.getValueType(), Setcc);
52609       DCI.CombineTo(N0.getNode(), Trunc);
52610     }
52611 
52612     return SDValue(N, 0);
52613   }
52614 
52615   if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
52616     return NewCMov;
52617 
52618   if (!DCI.isBeforeLegalizeOps())
52619     return SDValue();
52620 
52621   if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
52622     return V;
52623 
52624   if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0,
52625                                                  DAG, DCI, Subtarget))
52626     return V;
52627 
52628   if (VT.isVector()) {
52629     if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
52630       return R;
52631 
52632     if (N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
52633       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
52634   }
52635 
52636   if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
52637     return NewAdd;
52638 
52639   return SDValue();
52640 }
52641 
52642 // Inverting a constant vector is profitable if it can be eliminated and the
52643 // inverted vector is already present in DAG. Otherwise, it will be loaded
52644 // anyway.
52645 //
52646 // We determine which of the values can be completely eliminated and invert it.
52647 // If both are eliminable, select a vector with the first negative element.
52648 static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) {
52649   assert(ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()) &&
52650          "ConstantFP build vector expected");
52651   // Check if we can eliminate V. We assume if a value is only used in FMAs, we
52652   // can eliminate it. Since this function is invoked for each FMA with this
52653   // vector.
52654   auto IsNotFMA = [](SDNode *Use) {
52655     return Use->getOpcode() != ISD::FMA && Use->getOpcode() != ISD::STRICT_FMA;
52656   };
52657   if (llvm::any_of(V->uses(), IsNotFMA))
52658     return SDValue();
52659 
52660   SmallVector<SDValue, 8> Ops;
52661   EVT VT = V.getValueType();
52662   EVT EltVT = VT.getVectorElementType();
52663   for (auto Op : V->op_values()) {
52664     if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
52665       Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT));
52666     } else {
52667       assert(Op.isUndef());
52668       Ops.push_back(DAG.getUNDEF(EltVT));
52669     }
52670   }
52671 
52672   SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops);
52673   if (!NV)
52674     return SDValue();
52675 
52676   // If an inverted version cannot be eliminated, choose it instead of the
52677   // original version.
52678   if (llvm::any_of(NV->uses(), IsNotFMA))
52679     return SDValue(NV, 0);
52680 
52681   // If the inverted version also can be eliminated, we have to consistently
52682   // prefer one of the values. We prefer a constant with a negative value on
52683   // the first place.
52684   // N.B. We need to skip undefs that may precede a value.
52685   for (auto op : V->op_values()) {
52686     if (auto *Cst = dyn_cast<ConstantFPSDNode>(op)) {
52687       if (Cst->isNegative())
52688         return SDValue();
52689       break;
52690     }
52691   }
52692   return SDValue(NV, 0);
52693 }
52694 
52695 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
52696                           TargetLowering::DAGCombinerInfo &DCI,
52697                           const X86Subtarget &Subtarget) {
52698   SDLoc dl(N);
52699   EVT VT = N->getValueType(0);
52700   bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode();
52701 
52702   // Let legalize expand this if it isn't a legal type yet.
52703   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52704   if (!TLI.isTypeLegal(VT))
52705     return SDValue();
52706 
52707   SDValue A = N->getOperand(IsStrict ? 1 : 0);
52708   SDValue B = N->getOperand(IsStrict ? 2 : 1);
52709   SDValue C = N->getOperand(IsStrict ? 3 : 2);
52710 
52711   // If the operation allows fast-math and the target does not support FMA,
52712   // split this into mul+add to avoid libcall(s).
52713   SDNodeFlags Flags = N->getFlags();
52714   if (!IsStrict && Flags.hasAllowReassociation() &&
52715       TLI.isOperationExpand(ISD::FMA, VT)) {
52716     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags);
52717     return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags);
52718   }
52719 
52720   EVT ScalarVT = VT.getScalarType();
52721   if (((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
52722        !Subtarget.hasAnyFMA()) &&
52723       !(ScalarVT == MVT::f16 && Subtarget.hasFP16()))
52724     return SDValue();
52725 
52726   auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
52727     bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
52728     bool LegalOperations = !DCI.isBeforeLegalizeOps();
52729     if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
52730                                                        CodeSize)) {
52731       V = NegV;
52732       return true;
52733     }
52734     // Look through extract_vector_elts. If it comes from an FNEG, create a
52735     // new extract from the FNEG input.
52736     if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
52737         isNullConstant(V.getOperand(1))) {
52738       SDValue Vec = V.getOperand(0);
52739       if (SDValue NegV = TLI.getCheaperNegatedExpression(
52740               Vec, DAG, LegalOperations, CodeSize)) {
52741         V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(),
52742                         NegV, V.getOperand(1));
52743         return true;
52744       }
52745     }
52746     // Lookup if there is an inverted version of constant vector V in DAG.
52747     if (ISD::isBuildVectorOfConstantFPSDNodes(V.getNode())) {
52748       if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) {
52749         V = NegV;
52750         return true;
52751       }
52752     }
52753     return false;
52754   };
52755 
52756   // Do not convert the passthru input of scalar intrinsics.
52757   // FIXME: We could allow negations of the lower element only.
52758   bool NegA = invertIfNegative(A);
52759   bool NegB = invertIfNegative(B);
52760   bool NegC = invertIfNegative(C);
52761 
52762   if (!NegA && !NegB && !NegC)
52763     return SDValue();
52764 
52765   unsigned NewOpcode =
52766       negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC, false);
52767 
52768   // Propagate fast-math-flags to new FMA node.
52769   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
52770   if (IsStrict) {
52771     assert(N->getNumOperands() == 4 && "Shouldn't be greater than 4");
52772     return DAG.getNode(NewOpcode, dl, {VT, MVT::Other},
52773                        {N->getOperand(0), A, B, C});
52774   } else {
52775     if (N->getNumOperands() == 4)
52776       return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));
52777     return DAG.getNode(NewOpcode, dl, VT, A, B, C);
52778   }
52779 }
52780 
52781 // Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
52782 // Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
52783 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
52784                                TargetLowering::DAGCombinerInfo &DCI) {
52785   SDLoc dl(N);
52786   EVT VT = N->getValueType(0);
52787   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52788   bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
52789   bool LegalOperations = !DCI.isBeforeLegalizeOps();
52790 
52791   SDValue N2 = N->getOperand(2);
52792 
52793   SDValue NegN2 =
52794       TLI.getCheaperNegatedExpression(N2, DAG, LegalOperations, CodeSize);
52795   if (!NegN2)
52796     return SDValue();
52797   unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), false, true, false);
52798 
52799   if (N->getNumOperands() == 4)
52800     return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
52801                        NegN2, N->getOperand(3));
52802   return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
52803                      NegN2);
52804 }
52805 
52806 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
52807                            TargetLowering::DAGCombinerInfo &DCI,
52808                            const X86Subtarget &Subtarget) {
52809   SDLoc dl(N);
52810   SDValue N0 = N->getOperand(0);
52811   EVT VT = N->getValueType(0);
52812 
52813   // (i32 (aext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
52814   // FIXME: Is this needed? We don't seem to have any tests for it.
52815   if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND &&
52816       N0.getOpcode() == X86ISD::SETCC_CARRY) {
52817     SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
52818                                  N0->getOperand(1));
52819     bool ReplaceOtherUses = !N0.hasOneUse();
52820     DCI.CombineTo(N, Setcc);
52821     // Replace other uses with a truncate of the widened setcc_carry.
52822     if (ReplaceOtherUses) {
52823       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
52824                                   N0.getValueType(), Setcc);
52825       DCI.CombineTo(N0.getNode(), Trunc);
52826     }
52827 
52828     return SDValue(N, 0);
52829   }
52830 
52831   if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
52832     return NewCMov;
52833 
52834   if (DCI.isBeforeLegalizeOps())
52835     if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
52836       return V;
52837 
52838   if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0,
52839                                                  DAG, DCI, Subtarget))
52840     return V;
52841 
52842   if (VT.isVector())
52843     if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
52844       return R;
52845 
52846   if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
52847     return NewAdd;
52848 
52849   if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
52850     return R;
52851 
52852   // TODO: Combine with any target/faux shuffle.
52853   if (N0.getOpcode() == X86ISD::PACKUS && N0.getValueSizeInBits() == 128 &&
52854       VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) {
52855     SDValue N00 = N0.getOperand(0);
52856     SDValue N01 = N0.getOperand(1);
52857     unsigned NumSrcEltBits = N00.getScalarValueSizeInBits();
52858     APInt ZeroMask = APInt::getHighBitsSet(NumSrcEltBits, NumSrcEltBits / 2);
52859     if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) &&
52860         (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) {
52861       return concatSubVectors(N00, N01, DAG, dl);
52862     }
52863   }
52864 
52865   return SDValue();
52866 }
52867 
52868 /// If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just
52869 /// pre-promote its result type since vXi1 vectors don't get promoted
52870 /// during type legalization.
52871 static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS,
52872                                         SDValue RHS, ISD::CondCode CC,
52873                                         const SDLoc &DL, SelectionDAG &DAG,
52874                                         const X86Subtarget &Subtarget) {
52875   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&
52876       VT.getVectorElementType() == MVT::i1 &&
52877       (OpVT.getVectorElementType() == MVT::i8 ||
52878        OpVT.getVectorElementType() == MVT::i16)) {
52879     SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
52880     return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
52881   }
52882   return SDValue();
52883 }
52884 
52885 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
52886                             TargetLowering::DAGCombinerInfo &DCI,
52887                             const X86Subtarget &Subtarget) {
52888   const ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
52889   const SDValue LHS = N->getOperand(0);
52890   const SDValue RHS = N->getOperand(1);
52891   EVT VT = N->getValueType(0);
52892   EVT OpVT = LHS.getValueType();
52893   SDLoc DL(N);
52894 
52895   if (CC == ISD::SETNE || CC == ISD::SETEQ) {
52896     if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
52897                                                     Subtarget))
52898       return V;
52899 
52900     if (VT == MVT::i1) {
52901       X86::CondCode X86CC;
52902       if (SDValue V =
52903               MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
52904         return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
52905     }
52906 
52907     if (OpVT.isScalarInteger()) {
52908       // cmpeq(or(X,Y),X) --> cmpeq(and(~X,Y),0)
52909       // cmpne(or(X,Y),X) --> cmpne(and(~X,Y),0)
52910       auto MatchOrCmpEq = [&](SDValue N0, SDValue N1) {
52911         if (N0.getOpcode() == ISD::OR && N0->hasOneUse()) {
52912           if (N0.getOperand(0) == N1)
52913             return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
52914                                N0.getOperand(1));
52915           if (N0.getOperand(1) == N1)
52916             return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
52917                                N0.getOperand(0));
52918         }
52919         return SDValue();
52920       };
52921       if (SDValue AndN = MatchOrCmpEq(LHS, RHS))
52922         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52923       if (SDValue AndN = MatchOrCmpEq(RHS, LHS))
52924         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52925 
52926       // cmpeq(and(X,Y),Y) --> cmpeq(and(~X,Y),0)
52927       // cmpne(and(X,Y),Y) --> cmpne(and(~X,Y),0)
52928       auto MatchAndCmpEq = [&](SDValue N0, SDValue N1) {
52929         if (N0.getOpcode() == ISD::AND && N0->hasOneUse()) {
52930           if (N0.getOperand(0) == N1)
52931             return DAG.getNode(ISD::AND, DL, OpVT, N1,
52932                                DAG.getNOT(DL, N0.getOperand(1), OpVT));
52933           if (N0.getOperand(1) == N1)
52934             return DAG.getNode(ISD::AND, DL, OpVT, N1,
52935                                DAG.getNOT(DL, N0.getOperand(0), OpVT));
52936         }
52937         return SDValue();
52938       };
52939       if (SDValue AndN = MatchAndCmpEq(LHS, RHS))
52940         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52941       if (SDValue AndN = MatchAndCmpEq(RHS, LHS))
52942         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52943 
52944       // cmpeq(trunc(x),C) --> cmpeq(x,C)
52945       // cmpne(trunc(x),C) --> cmpne(x,C)
52946       // iff x upper bits are zero.
52947       if (LHS.getOpcode() == ISD::TRUNCATE &&
52948           LHS.getOperand(0).getScalarValueSizeInBits() >= 32 &&
52949           isa<ConstantSDNode>(RHS) && !DCI.isBeforeLegalize()) {
52950         EVT SrcVT = LHS.getOperand(0).getValueType();
52951         APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
52952                                                 OpVT.getScalarSizeInBits());
52953         const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52954         auto *C = cast<ConstantSDNode>(RHS);
52955         if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) &&
52956             TLI.isTypeLegal(LHS.getOperand(0).getValueType()))
52957           return DAG.getSetCC(DL, VT, LHS.getOperand(0),
52958                               DAG.getConstant(C->getAPIntValue().zextOrTrunc(
52959                                                   SrcVT.getScalarSizeInBits()),
52960                                               DL, SrcVT),
52961                               CC);
52962       }
52963 
52964       // With C as a power of 2 and C != 0 and C != INT_MIN:
52965       //    icmp eq Abs(X) C ->
52966       //        (icmp eq A, C) | (icmp eq A, -C)
52967       //    icmp ne Abs(X) C ->
52968       //        (icmp ne A, C) & (icmp ne A, -C)
52969       // Both of these patterns can be better optimized in
52970       // DAGCombiner::foldAndOrOfSETCC. Note this only applies for scalar
52971       // integers which is checked above.
52972       if (LHS.getOpcode() == ISD::ABS && LHS.hasOneUse()) {
52973         if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
52974           const APInt &CInt = C->getAPIntValue();
52975           // We can better optimize this case in DAGCombiner::foldAndOrOfSETCC.
52976           if (CInt.isPowerOf2() && !CInt.isMinSignedValue()) {
52977             SDValue BaseOp = LHS.getOperand(0);
52978             SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC);
52979             SDValue SETCC1 = DAG.getSetCC(
52980                 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC);
52981             return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT,
52982                                SETCC0, SETCC1);
52983           }
52984         }
52985       }
52986     }
52987   }
52988 
52989   if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
52990       (CC == ISD::SETNE || CC == ISD::SETEQ || ISD::isSignedIntSetCC(CC))) {
52991     // Using temporaries to avoid messing up operand ordering for later
52992     // transformations if this doesn't work.
52993     SDValue Op0 = LHS;
52994     SDValue Op1 = RHS;
52995     ISD::CondCode TmpCC = CC;
52996     // Put build_vector on the right.
52997     if (Op0.getOpcode() == ISD::BUILD_VECTOR) {
52998       std::swap(Op0, Op1);
52999       TmpCC = ISD::getSetCCSwappedOperands(TmpCC);
53000     }
53001 
53002     bool IsSEXT0 =
53003         (Op0.getOpcode() == ISD::SIGN_EXTEND) &&
53004         (Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1);
53005     bool IsVZero1 = ISD::isBuildVectorAllZeros(Op1.getNode());
53006 
53007     if (IsSEXT0 && IsVZero1) {
53008       assert(VT == Op0.getOperand(0).getValueType() &&
53009              "Unexpected operand type");
53010       if (TmpCC == ISD::SETGT)
53011         return DAG.getConstant(0, DL, VT);
53012       if (TmpCC == ISD::SETLE)
53013         return DAG.getConstant(1, DL, VT);
53014       if (TmpCC == ISD::SETEQ || TmpCC == ISD::SETGE)
53015         return DAG.getNOT(DL, Op0.getOperand(0), VT);
53016 
53017       assert((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) &&
53018              "Unexpected condition code!");
53019       return Op0.getOperand(0);
53020     }
53021   }
53022 
53023   // Try and make unsigned vector comparison signed. On pre AVX512 targets there
53024   // only are unsigned comparisons (`PCMPGT`) and on AVX512 its often better to
53025   // use `PCMPGT` if the result is mean to stay in a vector (and if its going to
53026   // a mask, there are signed AVX512 comparisons).
53027   if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) {
53028     bool CanMakeSigned = false;
53029     if (ISD::isUnsignedIntSetCC(CC)) {
53030       KnownBits CmpKnown =
53031           DAG.computeKnownBits(LHS).intersectWith(DAG.computeKnownBits(RHS));
53032       // If we know LHS/RHS share the same sign bit at each element we can
53033       // make this signed.
53034       // NOTE: `computeKnownBits` on a vector type aggregates common bits
53035       // across all lanes. So a pattern where the sign varies from lane to
53036       // lane, but at each lane Sign(LHS) is known to equal Sign(RHS), will be
53037       // missed. We could get around this by demanding each lane
53038       // independently, but this isn't the most important optimization and
53039       // that may eat into compile time.
53040       CanMakeSigned =
53041           CmpKnown.Zero.isSignBitSet() || CmpKnown.One.isSignBitSet();
53042     }
53043     if (CanMakeSigned || ISD::isSignedIntSetCC(CC)) {
53044       SDValue LHSOut = LHS;
53045       SDValue RHSOut = RHS;
53046       ISD::CondCode NewCC = CC;
53047       switch (CC) {
53048       case ISD::SETGE:
53049       case ISD::SETUGE:
53050         if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
53051                                                   /*NSW*/ true))
53052           LHSOut = NewLHS;
53053         else if (SDValue NewRHS = incDecVectorConstant(
53054                      RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
53055           RHSOut = NewRHS;
53056         else
53057           break;
53058 
53059         [[fallthrough]];
53060       case ISD::SETUGT:
53061         NewCC = ISD::SETGT;
53062         break;
53063 
53064       case ISD::SETLE:
53065       case ISD::SETULE:
53066         if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
53067                                                   /*NSW*/ true))
53068           LHSOut = NewLHS;
53069         else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
53070                                                        /*NSW*/ true))
53071           RHSOut = NewRHS;
53072         else
53073           break;
53074 
53075         [[fallthrough]];
53076       case ISD::SETULT:
53077         // Will be swapped to SETGT in LowerVSETCC*.
53078         NewCC = ISD::SETLT;
53079         break;
53080       default:
53081         break;
53082       }
53083       if (NewCC != CC) {
53084         if (SDValue R = truncateAVX512SetCCNoBWI(VT, OpVT, LHSOut, RHSOut,
53085                                                  NewCC, DL, DAG, Subtarget))
53086           return R;
53087         return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC);
53088       }
53089     }
53090   }
53091 
53092   if (SDValue R =
53093           truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget))
53094     return R;
53095 
53096   // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early
53097   // to avoid scalarization via legalization because v4i32 is not a legal type.
53098   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&
53099       LHS.getValueType() == MVT::v4f32)
53100     return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
53101 
53102   // X pred 0.0 --> X pred -X
53103   // If the negation of X already exists, use it in the comparison. This removes
53104   // the need to materialize 0.0 and allows matching to SSE's MIN/MAX
53105   // instructions in patterns with a 'select' node.
53106   if (isNullFPScalarOrVectorConst(RHS)) {
53107     SDVTList FNegVT = DAG.getVTList(OpVT);
53108     if (SDNode *FNeg = DAG.getNodeIfExists(ISD::FNEG, FNegVT, {LHS}))
53109       return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC);
53110   }
53111 
53112   return SDValue();
53113 }
53114 
53115 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
53116                              TargetLowering::DAGCombinerInfo &DCI,
53117                              const X86Subtarget &Subtarget) {
53118   SDValue Src = N->getOperand(0);
53119   MVT SrcVT = Src.getSimpleValueType();
53120   MVT VT = N->getSimpleValueType(0);
53121   unsigned NumBits = VT.getScalarSizeInBits();
53122   unsigned NumElts = SrcVT.getVectorNumElements();
53123   unsigned NumBitsPerElt = SrcVT.getScalarSizeInBits();
53124   assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types");
53125 
53126   // Perform constant folding.
53127   APInt UndefElts;
53128   SmallVector<APInt, 32> EltBits;
53129   if (getTargetConstantBitsFromNode(Src, NumBitsPerElt, UndefElts, EltBits)) {
53130     APInt Imm(32, 0);
53131     for (unsigned Idx = 0; Idx != NumElts; ++Idx)
53132       if (!UndefElts[Idx] && EltBits[Idx].isNegative())
53133         Imm.setBit(Idx);
53134 
53135     return DAG.getConstant(Imm, SDLoc(N), VT);
53136   }
53137 
53138   // Look through int->fp bitcasts that don't change the element width.
53139   unsigned EltWidth = SrcVT.getScalarSizeInBits();
53140   if (Subtarget.hasSSE2() && Src.getOpcode() == ISD::BITCAST &&
53141       Src.getOperand(0).getScalarValueSizeInBits() == EltWidth)
53142     return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
53143 
53144   // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results
53145   // with scalar comparisons.
53146   if (SDValue NotSrc = IsNOT(Src, DAG)) {
53147     SDLoc DL(N);
53148     APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
53149     NotSrc = DAG.getBitcast(SrcVT, NotSrc);
53150     return DAG.getNode(ISD::XOR, DL, VT,
53151                        DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
53152                        DAG.getConstant(NotMask, DL, VT));
53153   }
53154 
53155   // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk
53156   // results with scalar comparisons.
53157   if (Src.getOpcode() == X86ISD::PCMPGT &&
53158       ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) {
53159     SDLoc DL(N);
53160     APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
53161     return DAG.getNode(ISD::XOR, DL, VT,
53162                        DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)),
53163                        DAG.getConstant(NotMask, DL, VT));
53164   }
53165 
53166   // Fold movmsk(icmp_eq(and(x,c1),c1)) -> movmsk(shl(x,c2))
53167   // Fold movmsk(icmp_eq(and(x,c1),0)) -> movmsk(not(shl(x,c2)))
53168   // iff pow2splat(c1).
53169   // Use KnownBits to determine if only a single bit is non-zero
53170   // in each element (pow2 or zero), and shift that bit to the msb.
53171   if (Src.getOpcode() == X86ISD::PCMPEQ) {
53172     KnownBits KnownLHS = DAG.computeKnownBits(Src.getOperand(0));
53173     KnownBits KnownRHS = DAG.computeKnownBits(Src.getOperand(1));
53174     unsigned ShiftAmt = KnownLHS.countMinLeadingZeros();
53175     if (KnownLHS.countMaxPopulation() == 1 &&
53176         (KnownRHS.isZero() || (KnownRHS.countMaxPopulation() == 1 &&
53177                                ShiftAmt == KnownRHS.countMinLeadingZeros()))) {
53178       SDLoc DL(N);
53179       MVT ShiftVT = SrcVT;
53180       SDValue ShiftLHS = Src.getOperand(0);
53181       SDValue ShiftRHS = Src.getOperand(1);
53182       if (ShiftVT.getScalarType() == MVT::i8) {
53183         // vXi8 shifts - we only care about the signbit so can use PSLLW.
53184         ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
53185         ShiftLHS = DAG.getBitcast(ShiftVT, ShiftLHS);
53186         ShiftRHS = DAG.getBitcast(ShiftVT, ShiftRHS);
53187       }
53188       ShiftLHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
53189                                             ShiftLHS, ShiftAmt, DAG);
53190       ShiftRHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
53191                                             ShiftRHS, ShiftAmt, DAG);
53192       ShiftLHS = DAG.getBitcast(SrcVT, ShiftLHS);
53193       ShiftRHS = DAG.getBitcast(SrcVT, ShiftRHS);
53194       SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS);
53195       return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT));
53196     }
53197   }
53198 
53199   // Fold movmsk(logic(X,C)) -> logic(movmsk(X),C)
53200   if (N->isOnlyUserOf(Src.getNode())) {
53201     SDValue SrcBC = peekThroughOneUseBitcasts(Src);
53202     if (ISD::isBitwiseLogicOp(SrcBC.getOpcode())) {
53203       APInt UndefElts;
53204       SmallVector<APInt, 32> EltBits;
53205       if (getTargetConstantBitsFromNode(SrcBC.getOperand(1), NumBitsPerElt,
53206                                         UndefElts, EltBits)) {
53207         APInt Mask = APInt::getZero(NumBits);
53208         for (unsigned Idx = 0; Idx != NumElts; ++Idx) {
53209           if (!UndefElts[Idx] && EltBits[Idx].isNegative())
53210             Mask.setBit(Idx);
53211         }
53212         SDLoc DL(N);
53213         SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0));
53214         SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc);
53215         return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk,
53216                            DAG.getConstant(Mask, DL, VT));
53217       }
53218     }
53219   }
53220 
53221   // Simplify the inputs.
53222   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53223   APInt DemandedMask(APInt::getAllOnes(NumBits));
53224   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53225     return SDValue(N, 0);
53226 
53227   return SDValue();
53228 }
53229 
53230 static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG,
53231                             TargetLowering::DAGCombinerInfo &DCI,
53232                             const X86Subtarget &Subtarget) {
53233   MVT VT = N->getSimpleValueType(0);
53234   unsigned NumBits = VT.getScalarSizeInBits();
53235 
53236   // Simplify the inputs.
53237   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53238   APInt DemandedMask(APInt::getAllOnes(NumBits));
53239   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53240     return SDValue(N, 0);
53241 
53242   return SDValue();
53243 }
53244 
53245 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
53246                                        TargetLowering::DAGCombinerInfo &DCI) {
53247   auto *MemOp = cast<X86MaskedGatherScatterSDNode>(N);
53248   SDValue Mask = MemOp->getMask();
53249 
53250   // With vector masks we only demand the upper bit of the mask.
53251   if (Mask.getScalarValueSizeInBits() != 1) {
53252     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53253     APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
53254     if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
53255       if (N->getOpcode() != ISD::DELETED_NODE)
53256         DCI.AddToWorklist(N);
53257       return SDValue(N, 0);
53258     }
53259   }
53260 
53261   return SDValue();
53262 }
53263 
53264 static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
53265                                     SDValue Index, SDValue Base, SDValue Scale,
53266                                     SelectionDAG &DAG) {
53267   SDLoc DL(GorS);
53268 
53269   if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
53270     SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(),
53271                       Gather->getMask(), Base, Index, Scale } ;
53272     return DAG.getMaskedGather(Gather->getVTList(),
53273                                Gather->getMemoryVT(), DL, Ops,
53274                                Gather->getMemOperand(),
53275                                Gather->getIndexType(),
53276                                Gather->getExtensionType());
53277   }
53278   auto *Scatter = cast<MaskedScatterSDNode>(GorS);
53279   SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
53280                     Scatter->getMask(), Base, Index, Scale };
53281   return DAG.getMaskedScatter(Scatter->getVTList(),
53282                               Scatter->getMemoryVT(), DL,
53283                               Ops, Scatter->getMemOperand(),
53284                               Scatter->getIndexType(),
53285                               Scatter->isTruncatingStore());
53286 }
53287 
53288 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
53289                                     TargetLowering::DAGCombinerInfo &DCI) {
53290   SDLoc DL(N);
53291   auto *GorS = cast<MaskedGatherScatterSDNode>(N);
53292   SDValue Index = GorS->getIndex();
53293   SDValue Base = GorS->getBasePtr();
53294   SDValue Scale = GorS->getScale();
53295 
53296   if (DCI.isBeforeLegalize()) {
53297     unsigned IndexWidth = Index.getScalarValueSizeInBits();
53298 
53299     // Shrink constant indices if they are larger than 32-bits.
53300     // Only do this before legalize types since v2i64 could become v2i32.
53301     // FIXME: We could check that the type is legal if we're after legalize
53302     // types, but then we would need to construct test cases where that happens.
53303     // FIXME: We could support more than just constant vectors, but we need to
53304     // careful with costing. A truncate that can be optimized out would be fine.
53305     // Otherwise we might only want to create a truncate if it avoids a split.
53306     if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
53307       if (BV->isConstant() && IndexWidth > 32 &&
53308           DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
53309         EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
53310         Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
53311         return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53312       }
53313     }
53314 
53315     // Shrink any sign/zero extends from 32 or smaller to larger than 32 if
53316     // there are sufficient sign bits. Only do this before legalize types to
53317     // avoid creating illegal types in truncate.
53318     if ((Index.getOpcode() == ISD::SIGN_EXTEND ||
53319          Index.getOpcode() == ISD::ZERO_EXTEND) &&
53320         IndexWidth > 32 &&
53321         Index.getOperand(0).getScalarValueSizeInBits() <= 32 &&
53322         DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
53323       EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
53324       Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
53325       return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53326     }
53327   }
53328 
53329   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53330   EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
53331   // Try to move splat constant adders from the index operand to the base
53332   // pointer operand. Taking care to multiply by the scale. We can only do
53333   // this when index element type is the same as the pointer type.
53334   // Otherwise we need to be sure the math doesn't wrap before the scale.
53335   if (Index.getOpcode() == ISD::ADD &&
53336       Index.getValueType().getVectorElementType() == PtrVT &&
53337       isa<ConstantSDNode>(Scale)) {
53338     uint64_t ScaleAmt = Scale->getAsZExtVal();
53339     if (auto *BV = dyn_cast<BuildVectorSDNode>(Index.getOperand(1))) {
53340       BitVector UndefElts;
53341       if (ConstantSDNode *C = BV->getConstantSplatNode(&UndefElts)) {
53342         // FIXME: Allow non-constant?
53343         if (UndefElts.none()) {
53344           // Apply the scale.
53345           APInt Adder = C->getAPIntValue() * ScaleAmt;
53346           // Add it to the existing base.
53347           Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
53348                              DAG.getConstant(Adder, DL, PtrVT));
53349           Index = Index.getOperand(0);
53350           return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53351         }
53352       }
53353 
53354       // It's also possible base is just a constant. In that case, just
53355       // replace it with 0 and move the displacement into the index.
53356       if (BV->isConstant() && isa<ConstantSDNode>(Base) &&
53357           isOneConstant(Scale)) {
53358         SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base);
53359         // Combine the constant build_vector and the constant base.
53360         Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
53361                             Index.getOperand(1), Splat);
53362         // Add to the LHS of the original Index add.
53363         Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
53364                             Index.getOperand(0), Splat);
53365         Base = DAG.getConstant(0, DL, Base.getValueType());
53366         return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53367       }
53368     }
53369   }
53370 
53371   if (DCI.isBeforeLegalizeOps()) {
53372     unsigned IndexWidth = Index.getScalarValueSizeInBits();
53373 
53374     // Make sure the index is either i32 or i64
53375     if (IndexWidth != 32 && IndexWidth != 64) {
53376       MVT EltVT = IndexWidth > 32 ? MVT::i64 : MVT::i32;
53377       EVT IndexVT = Index.getValueType().changeVectorElementType(EltVT);
53378       Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
53379       return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53380     }
53381   }
53382 
53383   // With vector masks we only demand the upper bit of the mask.
53384   SDValue Mask = GorS->getMask();
53385   if (Mask.getScalarValueSizeInBits() != 1) {
53386     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53387     APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
53388     if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
53389       if (N->getOpcode() != ISD::DELETED_NODE)
53390         DCI.AddToWorklist(N);
53391       return SDValue(N, 0);
53392     }
53393   }
53394 
53395   return SDValue();
53396 }
53397 
53398 // Optimize  RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
53399 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
53400                                const X86Subtarget &Subtarget) {
53401   SDLoc DL(N);
53402   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
53403   SDValue EFLAGS = N->getOperand(1);
53404 
53405   // Try to simplify the EFLAGS and condition code operands.
53406   if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
53407     return getSETCC(CC, Flags, DL, DAG);
53408 
53409   return SDValue();
53410 }
53411 
53412 /// Optimize branch condition evaluation.
53413 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
53414                              const X86Subtarget &Subtarget) {
53415   SDLoc DL(N);
53416   SDValue EFLAGS = N->getOperand(3);
53417   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
53418 
53419   // Try to simplify the EFLAGS and condition code operands.
53420   // Make sure to not keep references to operands, as combineSetCCEFLAGS can
53421   // RAUW them under us.
53422   if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
53423     SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
53424     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
53425                        N->getOperand(1), Cond, Flags);
53426   }
53427 
53428   return SDValue();
53429 }
53430 
53431 // TODO: Could we move this to DAGCombine?
53432 static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
53433                                                   SelectionDAG &DAG) {
53434   // Take advantage of vector comparisons (etc.) producing 0 or -1 in each lane
53435   // to optimize away operation when it's from a constant.
53436   //
53437   // The general transformation is:
53438   //    UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
53439   //       AND(VECTOR_CMP(x,y), constant2)
53440   //    constant2 = UNARYOP(constant)
53441 
53442   // Early exit if this isn't a vector operation, the operand of the
53443   // unary operation isn't a bitwise AND, or if the sizes of the operations
53444   // aren't the same.
53445   EVT VT = N->getValueType(0);
53446   bool IsStrict = N->isStrictFPOpcode();
53447   unsigned NumEltBits = VT.getScalarSizeInBits();
53448   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53449   if (!VT.isVector() || Op0.getOpcode() != ISD::AND ||
53450       DAG.ComputeNumSignBits(Op0.getOperand(0)) != NumEltBits ||
53451       VT.getSizeInBits() != Op0.getValueSizeInBits())
53452     return SDValue();
53453 
53454   // Now check that the other operand of the AND is a constant. We could
53455   // make the transformation for non-constant splats as well, but it's unclear
53456   // that would be a benefit as it would not eliminate any operations, just
53457   // perform one more step in scalar code before moving to the vector unit.
53458   if (auto *BV = dyn_cast<BuildVectorSDNode>(Op0.getOperand(1))) {
53459     // Bail out if the vector isn't a constant.
53460     if (!BV->isConstant())
53461       return SDValue();
53462 
53463     // Everything checks out. Build up the new and improved node.
53464     SDLoc DL(N);
53465     EVT IntVT = BV->getValueType(0);
53466     // Create a new constant of the appropriate type for the transformed
53467     // DAG.
53468     SDValue SourceConst;
53469     if (IsStrict)
53470       SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
53471                                 {N->getOperand(0), SDValue(BV, 0)});
53472     else
53473       SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
53474     // The AND node needs bitcasts to/from an integer vector type around it.
53475     SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
53476     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
53477                                  MaskConst);
53478     SDValue Res = DAG.getBitcast(VT, NewAnd);
53479     if (IsStrict)
53480       return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
53481     return Res;
53482   }
53483 
53484   return SDValue();
53485 }
53486 
53487 /// If we are converting a value to floating-point, try to replace scalar
53488 /// truncate of an extracted vector element with a bitcast. This tries to keep
53489 /// the sequence on XMM registers rather than moving between vector and GPRs.
53490 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
53491   // TODO: This is currently only used by combineSIntToFP, but it is generalized
53492   //       to allow being called by any similar cast opcode.
53493   // TODO: Consider merging this into lowering: vectorizeExtractedCast().
53494   SDValue Trunc = N->getOperand(0);
53495   if (!Trunc.hasOneUse() || Trunc.getOpcode() != ISD::TRUNCATE)
53496     return SDValue();
53497 
53498   SDValue ExtElt = Trunc.getOperand(0);
53499   if (!ExtElt.hasOneUse() || ExtElt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
53500       !isNullConstant(ExtElt.getOperand(1)))
53501     return SDValue();
53502 
53503   EVT TruncVT = Trunc.getValueType();
53504   EVT SrcVT = ExtElt.getValueType();
53505   unsigned DestWidth = TruncVT.getSizeInBits();
53506   unsigned SrcWidth = SrcVT.getSizeInBits();
53507   if (SrcWidth % DestWidth != 0)
53508     return SDValue();
53509 
53510   // inttofp (trunc (extelt X, 0)) --> inttofp (extelt (bitcast X), 0)
53511   EVT SrcVecVT = ExtElt.getOperand(0).getValueType();
53512   unsigned VecWidth = SrcVecVT.getSizeInBits();
53513   unsigned NumElts = VecWidth / DestWidth;
53514   EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts);
53515   SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
53516   SDLoc DL(N);
53517   SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
53518                                   BitcastVec, ExtElt.getOperand(1));
53519   return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
53520 }
53521 
53522 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
53523                                const X86Subtarget &Subtarget) {
53524   bool IsStrict = N->isStrictFPOpcode();
53525   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53526   EVT VT = N->getValueType(0);
53527   EVT InVT = Op0.getValueType();
53528 
53529   // Using i16 as an intermediate type is a bad idea, unless we have HW support
53530   // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
53531   // if hasFP16 support:
53532   //   UINT_TO_FP(vXi1~15)  -> SINT_TO_FP(ZEXT(vXi1~15  to vXi16))
53533   //   UINT_TO_FP(vXi17~31) -> SINT_TO_FP(ZEXT(vXi17~31 to vXi32))
53534   // else
53535   //   UINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
53536   // UINT_TO_FP(vXi33~63) -> SINT_TO_FP(ZEXT(vXi33~63 to vXi64))
53537   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
53538     unsigned ScalarSize = InVT.getScalarSizeInBits();
53539     if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
53540         ScalarSize >= 64)
53541       return SDValue();
53542     SDLoc dl(N);
53543     EVT DstVT =
53544         EVT::getVectorVT(*DAG.getContext(),
53545                          (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
53546                          : ScalarSize < 32                        ? MVT::i32
53547                                                                   : MVT::i64,
53548                          InVT.getVectorNumElements());
53549     SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
53550     if (IsStrict)
53551       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53552                          {N->getOperand(0), P});
53553     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53554   }
53555 
53556   // UINT_TO_FP(vXi1) -> SINT_TO_FP(ZEXT(vXi1 to vXi32))
53557   // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32))
53558   // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32))
53559   if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
53560       VT.getScalarType() != MVT::f16) {
53561     SDLoc dl(N);
53562     EVT DstVT = InVT.changeVectorElementType(MVT::i32);
53563     SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
53564 
53565     // UINT_TO_FP isn't legal without AVX512 so use SINT_TO_FP.
53566     if (IsStrict)
53567       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53568                          {N->getOperand(0), P});
53569     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53570   }
53571 
53572   // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
53573   // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
53574   // the optimization here.
53575   if (DAG.SignBitIsZero(Op0)) {
53576     if (IsStrict)
53577       return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},
53578                          {N->getOperand(0), Op0});
53579     return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
53580   }
53581 
53582   return SDValue();
53583 }
53584 
53585 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
53586                                TargetLowering::DAGCombinerInfo &DCI,
53587                                const X86Subtarget &Subtarget) {
53588   // First try to optimize away the conversion entirely when it's
53589   // conditionally from a constant. Vectors only.
53590   bool IsStrict = N->isStrictFPOpcode();
53591   if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
53592     return Res;
53593 
53594   // Now move on to more general possibilities.
53595   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53596   EVT VT = N->getValueType(0);
53597   EVT InVT = Op0.getValueType();
53598 
53599   // Using i16 as an intermediate type is a bad idea, unless we have HW support
53600   // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
53601   // if hasFP16 support:
53602   //   SINT_TO_FP(vXi1~15)  -> SINT_TO_FP(SEXT(vXi1~15  to vXi16))
53603   //   SINT_TO_FP(vXi17~31) -> SINT_TO_FP(SEXT(vXi17~31 to vXi32))
53604   // else
53605   //   SINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
53606   // SINT_TO_FP(vXi33~63) -> SINT_TO_FP(SEXT(vXi33~63 to vXi64))
53607   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
53608     unsigned ScalarSize = InVT.getScalarSizeInBits();
53609     if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
53610         ScalarSize >= 64)
53611       return SDValue();
53612     SDLoc dl(N);
53613     EVT DstVT =
53614         EVT::getVectorVT(*DAG.getContext(),
53615                          (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
53616                          : ScalarSize < 32                        ? MVT::i32
53617                                                                   : MVT::i64,
53618                          InVT.getVectorNumElements());
53619     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
53620     if (IsStrict)
53621       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53622                          {N->getOperand(0), P});
53623     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53624   }
53625 
53626   // SINT_TO_FP(vXi1) -> SINT_TO_FP(SEXT(vXi1 to vXi32))
53627   // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32))
53628   // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32))
53629   if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
53630       VT.getScalarType() != MVT::f16) {
53631     SDLoc dl(N);
53632     EVT DstVT = InVT.changeVectorElementType(MVT::i32);
53633     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
53634     if (IsStrict)
53635       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53636                          {N->getOperand(0), P});
53637     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53638   }
53639 
53640   // Without AVX512DQ we only support i64 to float scalar conversion. For both
53641   // vectors and scalars, see if we know that the upper bits are all the sign
53642   // bit, in which case we can truncate the input to i32 and convert from that.
53643   if (InVT.getScalarSizeInBits() > 32 && !Subtarget.hasDQI()) {
53644     unsigned BitWidth = InVT.getScalarSizeInBits();
53645     unsigned NumSignBits = DAG.ComputeNumSignBits(Op0);
53646     if (NumSignBits >= (BitWidth - 31)) {
53647       EVT TruncVT = MVT::i32;
53648       if (InVT.isVector())
53649         TruncVT = InVT.changeVectorElementType(TruncVT);
53650       SDLoc dl(N);
53651       if (DCI.isBeforeLegalize() || TruncVT != MVT::v2i32) {
53652         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
53653         if (IsStrict)
53654           return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53655                              {N->getOperand(0), Trunc});
53656         return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);
53657       }
53658       // If we're after legalize and the type is v2i32 we need to shuffle and
53659       // use CVTSI2P.
53660       assert(InVT == MVT::v2i64 && "Unexpected VT!");
53661       SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
53662       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
53663                                           { 0, 2, -1, -1 });
53664       if (IsStrict)
53665         return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
53666                            {N->getOperand(0), Shuf});
53667       return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf);
53668     }
53669   }
53670 
53671   // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
53672   // a 32-bit target where SSE doesn't support i64->FP operations.
53673   if (!Subtarget.useSoftFloat() && Subtarget.hasX87() &&
53674       Op0.getOpcode() == ISD::LOAD) {
53675     LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
53676 
53677     // This transformation is not supported if the result type is f16 or f128.
53678     if (VT == MVT::f16 || VT == MVT::f128)
53679       return SDValue();
53680 
53681     // If we have AVX512DQ we can use packed conversion instructions unless
53682     // the VT is f80.
53683     if (Subtarget.hasDQI() && VT != MVT::f80)
53684       return SDValue();
53685 
53686     if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) &&
53687         Op0.hasOneUse() && !Subtarget.is64Bit() && InVT == MVT::i64) {
53688       std::pair<SDValue, SDValue> Tmp =
53689           Subtarget.getTargetLowering()->BuildFILD(
53690               VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(),
53691               Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG);
53692       DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
53693       return Tmp.first;
53694     }
53695   }
53696 
53697   if (IsStrict)
53698     return SDValue();
53699 
53700   if (SDValue V = combineToFPTruncExtElt(N, DAG))
53701     return V;
53702 
53703   return SDValue();
53704 }
53705 
53706 static bool needCarryOrOverflowFlag(SDValue Flags) {
53707   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
53708 
53709   for (const SDNode *User : Flags->uses()) {
53710     X86::CondCode CC;
53711     switch (User->getOpcode()) {
53712     default:
53713       // Be conservative.
53714       return true;
53715     case X86ISD::SETCC:
53716     case X86ISD::SETCC_CARRY:
53717       CC = (X86::CondCode)User->getConstantOperandVal(0);
53718       break;
53719     case X86ISD::BRCOND:
53720     case X86ISD::CMOV:
53721       CC = (X86::CondCode)User->getConstantOperandVal(2);
53722       break;
53723     }
53724 
53725     switch (CC) {
53726     default: break;
53727     case X86::COND_A: case X86::COND_AE:
53728     case X86::COND_B: case X86::COND_BE:
53729     case X86::COND_O: case X86::COND_NO:
53730     case X86::COND_G: case X86::COND_GE:
53731     case X86::COND_L: case X86::COND_LE:
53732       return true;
53733     }
53734   }
53735 
53736   return false;
53737 }
53738 
53739 static bool onlyZeroFlagUsed(SDValue Flags) {
53740   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
53741 
53742   for (const SDNode *User : Flags->uses()) {
53743     unsigned CCOpNo;
53744     switch (User->getOpcode()) {
53745     default:
53746       // Be conservative.
53747       return false;
53748     case X86ISD::SETCC:
53749     case X86ISD::SETCC_CARRY:
53750       CCOpNo = 0;
53751       break;
53752     case X86ISD::BRCOND:
53753     case X86ISD::CMOV:
53754       CCOpNo = 2;
53755       break;
53756     }
53757 
53758     X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
53759     if (CC != X86::COND_E && CC != X86::COND_NE)
53760       return false;
53761   }
53762 
53763   return true;
53764 }
53765 
53766 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
53767                           const X86Subtarget &Subtarget) {
53768   // Only handle test patterns.
53769   if (!isNullConstant(N->getOperand(1)))
53770     return SDValue();
53771 
53772   // If we have a CMP of a truncated binop, see if we can make a smaller binop
53773   // and use its flags directly.
53774   // TODO: Maybe we should try promoting compares that only use the zero flag
53775   // first if we can prove the upper bits with computeKnownBits?
53776   SDLoc dl(N);
53777   SDValue Op = N->getOperand(0);
53778   EVT VT = Op.getValueType();
53779   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53780 
53781   // If we have a constant logical shift that's only used in a comparison
53782   // against zero turn it into an equivalent AND. This allows turning it into
53783   // a TEST instruction later.
53784   if ((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) &&
53785       Op.hasOneUse() && isa<ConstantSDNode>(Op.getOperand(1)) &&
53786       onlyZeroFlagUsed(SDValue(N, 0))) {
53787     unsigned BitWidth = VT.getSizeInBits();
53788     const APInt &ShAmt = Op.getConstantOperandAPInt(1);
53789     if (ShAmt.ult(BitWidth)) { // Avoid undefined shifts.
53790       unsigned MaskBits = BitWidth - ShAmt.getZExtValue();
53791       APInt Mask = Op.getOpcode() == ISD::SRL
53792                        ? APInt::getHighBitsSet(BitWidth, MaskBits)
53793                        : APInt::getLowBitsSet(BitWidth, MaskBits);
53794       if (Mask.isSignedIntN(32)) {
53795         Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),
53796                          DAG.getConstant(Mask, dl, VT));
53797         return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53798                            DAG.getConstant(0, dl, VT));
53799       }
53800     }
53801   }
53802 
53803   // If we're extracting from a avx512 bool vector and comparing against zero,
53804   // then try to just bitcast the vector to an integer to use TEST/BT directly.
53805   // (and (extract_elt (kshiftr vXi1, C), 0), 1) -> (and (bc vXi1), 1<<C)
53806   if (Op.getOpcode() == ISD::AND && isOneConstant(Op.getOperand(1)) &&
53807       Op.hasOneUse() && onlyZeroFlagUsed(SDValue(N, 0))) {
53808     SDValue Src = Op.getOperand(0);
53809     if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
53810         isNullConstant(Src.getOperand(1)) &&
53811         Src.getOperand(0).getValueType().getScalarType() == MVT::i1) {
53812       SDValue BoolVec = Src.getOperand(0);
53813       unsigned ShAmt = 0;
53814       if (BoolVec.getOpcode() == X86ISD::KSHIFTR) {
53815         ShAmt = BoolVec.getConstantOperandVal(1);
53816         BoolVec = BoolVec.getOperand(0);
53817       }
53818       BoolVec = widenMaskVector(BoolVec, false, Subtarget, DAG, dl);
53819       EVT VecVT = BoolVec.getValueType();
53820       unsigned BitWidth = VecVT.getVectorNumElements();
53821       EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth);
53822       if (TLI.isTypeLegal(VecVT) && TLI.isTypeLegal(BCVT)) {
53823         APInt Mask = APInt::getOneBitSet(BitWidth, ShAmt);
53824         Op = DAG.getBitcast(BCVT, BoolVec);
53825         Op = DAG.getNode(ISD::AND, dl, BCVT, Op,
53826                          DAG.getConstant(Mask, dl, BCVT));
53827         return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53828                            DAG.getConstant(0, dl, BCVT));
53829       }
53830     }
53831   }
53832 
53833   // Peek through any zero-extend if we're only testing for a zero result.
53834   if (Op.getOpcode() == ISD::ZERO_EXTEND && onlyZeroFlagUsed(SDValue(N, 0))) {
53835     SDValue Src = Op.getOperand(0);
53836     EVT SrcVT = Src.getValueType();
53837     if (SrcVT.getScalarSizeInBits() >= 8 && TLI.isTypeLegal(SrcVT))
53838       return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,
53839                          DAG.getConstant(0, dl, SrcVT));
53840   }
53841 
53842   // Look for a truncate.
53843   if (Op.getOpcode() != ISD::TRUNCATE)
53844     return SDValue();
53845 
53846   SDValue Trunc = Op;
53847   Op = Op.getOperand(0);
53848 
53849   // See if we can compare with zero against the truncation source,
53850   // which should help using the Z flag from many ops. Only do this for
53851   // i32 truncated op to prevent partial-reg compares of promoted ops.
53852   EVT OpVT = Op.getValueType();
53853   APInt UpperBits =
53854       APInt::getBitsSetFrom(OpVT.getSizeInBits(), VT.getSizeInBits());
53855   if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) &&
53856       onlyZeroFlagUsed(SDValue(N, 0))) {
53857     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53858                        DAG.getConstant(0, dl, OpVT));
53859   }
53860 
53861   // After this the truncate and arithmetic op must have a single use.
53862   if (!Trunc.hasOneUse() || !Op.hasOneUse())
53863       return SDValue();
53864 
53865   unsigned NewOpc;
53866   switch (Op.getOpcode()) {
53867   default: return SDValue();
53868   case ISD::AND:
53869     // Skip and with constant. We have special handling for and with immediate
53870     // during isel to generate test instructions.
53871     if (isa<ConstantSDNode>(Op.getOperand(1)))
53872       return SDValue();
53873     NewOpc = X86ISD::AND;
53874     break;
53875   case ISD::OR:  NewOpc = X86ISD::OR;  break;
53876   case ISD::XOR: NewOpc = X86ISD::XOR; break;
53877   case ISD::ADD:
53878     // If the carry or overflow flag is used, we can't truncate.
53879     if (needCarryOrOverflowFlag(SDValue(N, 0)))
53880       return SDValue();
53881     NewOpc = X86ISD::ADD;
53882     break;
53883   case ISD::SUB:
53884     // If the carry or overflow flag is used, we can't truncate.
53885     if (needCarryOrOverflowFlag(SDValue(N, 0)))
53886       return SDValue();
53887     NewOpc = X86ISD::SUB;
53888     break;
53889   }
53890 
53891   // We found an op we can narrow. Truncate its inputs.
53892   SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
53893   SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
53894 
53895   // Use a X86 specific opcode to avoid DAG combine messing with it.
53896   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
53897   Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1);
53898 
53899   // For AND, keep a CMP so that we can match the test pattern.
53900   if (NewOpc == X86ISD::AND)
53901     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53902                        DAG.getConstant(0, dl, VT));
53903 
53904   // Return the flags.
53905   return Op.getValue(1);
53906 }
53907 
53908 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
53909                                 TargetLowering::DAGCombinerInfo &DCI) {
53910   assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) &&
53911          "Expected X86ISD::ADD or X86ISD::SUB");
53912 
53913   SDLoc DL(N);
53914   SDValue LHS = N->getOperand(0);
53915   SDValue RHS = N->getOperand(1);
53916   MVT VT = LHS.getSimpleValueType();
53917   bool IsSub = X86ISD::SUB == N->getOpcode();
53918   unsigned GenericOpc = IsSub ? ISD::SUB : ISD::ADD;
53919 
53920   // If we don't use the flag result, simplify back to a generic ADD/SUB.
53921   if (!N->hasAnyUseOfValue(1)) {
53922     SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
53923     return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
53924   }
53925 
53926   // Fold any similar generic ADD/SUB opcodes to reuse this node.
53927   auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
53928     SDValue Ops[] = {N0, N1};
53929     SDVTList VTs = DAG.getVTList(N->getValueType(0));
53930     if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) {
53931       SDValue Op(N, 0);
53932       if (Negate)
53933         Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
53934       DCI.CombineTo(GenericAddSub, Op);
53935     }
53936   };
53937   MatchGeneric(LHS, RHS, false);
53938   MatchGeneric(RHS, LHS, X86ISD::SUB == N->getOpcode());
53939 
53940   // TODO: Can we drop the ZeroSecondOpOnly limit? This is to guarantee that the
53941   // EFLAGS result doesn't change.
53942   return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG,
53943                                    /*ZeroSecondOpOnly*/ true);
53944 }
53945 
53946 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
53947   SDValue LHS = N->getOperand(0);
53948   SDValue RHS = N->getOperand(1);
53949   SDValue BorrowIn = N->getOperand(2);
53950 
53951   if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) {
53952     MVT VT = N->getSimpleValueType(0);
53953     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
53954     return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags);
53955   }
53956 
53957   // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry)
53958   // iff the flag result is dead.
53959   if (LHS.getOpcode() == ISD::SUB && isNullConstant(RHS) &&
53960       !N->hasAnyUseOfValue(1))
53961     return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0),
53962                        LHS.getOperand(1), BorrowIn);
53963 
53964   return SDValue();
53965 }
53966 
53967 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
53968 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
53969                           TargetLowering::DAGCombinerInfo &DCI) {
53970   SDValue LHS = N->getOperand(0);
53971   SDValue RHS = N->getOperand(1);
53972   SDValue CarryIn = N->getOperand(2);
53973   auto *LHSC = dyn_cast<ConstantSDNode>(LHS);
53974   auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
53975 
53976   // Canonicalize constant to RHS.
53977   if (LHSC && !RHSC)
53978     return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS,
53979                        CarryIn);
53980 
53981   // If the LHS and RHS of the ADC node are zero, then it can't overflow and
53982   // the result is either zero or one (depending on the input carry bit).
53983   // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
53984   if (LHSC && RHSC && LHSC->isZero() && RHSC->isZero() &&
53985       // We don't have a good way to replace an EFLAGS use, so only do this when
53986       // dead right now.
53987       SDValue(N, 1).use_empty()) {
53988     SDLoc DL(N);
53989     EVT VT = N->getValueType(0);
53990     SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
53991     SDValue Res1 = DAG.getNode(
53992         ISD::AND, DL, VT,
53993         DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
53994                     DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),
53995         DAG.getConstant(1, DL, VT));
53996     return DCI.CombineTo(N, Res1, CarryOut);
53997   }
53998 
53999   // Fold ADC(C1,C2,Carry) -> ADC(0,C1+C2,Carry)
54000   // iff the flag result is dead.
54001   // TODO: Allow flag result if C1+C2 doesn't signed/unsigned overflow.
54002   if (LHSC && RHSC && !LHSC->isZero() && !N->hasAnyUseOfValue(1)) {
54003     SDLoc DL(N);
54004     APInt Sum = LHSC->getAPIntValue() + RHSC->getAPIntValue();
54005     return DAG.getNode(X86ISD::ADC, DL, N->getVTList(),
54006                        DAG.getConstant(0, DL, LHS.getValueType()),
54007                        DAG.getConstant(Sum, DL, LHS.getValueType()), CarryIn);
54008   }
54009 
54010   if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) {
54011     MVT VT = N->getSimpleValueType(0);
54012     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
54013     return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags);
54014   }
54015 
54016   // Fold ADC(ADD(X,Y),0,Carry) -> ADC(X,Y,Carry)
54017   // iff the flag result is dead.
54018   if (LHS.getOpcode() == ISD::ADD && RHSC && RHSC->isZero() &&
54019       !N->hasAnyUseOfValue(1))
54020     return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0),
54021                        LHS.getOperand(1), CarryIn);
54022 
54023   return SDValue();
54024 }
54025 
54026 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
54027                             const SDLoc &DL, EVT VT,
54028                             const X86Subtarget &Subtarget) {
54029   // Example of pattern we try to detect:
54030   // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1))))
54031   //(add (build_vector (extract_elt t, 0),
54032   //                   (extract_elt t, 2),
54033   //                   (extract_elt t, 4),
54034   //                   (extract_elt t, 6)),
54035   //     (build_vector (extract_elt t, 1),
54036   //                   (extract_elt t, 3),
54037   //                   (extract_elt t, 5),
54038   //                   (extract_elt t, 7)))
54039 
54040   if (!Subtarget.hasSSE2())
54041     return SDValue();
54042 
54043   if (Op0.getOpcode() != ISD::BUILD_VECTOR ||
54044       Op1.getOpcode() != ISD::BUILD_VECTOR)
54045     return SDValue();
54046 
54047   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
54048       VT.getVectorNumElements() < 4 ||
54049       !isPowerOf2_32(VT.getVectorNumElements()))
54050     return SDValue();
54051 
54052   // Check if one of Op0,Op1 is of the form:
54053   // (build_vector (extract_elt Mul, 0),
54054   //               (extract_elt Mul, 2),
54055   //               (extract_elt Mul, 4),
54056   //                   ...
54057   // the other is of the form:
54058   // (build_vector (extract_elt Mul, 1),
54059   //               (extract_elt Mul, 3),
54060   //               (extract_elt Mul, 5),
54061   //                   ...
54062   // and identify Mul.
54063   SDValue Mul;
54064   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) {
54065     SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
54066             Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1);
54067     // TODO: Be more tolerant to undefs.
54068     if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54069         Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54070         Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54071         Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
54072       return SDValue();
54073     auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1));
54074     auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1));
54075     auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1));
54076     auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1));
54077     if (!Const0L || !Const1L || !Const0H || !Const1H)
54078       return SDValue();
54079     unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(),
54080              Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue();
54081     // Commutativity of mul allows factors of a product to reorder.
54082     if (Idx0L > Idx1L)
54083       std::swap(Idx0L, Idx1L);
54084     if (Idx0H > Idx1H)
54085       std::swap(Idx0H, Idx1H);
54086     // Commutativity of add allows pairs of factors to reorder.
54087     if (Idx0L > Idx0H) {
54088       std::swap(Idx0L, Idx0H);
54089       std::swap(Idx1L, Idx1H);
54090     }
54091     if (Idx0L != 2 * i || Idx1L != 2 * i + 1 || Idx0H != 2 * i + 2 ||
54092         Idx1H != 2 * i + 3)
54093       return SDValue();
54094     if (!Mul) {
54095       // First time an extract_elt's source vector is visited. Must be a MUL
54096       // with 2X number of vector elements than the BUILD_VECTOR.
54097       // Both extracts must be from same MUL.
54098       Mul = Op0L->getOperand(0);
54099       if (Mul->getOpcode() != ISD::MUL ||
54100           Mul.getValueType().getVectorNumElements() != 2 * e)
54101         return SDValue();
54102     }
54103     // Check that the extract is from the same MUL previously seen.
54104     if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) ||
54105         Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0))
54106       return SDValue();
54107   }
54108 
54109   // Check if the Mul source can be safely shrunk.
54110   ShrinkMode Mode;
54111   if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) ||
54112       Mode == ShrinkMode::MULU16)
54113     return SDValue();
54114 
54115   EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
54116                                  VT.getVectorNumElements() * 2);
54117   SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
54118   SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
54119 
54120   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
54121                          ArrayRef<SDValue> Ops) {
54122     EVT InVT = Ops[0].getValueType();
54123     assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
54124     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
54125                                  InVT.getVectorNumElements() / 2);
54126     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
54127   };
54128   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
54129 }
54130 
54131 // Attempt to turn this pattern into PMADDWD.
54132 // (add (mul (sext (build_vector)), (sext (build_vector))),
54133 //      (mul (sext (build_vector)), (sext (build_vector)))
54134 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
54135                               const SDLoc &DL, EVT VT,
54136                               const X86Subtarget &Subtarget) {
54137   if (!Subtarget.hasSSE2())
54138     return SDValue();
54139 
54140   if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
54141     return SDValue();
54142 
54143   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
54144       VT.getVectorNumElements() < 4 ||
54145       !isPowerOf2_32(VT.getVectorNumElements()))
54146     return SDValue();
54147 
54148   SDValue N00 = N0.getOperand(0);
54149   SDValue N01 = N0.getOperand(1);
54150   SDValue N10 = N1.getOperand(0);
54151   SDValue N11 = N1.getOperand(1);
54152 
54153   // All inputs need to be sign extends.
54154   // TODO: Support ZERO_EXTEND from known positive?
54155   if (N00.getOpcode() != ISD::SIGN_EXTEND ||
54156       N01.getOpcode() != ISD::SIGN_EXTEND ||
54157       N10.getOpcode() != ISD::SIGN_EXTEND ||
54158       N11.getOpcode() != ISD::SIGN_EXTEND)
54159     return SDValue();
54160 
54161   // Peek through the extends.
54162   N00 = N00.getOperand(0);
54163   N01 = N01.getOperand(0);
54164   N10 = N10.getOperand(0);
54165   N11 = N11.getOperand(0);
54166 
54167   // Must be extending from vXi16.
54168   EVT InVT = N00.getValueType();
54169   if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||
54170       N10.getValueType() != InVT || N11.getValueType() != InVT)
54171     return SDValue();
54172 
54173   // All inputs should be build_vectors.
54174   if (N00.getOpcode() != ISD::BUILD_VECTOR ||
54175       N01.getOpcode() != ISD::BUILD_VECTOR ||
54176       N10.getOpcode() != ISD::BUILD_VECTOR ||
54177       N11.getOpcode() != ISD::BUILD_VECTOR)
54178     return SDValue();
54179 
54180   // For each element, we need to ensure we have an odd element from one vector
54181   // multiplied by the odd element of another vector and the even element from
54182   // one of the same vectors being multiplied by the even element from the
54183   // other vector. So we need to make sure for each element i, this operator
54184   // is being performed:
54185   //  A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
54186   SDValue In0, In1;
54187   for (unsigned i = 0; i != N00.getNumOperands(); ++i) {
54188     SDValue N00Elt = N00.getOperand(i);
54189     SDValue N01Elt = N01.getOperand(i);
54190     SDValue N10Elt = N10.getOperand(i);
54191     SDValue N11Elt = N11.getOperand(i);
54192     // TODO: Be more tolerant to undefs.
54193     if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54194         N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54195         N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54196         N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
54197       return SDValue();
54198     auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
54199     auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
54200     auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
54201     auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
54202     if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
54203       return SDValue();
54204     unsigned IdxN00 = ConstN00Elt->getZExtValue();
54205     unsigned IdxN01 = ConstN01Elt->getZExtValue();
54206     unsigned IdxN10 = ConstN10Elt->getZExtValue();
54207     unsigned IdxN11 = ConstN11Elt->getZExtValue();
54208     // Add is commutative so indices can be reordered.
54209     if (IdxN00 > IdxN10) {
54210       std::swap(IdxN00, IdxN10);
54211       std::swap(IdxN01, IdxN11);
54212     }
54213     // N0 indices be the even element. N1 indices must be the next odd element.
54214     if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
54215         IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
54216       return SDValue();
54217     SDValue N00In = N00Elt.getOperand(0);
54218     SDValue N01In = N01Elt.getOperand(0);
54219     SDValue N10In = N10Elt.getOperand(0);
54220     SDValue N11In = N11Elt.getOperand(0);
54221 
54222     // First time we find an input capture it.
54223     if (!In0) {
54224       In0 = N00In;
54225       In1 = N01In;
54226 
54227       // The input vectors must be at least as wide as the output.
54228       // If they are larger than the output, we extract subvector below.
54229       if (In0.getValueSizeInBits() < VT.getSizeInBits() ||
54230           In1.getValueSizeInBits() < VT.getSizeInBits())
54231         return SDValue();
54232     }
54233     // Mul is commutative so the input vectors can be in any order.
54234     // Canonicalize to make the compares easier.
54235     if (In0 != N00In)
54236       std::swap(N00In, N01In);
54237     if (In0 != N10In)
54238       std::swap(N10In, N11In);
54239     if (In0 != N00In || In1 != N01In || In0 != N10In || In1 != N11In)
54240       return SDValue();
54241   }
54242 
54243   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
54244                          ArrayRef<SDValue> Ops) {
54245     EVT OpVT = Ops[0].getValueType();
54246     assert(OpVT.getScalarType() == MVT::i16 &&
54247            "Unexpected scalar element type");
54248     assert(OpVT == Ops[1].getValueType() && "Operands' types mismatch");
54249     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
54250                                  OpVT.getVectorNumElements() / 2);
54251     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
54252   };
54253 
54254   // If the output is narrower than an input, extract the low part of the input
54255   // vector.
54256   EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
54257                                VT.getVectorNumElements() * 2);
54258   if (OutVT16.bitsLT(In0.getValueType())) {
54259     In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
54260                       DAG.getIntPtrConstant(0, DL));
54261   }
54262   if (OutVT16.bitsLT(In1.getValueType())) {
54263     In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
54264                       DAG.getIntPtrConstant(0, DL));
54265   }
54266   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
54267                           PMADDBuilder);
54268 }
54269 
54270 // ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
54271 // If upper element in each pair of both VPMADDWD are zero then we can merge
54272 // the operand elements and use the implicit add of VPMADDWD.
54273 // TODO: Add support for VPMADDUBSW (which isn't commutable).
54274 static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
54275                                    const SDLoc &DL, EVT VT) {
54276   if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
54277     return SDValue();
54278 
54279   // TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
54280   if (VT.getSizeInBits() > 128)
54281     return SDValue();
54282 
54283   unsigned NumElts = VT.getVectorNumElements();
54284   MVT OpVT = N0.getOperand(0).getSimpleValueType();
54285   APInt DemandedBits = APInt::getAllOnes(OpVT.getScalarSizeInBits());
54286   APInt DemandedHiElts = APInt::getSplat(2 * NumElts, APInt(2, 2));
54287 
54288   bool Op0HiZero =
54289       DAG.MaskedValueIsZero(N0.getOperand(0), DemandedBits, DemandedHiElts) ||
54290       DAG.MaskedValueIsZero(N0.getOperand(1), DemandedBits, DemandedHiElts);
54291   bool Op1HiZero =
54292       DAG.MaskedValueIsZero(N1.getOperand(0), DemandedBits, DemandedHiElts) ||
54293       DAG.MaskedValueIsZero(N1.getOperand(1), DemandedBits, DemandedHiElts);
54294 
54295   // TODO: Check for zero lower elements once we have actual codegen that
54296   // creates them.
54297   if (!Op0HiZero || !Op1HiZero)
54298     return SDValue();
54299 
54300   // Create a shuffle mask packing the lower elements from each VPMADDWD.
54301   SmallVector<int> Mask;
54302   for (int i = 0; i != (int)NumElts; ++i) {
54303     Mask.push_back(2 * i);
54304     Mask.push_back(2 * (i + NumElts));
54305   }
54306 
54307   SDValue LHS =
54308       DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
54309   SDValue RHS =
54310       DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
54311   return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
54312 }
54313 
54314 /// CMOV of constants requires materializing constant operands in registers.
54315 /// Try to fold those constants into an 'add' instruction to reduce instruction
54316 /// count. We do this with CMOV rather the generic 'select' because there are
54317 /// earlier folds that may be used to turn select-of-constants into logic hacks.
54318 static SDValue pushAddIntoCmovOfConsts(SDNode *N, SelectionDAG &DAG,
54319                                        const X86Subtarget &Subtarget) {
54320   // If an operand is zero, add-of-0 gets simplified away, so that's clearly
54321   // better because we eliminate 1-2 instructions. This transform is still
54322   // an improvement without zero operands because we trade 2 move constants and
54323   // 1 add for 2 adds (LEA) as long as the constants can be represented as
54324   // immediate asm operands (fit in 32-bits).
54325   auto isSuitableCmov = [](SDValue V) {
54326     if (V.getOpcode() != X86ISD::CMOV || !V.hasOneUse())
54327       return false;
54328     if (!isa<ConstantSDNode>(V.getOperand(0)) ||
54329         !isa<ConstantSDNode>(V.getOperand(1)))
54330       return false;
54331     return isNullConstant(V.getOperand(0)) || isNullConstant(V.getOperand(1)) ||
54332            (V.getConstantOperandAPInt(0).isSignedIntN(32) &&
54333             V.getConstantOperandAPInt(1).isSignedIntN(32));
54334   };
54335 
54336   // Match an appropriate CMOV as the first operand of the add.
54337   SDValue Cmov = N->getOperand(0);
54338   SDValue OtherOp = N->getOperand(1);
54339   if (!isSuitableCmov(Cmov))
54340     std::swap(Cmov, OtherOp);
54341   if (!isSuitableCmov(Cmov))
54342     return SDValue();
54343 
54344   // Don't remove a load folding opportunity for the add. That would neutralize
54345   // any improvements from removing constant materializations.
54346   if (X86::mayFoldLoad(OtherOp, Subtarget))
54347     return SDValue();
54348 
54349   EVT VT = N->getValueType(0);
54350   SDLoc DL(N);
54351   SDValue FalseOp = Cmov.getOperand(0);
54352   SDValue TrueOp = Cmov.getOperand(1);
54353 
54354   // We will push the add through the select, but we can potentially do better
54355   // if we know there is another add in the sequence and this is pointer math.
54356   // In that case, we can absorb an add into the trailing memory op and avoid
54357   // a 3-operand LEA which is likely slower than a 2-operand LEA.
54358   // TODO: If target has "slow3OpsLEA", do this even without the trailing memop?
54359   if (OtherOp.getOpcode() == ISD::ADD && OtherOp.hasOneUse() &&
54360       !isa<ConstantSDNode>(OtherOp.getOperand(0)) &&
54361       all_of(N->uses(), [&](SDNode *Use) {
54362         auto *MemNode = dyn_cast<MemSDNode>(Use);
54363         return MemNode && MemNode->getBasePtr().getNode() == N;
54364       })) {
54365     // add (cmov C1, C2), add (X, Y) --> add (cmov (add X, C1), (add X, C2)), Y
54366     // TODO: We are arbitrarily choosing op0 as the 1st piece of the sum, but
54367     //       it is possible that choosing op1 might be better.
54368     SDValue X = OtherOp.getOperand(0), Y = OtherOp.getOperand(1);
54369     FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp);
54370     TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp);
54371     Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp,
54372                        Cmov.getOperand(2), Cmov.getOperand(3));
54373     return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y);
54374   }
54375 
54376   // add (cmov C1, C2), OtherOp --> cmov (add OtherOp, C1), (add OtherOp, C2)
54377   FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp);
54378   TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp);
54379   return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2),
54380                      Cmov.getOperand(3));
54381 }
54382 
54383 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
54384                           TargetLowering::DAGCombinerInfo &DCI,
54385                           const X86Subtarget &Subtarget) {
54386   EVT VT = N->getValueType(0);
54387   SDValue Op0 = N->getOperand(0);
54388   SDValue Op1 = N->getOperand(1);
54389   SDLoc DL(N);
54390 
54391   if (SDValue Select = pushAddIntoCmovOfConsts(N, DAG, Subtarget))
54392     return Select;
54393 
54394   if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget))
54395     return MAdd;
54396   if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget))
54397     return MAdd;
54398   if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
54399     return MAdd;
54400 
54401   // Try to synthesize horizontal adds from adds of shuffles.
54402   if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
54403     return V;
54404 
54405   // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
54406   // (sub Y, (sext (vXi1 X))).
54407   // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
54408   // generic DAG combine without a legal type check, but adding this there
54409   // caused regressions.
54410   if (VT.isVector()) {
54411     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54412     if (Op0.getOpcode() == ISD::ZERO_EXTEND &&
54413         Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
54414         TLI.isTypeLegal(Op0.getOperand(0).getValueType())) {
54415       SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
54416       return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
54417     }
54418 
54419     if (Op1.getOpcode() == ISD::ZERO_EXTEND &&
54420         Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
54421         TLI.isTypeLegal(Op1.getOperand(0).getValueType())) {
54422       SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
54423       return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
54424     }
54425   }
54426 
54427   // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W)
54428   if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() &&
54429       X86::isZeroNode(Op0.getOperand(1))) {
54430     assert(!Op0->hasAnyUseOfValue(1) && "Overflow bit in use");
54431     return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1,
54432                        Op0.getOperand(0), Op0.getOperand(2));
54433   }
54434 
54435   return combineAddOrSubToADCOrSBB(N, DAG);
54436 }
54437 
54438 // Try to fold (sub Y, cmovns X, -X) -> (add Y, cmovns -X, X) if the cmov
54439 // condition comes from the subtract node that produced -X. This matches the
54440 // cmov expansion for absolute value. By swapping the operands we convert abs
54441 // to nabs.
54442 static SDValue combineSubABS(SDNode *N, SelectionDAG &DAG) {
54443   SDValue N0 = N->getOperand(0);
54444   SDValue N1 = N->getOperand(1);
54445 
54446   if (N1.getOpcode() != X86ISD::CMOV || !N1.hasOneUse())
54447     return SDValue();
54448 
54449   X86::CondCode CC = (X86::CondCode)N1.getConstantOperandVal(2);
54450   if (CC != X86::COND_S && CC != X86::COND_NS)
54451     return SDValue();
54452 
54453   // Condition should come from a negate operation.
54454   SDValue Cond = N1.getOperand(3);
54455   if (Cond.getOpcode() != X86ISD::SUB || !isNullConstant(Cond.getOperand(0)))
54456     return SDValue();
54457   assert(Cond.getResNo() == 1 && "Unexpected result number");
54458 
54459   // Get the X and -X from the negate.
54460   SDValue NegX = Cond.getValue(0);
54461   SDValue X = Cond.getOperand(1);
54462 
54463   SDValue FalseOp = N1.getOperand(0);
54464   SDValue TrueOp = N1.getOperand(1);
54465 
54466   // Cmov operands should be X and NegX. Order doesn't matter.
54467   if (!(TrueOp == X && FalseOp == NegX) && !(TrueOp == NegX && FalseOp == X))
54468     return SDValue();
54469 
54470   // Build a new CMOV with the operands swapped.
54471   SDLoc DL(N);
54472   MVT VT = N->getSimpleValueType(0);
54473   SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp,
54474                              N1.getOperand(2), Cond);
54475   // Convert sub to add.
54476   return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov);
54477 }
54478 
54479 static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
54480   SDValue Op0 = N->getOperand(0);
54481   SDValue Op1 = N->getOperand(1);
54482 
54483   // (sub C (zero_extend (setcc)))
54484   // =>
54485   // (add (zero_extend (setcc inverted) C-1))   if C is a nonzero immediate
54486   // Don't disturb (sub 0 setcc), which is easily done with neg.
54487   EVT VT = N->getValueType(0);
54488   auto *Op0C = dyn_cast<ConstantSDNode>(Op0);
54489   if (Op1.getOpcode() == ISD::ZERO_EXTEND && Op1.hasOneUse() && Op0C &&
54490       !Op0C->isZero() && Op1.getOperand(0).getOpcode() == X86ISD::SETCC &&
54491       Op1.getOperand(0).hasOneUse()) {
54492     SDValue SetCC = Op1.getOperand(0);
54493     X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
54494     X86::CondCode NewCC = X86::GetOppositeBranchCondition(CC);
54495     APInt NewImm = Op0C->getAPIntValue() - 1;
54496     SDLoc DL(Op1);
54497     SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG);
54498     NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC);
54499     return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC,
54500                        DAG.getConstant(NewImm, DL, VT));
54501   }
54502 
54503   return SDValue();
54504 }
54505 
54506 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
54507                           TargetLowering::DAGCombinerInfo &DCI,
54508                           const X86Subtarget &Subtarget) {
54509   SDValue Op0 = N->getOperand(0);
54510   SDValue Op1 = N->getOperand(1);
54511 
54512   // TODO: Add NoOpaque handling to isConstantIntBuildVectorOrConstantInt.
54513   auto IsNonOpaqueConstant = [&](SDValue Op) {
54514     if (SDNode *C = DAG.isConstantIntBuildVectorOrConstantInt(Op)) {
54515       if (auto *Cst = dyn_cast<ConstantSDNode>(C))
54516         return !Cst->isOpaque();
54517       return true;
54518     }
54519     return false;
54520   };
54521 
54522   // X86 can't encode an immediate LHS of a sub. See if we can push the
54523   // negation into a preceding instruction. If the RHS of the sub is a XOR with
54524   // one use and a constant, invert the immediate, saving one register.
54525   // However, ignore cases where C1 is 0, as those will become a NEG.
54526   // sub(C1, xor(X, C2)) -> add(xor(X, ~C2), C1+1)
54527   if (Op1.getOpcode() == ISD::XOR && IsNonOpaqueConstant(Op0) &&
54528       !isNullConstant(Op0) && IsNonOpaqueConstant(Op1.getOperand(1)) &&
54529       Op1->hasOneUse()) {
54530     SDLoc DL(N);
54531     EVT VT = Op0.getValueType();
54532     SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0),
54533                                  DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT));
54534     SDValue NewAdd =
54535         DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT));
54536     return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd);
54537   }
54538 
54539   if (SDValue V = combineSubABS(N, DAG))
54540     return V;
54541 
54542   // Try to synthesize horizontal subs from subs of shuffles.
54543   if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
54544     return V;
54545 
54546   // Fold SUB(X,ADC(Y,0,W)) -> SBB(X,Y,W)
54547   if (Op1.getOpcode() == X86ISD::ADC && Op1->hasOneUse() &&
54548       X86::isZeroNode(Op1.getOperand(1))) {
54549     assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
54550     return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0,
54551                        Op1.getOperand(0), Op1.getOperand(2));
54552   }
54553 
54554   // Fold SUB(X,SBB(Y,Z,W)) -> SUB(ADC(X,Z,W),Y)
54555   // Don't fold to ADC(0,0,W)/SETCC_CARRY pattern which will prevent more folds.
54556   if (Op1.getOpcode() == X86ISD::SBB && Op1->hasOneUse() &&
54557       !(X86::isZeroNode(Op0) && X86::isZeroNode(Op1.getOperand(1)))) {
54558     assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
54559     SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0,
54560                               Op1.getOperand(1), Op1.getOperand(2));
54561     return DAG.getNode(ISD::SUB, SDLoc(N), Op0.getValueType(), ADC.getValue(0),
54562                        Op1.getOperand(0));
54563   }
54564 
54565   if (SDValue V = combineXorSubCTLZ(N, DAG, Subtarget))
54566     return V;
54567 
54568   if (SDValue V = combineAddOrSubToADCOrSBB(N, DAG))
54569     return V;
54570 
54571   return combineSubSetcc(N, DAG);
54572 }
54573 
54574 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
54575                                     const X86Subtarget &Subtarget) {
54576   MVT VT = N->getSimpleValueType(0);
54577   SDLoc DL(N);
54578 
54579   if (N->getOperand(0) == N->getOperand(1)) {
54580     if (N->getOpcode() == X86ISD::PCMPEQ)
54581       return DAG.getConstant(-1, DL, VT);
54582     if (N->getOpcode() == X86ISD::PCMPGT)
54583       return DAG.getConstant(0, DL, VT);
54584   }
54585 
54586   return SDValue();
54587 }
54588 
54589 /// Helper that combines an array of subvector ops as if they were the operands
54590 /// of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
54591 /// ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
54592 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
54593                                       ArrayRef<SDValue> Ops, SelectionDAG &DAG,
54594                                       TargetLowering::DAGCombinerInfo &DCI,
54595                                       const X86Subtarget &Subtarget) {
54596   assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors");
54597   unsigned EltSizeInBits = VT.getScalarSizeInBits();
54598 
54599   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
54600     return DAG.getUNDEF(VT);
54601 
54602   if (llvm::all_of(Ops, [](SDValue Op) {
54603         return ISD::isBuildVectorAllZeros(Op.getNode());
54604       }))
54605     return getZeroVector(VT, Subtarget, DAG, DL);
54606 
54607   SDValue Op0 = Ops[0];
54608   bool IsSplat = llvm::all_equal(Ops);
54609   unsigned NumOps = Ops.size();
54610   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54611   LLVMContext &Ctx = *DAG.getContext();
54612 
54613   // Repeated subvectors.
54614   if (IsSplat &&
54615       (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
54616     // If this broadcast is inserted into both halves, use a larger broadcast.
54617     if (Op0.getOpcode() == X86ISD::VBROADCAST)
54618       return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
54619 
54620     // concat_vectors(movddup(x),movddup(x)) -> broadcast(x)
54621     if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 &&
54622         (Subtarget.hasAVX2() ||
54623          X86::mayFoldLoadIntoBroadcastFromMem(Op0.getOperand(0),
54624                                               VT.getScalarType(), Subtarget)))
54625       return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
54626                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
54627                                      Op0.getOperand(0),
54628                                      DAG.getIntPtrConstant(0, DL)));
54629 
54630     // concat_vectors(scalar_to_vector(x),scalar_to_vector(x)) -> broadcast(x)
54631     if (Op0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
54632         (Subtarget.hasAVX2() ||
54633          (EltSizeInBits >= 32 &&
54634           X86::mayFoldLoad(Op0.getOperand(0), Subtarget))) &&
54635         Op0.getOperand(0).getValueType() == VT.getScalarType())
54636       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
54637 
54638     // concat_vectors(extract_subvector(broadcast(x)),
54639     //                extract_subvector(broadcast(x))) -> broadcast(x)
54640     // concat_vectors(extract_subvector(subv_broadcast(x)),
54641     //                extract_subvector(subv_broadcast(x))) -> subv_broadcast(x)
54642     if (Op0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54643         Op0.getOperand(0).getValueType() == VT) {
54644       SDValue SrcVec = Op0.getOperand(0);
54645       if (SrcVec.getOpcode() == X86ISD::VBROADCAST ||
54646           SrcVec.getOpcode() == X86ISD::VBROADCAST_LOAD)
54647         return Op0.getOperand(0);
54648       if (SrcVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
54649           Op0.getValueType() == cast<MemSDNode>(SrcVec)->getMemoryVT())
54650         return Op0.getOperand(0);
54651     }
54652 
54653     // concat_vectors(permq(x),permq(x)) -> permq(concat_vectors(x,x))
54654     if (Op0.getOpcode() == X86ISD::VPERMI && Subtarget.useAVX512Regs() &&
54655         !X86::mayFoldLoad(Op0.getOperand(0), Subtarget))
54656       return DAG.getNode(Op0.getOpcode(), DL, VT,
54657                          DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
54658                                      Op0.getOperand(0), Op0.getOperand(0)),
54659                          Op0.getOperand(1));
54660   }
54661 
54662   // concat(extract_subvector(v0,c0), extract_subvector(v1,c1)) -> vperm2x128.
54663   // Only concat of subvector high halves which vperm2x128 is best at.
54664   // TODO: This should go in combineX86ShufflesRecursively eventually.
54665   if (VT.is256BitVector() && NumOps == 2) {
54666     SDValue Src0 = peekThroughBitcasts(Ops[0]);
54667     SDValue Src1 = peekThroughBitcasts(Ops[1]);
54668     if (Src0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54669         Src1.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
54670       EVT SrcVT0 = Src0.getOperand(0).getValueType();
54671       EVT SrcVT1 = Src1.getOperand(0).getValueType();
54672       unsigned NumSrcElts0 = SrcVT0.getVectorNumElements();
54673       unsigned NumSrcElts1 = SrcVT1.getVectorNumElements();
54674       if (SrcVT0.is256BitVector() && SrcVT1.is256BitVector() &&
54675           Src0.getConstantOperandAPInt(1) == (NumSrcElts0 / 2) &&
54676           Src1.getConstantOperandAPInt(1) == (NumSrcElts1 / 2)) {
54677         return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
54678                            DAG.getBitcast(VT, Src0.getOperand(0)),
54679                            DAG.getBitcast(VT, Src1.getOperand(0)),
54680                            DAG.getTargetConstant(0x31, DL, MVT::i8));
54681       }
54682     }
54683   }
54684 
54685   // Repeated opcode.
54686   // TODO - combineX86ShufflesRecursively should handle shuffle concatenation
54687   // but it currently struggles with different vector widths.
54688   if (llvm::all_of(Ops, [Op0](SDValue Op) {
54689         return Op.getOpcode() == Op0.getOpcode() && Op.hasOneUse();
54690       })) {
54691     auto ConcatSubOperand = [&](EVT VT, ArrayRef<SDValue> SubOps, unsigned I) {
54692       SmallVector<SDValue> Subs;
54693       for (SDValue SubOp : SubOps)
54694         Subs.push_back(SubOp.getOperand(I));
54695       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
54696     };
54697     auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
54698       bool AllConstants = true;
54699       bool AllSubVectors = true;
54700       for (unsigned I = 0, E = SubOps.size(); I != E; ++I) {
54701         SDValue Sub = SubOps[I].getOperand(Op);
54702         unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
54703         SDValue BC = peekThroughBitcasts(Sub);
54704         AllConstants &= ISD::isBuildVectorOfConstantSDNodes(BC.getNode()) ||
54705                         ISD::isBuildVectorOfConstantFPSDNodes(BC.getNode());
54706         AllSubVectors &= Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54707                          Sub.getOperand(0).getValueType() == VT &&
54708                          Sub.getConstantOperandAPInt(1) == (I * NumSubElts);
54709       }
54710       return AllConstants || AllSubVectors;
54711     };
54712 
54713     switch (Op0.getOpcode()) {
54714     case X86ISD::VBROADCAST: {
54715       if (!IsSplat && llvm::all_of(Ops, [](SDValue Op) {
54716             return Op.getOperand(0).getValueType().is128BitVector();
54717           })) {
54718         if (VT == MVT::v4f64 || VT == MVT::v4i64)
54719           return DAG.getNode(X86ISD::UNPCKL, DL, VT,
54720                              ConcatSubOperand(VT, Ops, 0),
54721                              ConcatSubOperand(VT, Ops, 0));
54722         // TODO: Add pseudo v8i32 PSHUFD handling to AVX1Only targets.
54723         if (VT == MVT::v8f32 || (VT == MVT::v8i32 && Subtarget.hasInt256()))
54724           return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI
54725                                               : X86ISD::PSHUFD,
54726                              DL, VT, ConcatSubOperand(VT, Ops, 0),
54727                              getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
54728       }
54729       break;
54730     }
54731     case X86ISD::MOVDDUP:
54732     case X86ISD::MOVSHDUP:
54733     case X86ISD::MOVSLDUP: {
54734       if (!IsSplat)
54735         return DAG.getNode(Op0.getOpcode(), DL, VT,
54736                            ConcatSubOperand(VT, Ops, 0));
54737       break;
54738     }
54739     case X86ISD::SHUFP: {
54740       // Add SHUFPD support if/when necessary.
54741       if (!IsSplat && VT.getScalarType() == MVT::f32 &&
54742           llvm::all_of(Ops, [Op0](SDValue Op) {
54743             return Op.getOperand(2) == Op0.getOperand(2);
54744           })) {
54745         return DAG.getNode(Op0.getOpcode(), DL, VT,
54746                            ConcatSubOperand(VT, Ops, 0),
54747                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
54748       }
54749       break;
54750     }
54751     case X86ISD::UNPCKH:
54752     case X86ISD::UNPCKL: {
54753       // Don't concatenate build_vector patterns.
54754       if (!IsSplat && EltSizeInBits >= 32 &&
54755           ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54756            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
54757           none_of(Ops, [](SDValue Op) {
54758             return peekThroughBitcasts(Op.getOperand(0)).getOpcode() ==
54759                        ISD::SCALAR_TO_VECTOR ||
54760                    peekThroughBitcasts(Op.getOperand(1)).getOpcode() ==
54761                        ISD::SCALAR_TO_VECTOR;
54762           })) {
54763         return DAG.getNode(Op0.getOpcode(), DL, VT,
54764                            ConcatSubOperand(VT, Ops, 0),
54765                            ConcatSubOperand(VT, Ops, 1));
54766       }
54767       break;
54768     }
54769     case X86ISD::PSHUFHW:
54770     case X86ISD::PSHUFLW:
54771     case X86ISD::PSHUFD:
54772       if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
54773           Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
54774         return DAG.getNode(Op0.getOpcode(), DL, VT,
54775                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54776       }
54777       [[fallthrough]];
54778     case X86ISD::VPERMILPI:
54779       if (!IsSplat && EltSizeInBits == 32 &&
54780           (VT.is256BitVector() ||
54781            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
54782           all_of(Ops, [&Op0](SDValue Op) {
54783             return Op0.getOperand(1) == Op.getOperand(1);
54784           })) {
54785         MVT FloatVT = VT.changeVectorElementType(MVT::f32);
54786         SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0));
54787         Res =
54788             DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1));
54789         return DAG.getBitcast(VT, Res);
54790       }
54791       if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) {
54792         uint64_t Idx0 = Ops[0].getConstantOperandVal(1);
54793         uint64_t Idx1 = Ops[1].getConstantOperandVal(1);
54794         uint64_t Idx = ((Idx1 & 3) << 2) | (Idx0 & 3);
54795         return DAG.getNode(Op0.getOpcode(), DL, VT,
54796                            ConcatSubOperand(VT, Ops, 0),
54797                            DAG.getTargetConstant(Idx, DL, MVT::i8));
54798       }
54799       break;
54800     case X86ISD::PSHUFB:
54801     case X86ISD::PSADBW:
54802       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54803                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
54804         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
54805         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
54806                                  NumOps * SrcVT.getVectorNumElements());
54807         return DAG.getNode(Op0.getOpcode(), DL, VT,
54808                            ConcatSubOperand(SrcVT, Ops, 0),
54809                            ConcatSubOperand(SrcVT, Ops, 1));
54810       }
54811       break;
54812     case X86ISD::VPERMV:
54813       if (!IsSplat && NumOps == 2 &&
54814           (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
54815         MVT OpVT = Op0.getSimpleValueType();
54816         int NumSrcElts = OpVT.getVectorNumElements();
54817         SmallVector<int, 64> ConcatMask;
54818         for (unsigned i = 0; i != NumOps; ++i) {
54819           SmallVector<int, 64> SubMask;
54820           SmallVector<SDValue, 2> SubOps;
54821           if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps,
54822                                     SubMask))
54823             break;
54824           for (int M : SubMask) {
54825             if (0 <= M)
54826               M += i * NumSrcElts;
54827             ConcatMask.push_back(M);
54828           }
54829         }
54830         if (ConcatMask.size() == (NumOps * NumSrcElts)) {
54831           SDValue Src = concatSubVectors(Ops[0].getOperand(1),
54832                                          Ops[1].getOperand(1), DAG, DL);
54833           MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
54834           MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
54835           SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
54836           return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src);
54837         }
54838       }
54839       break;
54840     case X86ISD::VPERMV3:
54841       if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
54842         MVT OpVT = Op0.getSimpleValueType();
54843         int NumSrcElts = OpVT.getVectorNumElements();
54844         SmallVector<int, 64> ConcatMask;
54845         for (unsigned i = 0; i != NumOps; ++i) {
54846           SmallVector<int, 64> SubMask;
54847           SmallVector<SDValue, 2> SubOps;
54848           if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps,
54849                                     SubMask))
54850             break;
54851           for (int M : SubMask) {
54852             if (0 <= M) {
54853               M += M < NumSrcElts ? 0 : NumSrcElts;
54854               M += i * NumSrcElts;
54855             }
54856             ConcatMask.push_back(M);
54857           }
54858         }
54859         if (ConcatMask.size() == (NumOps * NumSrcElts)) {
54860           SDValue Src0 = concatSubVectors(Ops[0].getOperand(0),
54861                                           Ops[1].getOperand(0), DAG, DL);
54862           SDValue Src1 = concatSubVectors(Ops[0].getOperand(2),
54863                                           Ops[1].getOperand(2), DAG, DL);
54864           MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
54865           MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
54866           SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
54867           return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1);
54868         }
54869       }
54870       break;
54871     case X86ISD::VPERM2X128: {
54872       if (!IsSplat && VT.is512BitVector() && Subtarget.useAVX512Regs()) {
54873         assert(NumOps == 2 && "Bad concat_vectors operands");
54874         unsigned Imm0 = Ops[0].getConstantOperandVal(2);
54875         unsigned Imm1 = Ops[1].getConstantOperandVal(2);
54876         // TODO: Handle zero'd subvectors.
54877         if ((Imm0 & 0x88) == 0 && (Imm1 & 0x88) == 0) {
54878           int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3), (int)(Imm1 & 0x03),
54879                          (int)((Imm1 >> 4) & 0x3)};
54880           MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
54881           SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
54882                                          Ops[0].getOperand(1), DAG, DL);
54883           SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
54884                                          Ops[1].getOperand(1), DAG, DL);
54885           SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
54886                                     DAG.getBitcast(ShuffleVT, LHS),
54887                                     DAG.getBitcast(ShuffleVT, RHS),
54888                                     getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
54889           return DAG.getBitcast(VT, Res);
54890         }
54891       }
54892       break;
54893     }
54894     case X86ISD::SHUF128: {
54895       if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
54896         unsigned Imm0 = Ops[0].getConstantOperandVal(2);
54897         unsigned Imm1 = Ops[1].getConstantOperandVal(2);
54898         unsigned Imm = ((Imm0 & 1) << 0) | ((Imm0 & 2) << 1) | 0x08 |
54899                        ((Imm1 & 1) << 4) | ((Imm1 & 2) << 5) | 0x80;
54900         SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
54901                                        Ops[0].getOperand(1), DAG, DL);
54902         SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
54903                                        Ops[1].getOperand(1), DAG, DL);
54904         return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS,
54905                            DAG.getTargetConstant(Imm, DL, MVT::i8));
54906       }
54907       break;
54908     }
54909     case ISD::TRUNCATE:
54910       if (!IsSplat && NumOps == 2 && VT.is256BitVector()) {
54911         EVT SrcVT = Ops[0].getOperand(0).getValueType();
54912         if (SrcVT.is256BitVector() && SrcVT.isSimple() &&
54913             SrcVT == Ops[1].getOperand(0).getValueType() &&
54914             Subtarget.useAVX512Regs() &&
54915             Subtarget.getPreferVectorWidth() >= 512 &&
54916             (SrcVT.getScalarSizeInBits() > 16 || Subtarget.useBWIRegs())) {
54917           EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx);
54918           return DAG.getNode(ISD::TRUNCATE, DL, VT,
54919                              ConcatSubOperand(NewSrcVT, Ops, 0));
54920         }
54921       }
54922       break;
54923     case X86ISD::VSHLI:
54924     case X86ISD::VSRLI:
54925       // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle.
54926       // TODO: Move this to LowerShiftByScalarImmediate?
54927       if (VT == MVT::v4i64 && !Subtarget.hasInt256() &&
54928           llvm::all_of(Ops, [](SDValue Op) {
54929             return Op.getConstantOperandAPInt(1) == 32;
54930           })) {
54931         SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
54932         SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
54933         if (Op0.getOpcode() == X86ISD::VSHLI) {
54934           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
54935                                      {8, 0, 8, 2, 8, 4, 8, 6});
54936         } else {
54937           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
54938                                      {1, 8, 3, 8, 5, 8, 7, 8});
54939         }
54940         return DAG.getBitcast(VT, Res);
54941       }
54942       [[fallthrough]];
54943     case X86ISD::VSRAI:
54944     case X86ISD::VSHL:
54945     case X86ISD::VSRL:
54946     case X86ISD::VSRA:
54947       if (((VT.is256BitVector() && Subtarget.hasInt256()) ||
54948            (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
54949             (EltSizeInBits >= 32 || Subtarget.useBWIRegs()))) &&
54950           llvm::all_of(Ops, [Op0](SDValue Op) {
54951             return Op0.getOperand(1) == Op.getOperand(1);
54952           })) {
54953         return DAG.getNode(Op0.getOpcode(), DL, VT,
54954                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54955       }
54956       break;
54957     case X86ISD::VPERMI:
54958     case X86ISD::VROTLI:
54959     case X86ISD::VROTRI:
54960       if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
54961           llvm::all_of(Ops, [Op0](SDValue Op) {
54962             return Op0.getOperand(1) == Op.getOperand(1);
54963           })) {
54964         return DAG.getNode(Op0.getOpcode(), DL, VT,
54965                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54966       }
54967       break;
54968     case ISD::AND:
54969     case ISD::OR:
54970     case ISD::XOR:
54971     case X86ISD::ANDNP:
54972       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54973                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
54974         return DAG.getNode(Op0.getOpcode(), DL, VT,
54975                            ConcatSubOperand(VT, Ops, 0),
54976                            ConcatSubOperand(VT, Ops, 1));
54977       }
54978       break;
54979     case X86ISD::PCMPEQ:
54980     case X86ISD::PCMPGT:
54981       if (!IsSplat && VT.is256BitVector() && Subtarget.hasInt256() &&
54982           (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) {
54983         return DAG.getNode(Op0.getOpcode(), DL, VT,
54984                            ConcatSubOperand(VT, Ops, 0),
54985                            ConcatSubOperand(VT, Ops, 1));
54986       }
54987       break;
54988     case ISD::CTPOP:
54989     case ISD::CTTZ:
54990     case ISD::CTLZ:
54991     case ISD::CTTZ_ZERO_UNDEF:
54992     case ISD::CTLZ_ZERO_UNDEF:
54993       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54994                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
54995         return DAG.getNode(Op0.getOpcode(), DL, VT,
54996                            ConcatSubOperand(VT, Ops, 0));
54997       }
54998       break;
54999     case X86ISD::GF2P8AFFINEQB:
55000       if (!IsSplat &&
55001           (VT.is256BitVector() ||
55002            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
55003           llvm::all_of(Ops, [Op0](SDValue Op) {
55004             return Op0.getOperand(2) == Op.getOperand(2);
55005           })) {
55006         return DAG.getNode(Op0.getOpcode(), DL, VT,
55007                            ConcatSubOperand(VT, Ops, 0),
55008                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
55009       }
55010       break;
55011     case ISD::ADD:
55012     case ISD::SUB:
55013     case ISD::MUL:
55014       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55015                        (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
55016                         (EltSizeInBits >= 32 || Subtarget.useBWIRegs())))) {
55017         return DAG.getNode(Op0.getOpcode(), DL, VT,
55018                            ConcatSubOperand(VT, Ops, 0),
55019                            ConcatSubOperand(VT, Ops, 1));
55020       }
55021       break;
55022     // Due to VADD, VSUB, VMUL can executed on more ports than VINSERT and
55023     // their latency are short, so here we don't replace them.
55024     case ISD::FDIV:
55025       if (!IsSplat && (VT.is256BitVector() ||
55026                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
55027         return DAG.getNode(Op0.getOpcode(), DL, VT,
55028                            ConcatSubOperand(VT, Ops, 0),
55029                            ConcatSubOperand(VT, Ops, 1));
55030       }
55031       break;
55032     case X86ISD::HADD:
55033     case X86ISD::HSUB:
55034     case X86ISD::FHADD:
55035     case X86ISD::FHSUB:
55036       if (!IsSplat && VT.is256BitVector() &&
55037           (VT.isFloatingPoint() || Subtarget.hasInt256())) {
55038         return DAG.getNode(Op0.getOpcode(), DL, VT,
55039                            ConcatSubOperand(VT, Ops, 0),
55040                            ConcatSubOperand(VT, Ops, 1));
55041       }
55042       break;
55043     case X86ISD::PACKSS:
55044     case X86ISD::PACKUS:
55045       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55046                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
55047         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
55048         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
55049                                  NumOps * SrcVT.getVectorNumElements());
55050         return DAG.getNode(Op0.getOpcode(), DL, VT,
55051                            ConcatSubOperand(SrcVT, Ops, 0),
55052                            ConcatSubOperand(SrcVT, Ops, 1));
55053       }
55054       break;
55055     case X86ISD::PALIGNR:
55056       if (!IsSplat &&
55057           ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55058            (VT.is512BitVector() && Subtarget.useBWIRegs())) &&
55059           llvm::all_of(Ops, [Op0](SDValue Op) {
55060             return Op0.getOperand(2) == Op.getOperand(2);
55061           })) {
55062         return DAG.getNode(Op0.getOpcode(), DL, VT,
55063                            ConcatSubOperand(VT, Ops, 0),
55064                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
55065       }
55066       break;
55067     case X86ISD::BLENDI:
55068       if (NumOps == 2 && VT.is512BitVector() && Subtarget.useBWIRegs()) {
55069         uint64_t Mask0 = Ops[0].getConstantOperandVal(2);
55070         uint64_t Mask1 = Ops[1].getConstantOperandVal(2);
55071         uint64_t Mask = (Mask1 << (VT.getVectorNumElements() / 2)) | Mask0;
55072         MVT MaskSVT = MVT::getIntegerVT(VT.getVectorNumElements());
55073         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
55074         SDValue Sel =
55075             DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT));
55076         return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1),
55077                              ConcatSubOperand(VT, Ops, 0));
55078       }
55079       break;
55080     case ISD::VSELECT:
55081       if (!IsSplat && Subtarget.hasAVX512() &&
55082           (VT.is256BitVector() ||
55083            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
55084           (EltSizeInBits >= 32 || Subtarget.hasBWI())) {
55085         EVT SelVT = Ops[0].getOperand(0).getValueType();
55086         if (SelVT.getVectorElementType() == MVT::i1) {
55087           SelVT = EVT::getVectorVT(Ctx, MVT::i1,
55088                                    NumOps * SelVT.getVectorNumElements());
55089           if (TLI.isTypeLegal(SelVT))
55090             return DAG.getNode(Op0.getOpcode(), DL, VT,
55091                                ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
55092                                ConcatSubOperand(VT, Ops, 1),
55093                                ConcatSubOperand(VT, Ops, 2));
55094         }
55095       }
55096       [[fallthrough]];
55097     case X86ISD::BLENDV:
55098       if (!IsSplat && VT.is256BitVector() && NumOps == 2 &&
55099           (EltSizeInBits >= 32 || Subtarget.hasInt256()) &&
55100           IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) {
55101         EVT SelVT = Ops[0].getOperand(0).getValueType();
55102         SelVT = SelVT.getDoubleNumVectorElementsVT(Ctx);
55103         if (TLI.isTypeLegal(SelVT))
55104           return DAG.getNode(Op0.getOpcode(), DL, VT,
55105                              ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
55106                              ConcatSubOperand(VT, Ops, 1),
55107                              ConcatSubOperand(VT, Ops, 2));
55108       }
55109       break;
55110     }
55111   }
55112 
55113   // Fold subvector loads into one.
55114   // If needed, look through bitcasts to get to the load.
55115   if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) {
55116     unsigned Fast;
55117     const X86TargetLowering *TLI = Subtarget.getTargetLowering();
55118     if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT,
55119                                 *FirstLd->getMemOperand(), &Fast) &&
55120         Fast) {
55121       if (SDValue Ld =
55122               EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
55123         return Ld;
55124     }
55125   }
55126 
55127   // Attempt to fold target constant loads.
55128   if (all_of(Ops, [](SDValue Op) { return getTargetConstantFromNode(Op); })) {
55129     SmallVector<APInt> EltBits;
55130     APInt UndefElts = APInt::getZero(VT.getVectorNumElements());
55131     for (unsigned I = 0; I != NumOps; ++I) {
55132       APInt OpUndefElts;
55133       SmallVector<APInt> OpEltBits;
55134       if (!getTargetConstantBitsFromNode(Ops[I], EltSizeInBits, OpUndefElts,
55135                                          OpEltBits, true, false))
55136         break;
55137       EltBits.append(OpEltBits);
55138       UndefElts.insertBits(OpUndefElts, I * OpUndefElts.getBitWidth());
55139     }
55140     if (EltBits.size() == VT.getVectorNumElements()) {
55141       Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx);
55142       MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
55143       SDValue CV = DAG.getConstantPool(C, PVT);
55144       MachineFunction &MF = DAG.getMachineFunction();
55145       MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
55146       SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI);
55147       SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits());
55148       DAG.ReplaceAllUsesOfValueWith(Op0, Sub);
55149       return Ld;
55150     }
55151   }
55152 
55153   // If this simple subvector or scalar/subvector broadcast_load is inserted
55154   // into both halves, use a larger broadcast_load. Update other uses to use
55155   // an extracted subvector.
55156   if (IsSplat &&
55157       (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
55158     if (ISD::isNormalLoad(Op0.getNode()) ||
55159         Op0.getOpcode() == X86ISD::VBROADCAST_LOAD ||
55160         Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
55161       auto *Mem = cast<MemSDNode>(Op0);
55162       unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD
55163                          ? X86ISD::VBROADCAST_LOAD
55164                          : X86ISD::SUBV_BROADCAST_LOAD;
55165       if (SDValue BcastLd =
55166               getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) {
55167         SDValue BcastSrc =
55168             extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits());
55169         DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc);
55170         return BcastLd;
55171       }
55172     }
55173   }
55174 
55175   // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly.
55176   if (IsSplat && NumOps == 4 && VT.is512BitVector() &&
55177       Subtarget.useAVX512Regs()) {
55178     MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
55179     SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
55180     Res = DAG.getBitcast(ShuffleVT, Res);
55181     Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, Res, Res,
55182                       getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
55183     return DAG.getBitcast(VT, Res);
55184   }
55185 
55186   return SDValue();
55187 }
55188 
55189 static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
55190                                      TargetLowering::DAGCombinerInfo &DCI,
55191                                      const X86Subtarget &Subtarget) {
55192   EVT VT = N->getValueType(0);
55193   EVT SrcVT = N->getOperand(0).getValueType();
55194   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55195   SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
55196 
55197   if (VT.getVectorElementType() == MVT::i1) {
55198     // Attempt to constant fold.
55199     unsigned SubSizeInBits = SrcVT.getSizeInBits();
55200     APInt Constant = APInt::getZero(VT.getSizeInBits());
55201     for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
55202       auto *C = dyn_cast<ConstantSDNode>(peekThroughBitcasts(Ops[I]));
55203       if (!C) break;
55204       Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits);
55205       if (I == (E - 1)) {
55206         EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
55207         if (TLI.isTypeLegal(IntVT))
55208           return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT));
55209       }
55210     }
55211 
55212     // Don't do anything else for i1 vectors.
55213     return SDValue();
55214   }
55215 
55216   if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) {
55217     if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
55218                                            DCI, Subtarget))
55219       return R;
55220   }
55221 
55222   return SDValue();
55223 }
55224 
55225 static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
55226                                        TargetLowering::DAGCombinerInfo &DCI,
55227                                        const X86Subtarget &Subtarget) {
55228   if (DCI.isBeforeLegalizeOps())
55229     return SDValue();
55230 
55231   MVT OpVT = N->getSimpleValueType(0);
55232 
55233   bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1;
55234 
55235   SDLoc dl(N);
55236   SDValue Vec = N->getOperand(0);
55237   SDValue SubVec = N->getOperand(1);
55238 
55239   uint64_t IdxVal = N->getConstantOperandVal(2);
55240   MVT SubVecVT = SubVec.getSimpleValueType();
55241 
55242   if (Vec.isUndef() && SubVec.isUndef())
55243     return DAG.getUNDEF(OpVT);
55244 
55245   // Inserting undefs/zeros into zeros/undefs is a zero vector.
55246   if ((Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())) &&
55247       (SubVec.isUndef() || ISD::isBuildVectorAllZeros(SubVec.getNode())))
55248     return getZeroVector(OpVT, Subtarget, DAG, dl);
55249 
55250   if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
55251     // If we're inserting into a zero vector and then into a larger zero vector,
55252     // just insert into the larger zero vector directly.
55253     if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
55254         ISD::isBuildVectorAllZeros(SubVec.getOperand(0).getNode())) {
55255       uint64_t Idx2Val = SubVec.getConstantOperandVal(2);
55256       return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55257                          getZeroVector(OpVT, Subtarget, DAG, dl),
55258                          SubVec.getOperand(1),
55259                          DAG.getIntPtrConstant(IdxVal + Idx2Val, dl));
55260     }
55261 
55262     // If we're inserting into a zero vector and our input was extracted from an
55263     // insert into a zero vector of the same type and the extraction was at
55264     // least as large as the original insertion. Just insert the original
55265     // subvector into a zero vector.
55266     if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR && IdxVal == 0 &&
55267         isNullConstant(SubVec.getOperand(1)) &&
55268         SubVec.getOperand(0).getOpcode() == ISD::INSERT_SUBVECTOR) {
55269       SDValue Ins = SubVec.getOperand(0);
55270       if (isNullConstant(Ins.getOperand(2)) &&
55271           ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
55272           Ins.getOperand(1).getValueSizeInBits().getFixedValue() <=
55273               SubVecVT.getFixedSizeInBits())
55274           return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55275                              getZeroVector(OpVT, Subtarget, DAG, dl),
55276                              Ins.getOperand(1), N->getOperand(2));
55277     }
55278   }
55279 
55280   // Stop here if this is an i1 vector.
55281   if (IsI1Vector)
55282     return SDValue();
55283 
55284   // Eliminate an intermediate vector widening:
55285   // insert_subvector X, (insert_subvector undef, Y, 0), Idx -->
55286   // insert_subvector X, Y, Idx
55287   // TODO: This is a more general version of a DAGCombiner fold, can we move it
55288   // there?
55289   if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
55290       SubVec.getOperand(0).isUndef() && isNullConstant(SubVec.getOperand(2)))
55291     return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec,
55292                        SubVec.getOperand(1), N->getOperand(2));
55293 
55294   // If this is an insert of an extract, combine to a shuffle. Don't do this
55295   // if the insert or extract can be represented with a subregister operation.
55296   if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
55297       SubVec.getOperand(0).getSimpleValueType() == OpVT &&
55298       (IdxVal != 0 ||
55299        !(Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())))) {
55300     int ExtIdxVal = SubVec.getConstantOperandVal(1);
55301     if (ExtIdxVal != 0) {
55302       int VecNumElts = OpVT.getVectorNumElements();
55303       int SubVecNumElts = SubVecVT.getVectorNumElements();
55304       SmallVector<int, 64> Mask(VecNumElts);
55305       // First create an identity shuffle mask.
55306       for (int i = 0; i != VecNumElts; ++i)
55307         Mask[i] = i;
55308       // Now insert the extracted portion.
55309       for (int i = 0; i != SubVecNumElts; ++i)
55310         Mask[i + IdxVal] = i + ExtIdxVal + VecNumElts;
55311 
55312       return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask);
55313     }
55314   }
55315 
55316   // Match concat_vector style patterns.
55317   SmallVector<SDValue, 2> SubVectorOps;
55318   if (collectConcatOps(N, SubVectorOps, DAG)) {
55319     if (SDValue Fold =
55320             combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
55321       return Fold;
55322 
55323     // If we're inserting all zeros into the upper half, change this to
55324     // a concat with zero. We will match this to a move
55325     // with implicit upper bit zeroing during isel.
55326     // We do this here because we don't want combineConcatVectorOps to
55327     // create INSERT_SUBVECTOR from CONCAT_VECTORS.
55328     if (SubVectorOps.size() == 2 &&
55329         ISD::isBuildVectorAllZeros(SubVectorOps[1].getNode()))
55330       return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55331                          getZeroVector(OpVT, Subtarget, DAG, dl),
55332                          SubVectorOps[0], DAG.getIntPtrConstant(0, dl));
55333 
55334     // Attempt to recursively combine to a shuffle.
55335     if (all_of(SubVectorOps, [](SDValue SubOp) {
55336           return isTargetShuffle(SubOp.getOpcode());
55337         })) {
55338       SDValue Op(N, 0);
55339       if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
55340         return Res;
55341     }
55342   }
55343 
55344   // If this is a broadcast insert into an upper undef, use a larger broadcast.
55345   if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
55346     return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
55347 
55348   // If this is a broadcast load inserted into an upper undef, use a larger
55349   // broadcast load.
55350   if (Vec.isUndef() && IdxVal != 0 && SubVec.hasOneUse() &&
55351       SubVec.getOpcode() == X86ISD::VBROADCAST_LOAD) {
55352     auto *MemIntr = cast<MemIntrinsicSDNode>(SubVec);
55353     SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
55354     SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
55355     SDValue BcastLd =
55356         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
55357                                 MemIntr->getMemoryVT(),
55358                                 MemIntr->getMemOperand());
55359     DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
55360     return BcastLd;
55361   }
55362 
55363   // If we're splatting the lower half subvector of a full vector load into the
55364   // upper half, attempt to create a subvector broadcast.
55365   if (IdxVal == (OpVT.getVectorNumElements() / 2) && SubVec.hasOneUse() &&
55366       Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits())) {
55367     auto *VecLd = dyn_cast<LoadSDNode>(Vec);
55368     auto *SubLd = dyn_cast<LoadSDNode>(SubVec);
55369     if (VecLd && SubLd &&
55370         DAG.areNonVolatileConsecutiveLoads(SubLd, VecLd,
55371                                            SubVec.getValueSizeInBits() / 8, 0))
55372       return getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, dl, OpVT, SubVecVT,
55373                                SubLd, 0, DAG);
55374   }
55375 
55376   return SDValue();
55377 }
55378 
55379 /// If we are extracting a subvector of a vector select and the select condition
55380 /// is composed of concatenated vectors, try to narrow the select width. This
55381 /// is a common pattern for AVX1 integer code because 256-bit selects may be
55382 /// legal, but there is almost no integer math/logic available for 256-bit.
55383 /// This function should only be called with legal types (otherwise, the calls
55384 /// to get simple value types will assert).
55385 static SDValue narrowExtractedVectorSelect(SDNode *Ext, SelectionDAG &DAG) {
55386   SDValue Sel = Ext->getOperand(0);
55387   if (Sel.getOpcode() != ISD::VSELECT ||
55388       !isFreeToSplitVector(Sel.getOperand(0).getNode(), DAG))
55389     return SDValue();
55390 
55391   // Note: We assume simple value types because this should only be called with
55392   //       legal operations/types.
55393   // TODO: This can be extended to handle extraction to 256-bits.
55394   MVT VT = Ext->getSimpleValueType(0);
55395   if (!VT.is128BitVector())
55396     return SDValue();
55397 
55398   MVT SelCondVT = Sel.getOperand(0).getSimpleValueType();
55399   if (!SelCondVT.is256BitVector() && !SelCondVT.is512BitVector())
55400     return SDValue();
55401 
55402   MVT WideVT = Ext->getOperand(0).getSimpleValueType();
55403   MVT SelVT = Sel.getSimpleValueType();
55404   assert((SelVT.is256BitVector() || SelVT.is512BitVector()) &&
55405          "Unexpected vector type with legal operations");
55406 
55407   unsigned SelElts = SelVT.getVectorNumElements();
55408   unsigned CastedElts = WideVT.getVectorNumElements();
55409   unsigned ExtIdx = Ext->getConstantOperandVal(1);
55410   if (SelElts % CastedElts == 0) {
55411     // The select has the same or more (narrower) elements than the extract
55412     // operand. The extraction index gets scaled by that factor.
55413     ExtIdx *= (SelElts / CastedElts);
55414   } else if (CastedElts % SelElts == 0) {
55415     // The select has less (wider) elements than the extract operand. Make sure
55416     // that the extraction index can be divided evenly.
55417     unsigned IndexDivisor = CastedElts / SelElts;
55418     if (ExtIdx % IndexDivisor != 0)
55419       return SDValue();
55420     ExtIdx /= IndexDivisor;
55421   } else {
55422     llvm_unreachable("Element count of simple vector types are not divisible?");
55423   }
55424 
55425   unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits();
55426   unsigned NarrowElts = SelElts / NarrowingFactor;
55427   MVT NarrowSelVT = MVT::getVectorVT(SelVT.getVectorElementType(), NarrowElts);
55428   SDLoc DL(Ext);
55429   SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
55430   SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
55431   SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
55432   SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
55433   return DAG.getBitcast(VT, NarrowSel);
55434 }
55435 
55436 static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
55437                                         TargetLowering::DAGCombinerInfo &DCI,
55438                                         const X86Subtarget &Subtarget) {
55439   // For AVX1 only, if we are extracting from a 256-bit and+not (which will
55440   // eventually get combined/lowered into ANDNP) with a concatenated operand,
55441   // split the 'and' into 128-bit ops to avoid the concatenate and extract.
55442   // We let generic combining take over from there to simplify the
55443   // insert/extract and 'not'.
55444   // This pattern emerges during AVX1 legalization. We handle it before lowering
55445   // to avoid complications like splitting constant vector loads.
55446 
55447   // Capture the original wide type in the likely case that we need to bitcast
55448   // back to this type.
55449   if (!N->getValueType(0).isSimple())
55450     return SDValue();
55451 
55452   MVT VT = N->getSimpleValueType(0);
55453   SDValue InVec = N->getOperand(0);
55454   unsigned IdxVal = N->getConstantOperandVal(1);
55455   SDValue InVecBC = peekThroughBitcasts(InVec);
55456   EVT InVecVT = InVec.getValueType();
55457   unsigned SizeInBits = VT.getSizeInBits();
55458   unsigned InSizeInBits = InVecVT.getSizeInBits();
55459   unsigned NumSubElts = VT.getVectorNumElements();
55460   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55461 
55462   if (Subtarget.hasAVX() && !Subtarget.hasAVX2() &&
55463       TLI.isTypeLegal(InVecVT) &&
55464       InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) {
55465     auto isConcatenatedNot = [](SDValue V) {
55466       V = peekThroughBitcasts(V);
55467       if (!isBitwiseNot(V))
55468         return false;
55469       SDValue NotOp = V->getOperand(0);
55470       return peekThroughBitcasts(NotOp).getOpcode() == ISD::CONCAT_VECTORS;
55471     };
55472     if (isConcatenatedNot(InVecBC.getOperand(0)) ||
55473         isConcatenatedNot(InVecBC.getOperand(1))) {
55474       // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1
55475       SDValue Concat = splitVectorIntBinary(InVecBC, DAG);
55476       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,
55477                          DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
55478     }
55479   }
55480 
55481   if (DCI.isBeforeLegalizeOps())
55482     return SDValue();
55483 
55484   if (SDValue V = narrowExtractedVectorSelect(N, DAG))
55485     return V;
55486 
55487   if (ISD::isBuildVectorAllZeros(InVec.getNode()))
55488     return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
55489 
55490   if (ISD::isBuildVectorAllOnes(InVec.getNode())) {
55491     if (VT.getScalarType() == MVT::i1)
55492       return DAG.getConstant(1, SDLoc(N), VT);
55493     return getOnesVector(VT, DAG, SDLoc(N));
55494   }
55495 
55496   if (InVec.getOpcode() == ISD::BUILD_VECTOR)
55497     return DAG.getBuildVector(VT, SDLoc(N),
55498                               InVec->ops().slice(IdxVal, NumSubElts));
55499 
55500   // If we are extracting from an insert into a larger vector, replace with a
55501   // smaller insert if we don't access less than the original subvector. Don't
55502   // do this for i1 vectors.
55503   // TODO: Relax the matching indices requirement?
55504   if (VT.getVectorElementType() != MVT::i1 &&
55505       InVec.getOpcode() == ISD::INSERT_SUBVECTOR && InVec.hasOneUse() &&
55506       IdxVal == InVec.getConstantOperandVal(2) &&
55507       InVec.getOperand(1).getValueSizeInBits() <= SizeInBits) {
55508     SDLoc DL(N);
55509     SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
55510                                  InVec.getOperand(0), N->getOperand(1));
55511     unsigned NewIdxVal = InVec.getConstantOperandVal(2) - IdxVal;
55512     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt,
55513                        InVec.getOperand(1),
55514                        DAG.getVectorIdxConstant(NewIdxVal, DL));
55515   }
55516 
55517   // If we're extracting an upper subvector from a broadcast we should just
55518   // extract the lowest subvector instead which should allow
55519   // SimplifyDemandedVectorElts do more simplifications.
55520   if (IdxVal != 0 && (InVec.getOpcode() == X86ISD::VBROADCAST ||
55521                       InVec.getOpcode() == X86ISD::VBROADCAST_LOAD ||
55522                       DAG.isSplatValue(InVec, /*AllowUndefs*/ false)))
55523     return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits);
55524 
55525   // If we're extracting a broadcasted subvector, just use the lowest subvector.
55526   if (IdxVal != 0 && InVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
55527       cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT)
55528     return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits);
55529 
55530   // Attempt to extract from the source of a shuffle vector.
55531   if ((InSizeInBits % SizeInBits) == 0 && (IdxVal % NumSubElts) == 0) {
55532     SmallVector<int, 32> ShuffleMask;
55533     SmallVector<int, 32> ScaledMask;
55534     SmallVector<SDValue, 2> ShuffleInputs;
55535     unsigned NumSubVecs = InSizeInBits / SizeInBits;
55536     // Decode the shuffle mask and scale it so its shuffling subvectors.
55537     if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
55538         scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
55539       unsigned SubVecIdx = IdxVal / NumSubElts;
55540       if (ScaledMask[SubVecIdx] == SM_SentinelUndef)
55541         return DAG.getUNDEF(VT);
55542       if (ScaledMask[SubVecIdx] == SM_SentinelZero)
55543         return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
55544       SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
55545       if (Src.getValueSizeInBits() == InSizeInBits) {
55546         unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs;
55547         unsigned SrcEltIdx = SrcSubVecIdx * NumSubElts;
55548         return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
55549                                 SDLoc(N), SizeInBits);
55550       }
55551     }
55552   }
55553 
55554   // If we're extracting the lowest subvector and we're the only user,
55555   // we may be able to perform this with a smaller vector width.
55556   unsigned InOpcode = InVec.getOpcode();
55557   if (InVec.hasOneUse()) {
55558     if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) {
55559       // v2f64 CVTDQ2PD(v4i32).
55560       if (InOpcode == ISD::SINT_TO_FP &&
55561           InVec.getOperand(0).getValueType() == MVT::v4i32) {
55562         return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), VT, InVec.getOperand(0));
55563       }
55564       // v2f64 CVTUDQ2PD(v4i32).
55565       if (InOpcode == ISD::UINT_TO_FP && Subtarget.hasVLX() &&
55566           InVec.getOperand(0).getValueType() == MVT::v4i32) {
55567         return DAG.getNode(X86ISD::CVTUI2P, SDLoc(N), VT, InVec.getOperand(0));
55568       }
55569       // v2f64 CVTPS2PD(v4f32).
55570       if (InOpcode == ISD::FP_EXTEND &&
55571           InVec.getOperand(0).getValueType() == MVT::v4f32) {
55572         return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0));
55573       }
55574     }
55575     if (IdxVal == 0 &&
55576         (ISD::isExtOpcode(InOpcode) || ISD::isExtVecInRegOpcode(InOpcode)) &&
55577         (SizeInBits == 128 || SizeInBits == 256) &&
55578         InVec.getOperand(0).getValueSizeInBits() >= SizeInBits) {
55579       SDLoc DL(N);
55580       SDValue Ext = InVec.getOperand(0);
55581       if (Ext.getValueSizeInBits() > SizeInBits)
55582         Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits);
55583       unsigned ExtOp = DAG.getOpcode_EXTEND_VECTOR_INREG(InOpcode);
55584       return DAG.getNode(ExtOp, DL, VT, Ext);
55585     }
55586     if (IdxVal == 0 && InOpcode == ISD::VSELECT &&
55587         InVec.getOperand(0).getValueType().is256BitVector() &&
55588         InVec.getOperand(1).getValueType().is256BitVector() &&
55589         InVec.getOperand(2).getValueType().is256BitVector()) {
55590       SDLoc DL(N);
55591       SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
55592       SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
55593       SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
55594       return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
55595     }
55596     if (IdxVal == 0 && InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() &&
55597         (VT.is128BitVector() || VT.is256BitVector())) {
55598       SDLoc DL(N);
55599       SDValue InVecSrc = InVec.getOperand(0);
55600       unsigned Scale = InVecSrc.getValueSizeInBits() / InSizeInBits;
55601       SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
55602       return DAG.getNode(InOpcode, DL, VT, Ext);
55603     }
55604     if (InOpcode == X86ISD::MOVDDUP &&
55605         (VT.is128BitVector() || VT.is256BitVector())) {
55606       SDLoc DL(N);
55607       SDValue Ext0 =
55608           extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
55609       return DAG.getNode(InOpcode, DL, VT, Ext0);
55610     }
55611   }
55612 
55613   // Always split vXi64 logical shifts where we're extracting the upper 32-bits
55614   // as this is very likely to fold into a shuffle/truncation.
55615   if ((InOpcode == X86ISD::VSHLI || InOpcode == X86ISD::VSRLI) &&
55616       InVecVT.getScalarSizeInBits() == 64 &&
55617       InVec.getConstantOperandAPInt(1) == 32) {
55618     SDLoc DL(N);
55619     SDValue Ext =
55620         extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
55621     return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1));
55622   }
55623 
55624   return SDValue();
55625 }
55626 
55627 static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
55628   EVT VT = N->getValueType(0);
55629   SDValue Src = N->getOperand(0);
55630   SDLoc DL(N);
55631 
55632   // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
55633   // This occurs frequently in our masked scalar intrinsic code and our
55634   // floating point select lowering with AVX512.
55635   // TODO: SimplifyDemandedBits instead?
55636   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse() &&
55637       isOneConstant(Src.getOperand(1)))
55638     return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Src.getOperand(0));
55639 
55640   // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
55641   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
55642       Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
55643       Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
55644       isNullConstant(Src.getOperand(1)))
55645     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
55646                        Src.getOperand(1));
55647 
55648   // Reduce v2i64 to v4i32 if we don't need the upper bits or are known zero.
55649   // TODO: Move to DAGCombine/SimplifyDemandedBits?
55650   if ((VT == MVT::v2i64 || VT == MVT::v2f64) && Src.hasOneUse()) {
55651     auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) {
55652       if (Op.getValueType() != MVT::i64)
55653         return SDValue();
55654       unsigned Opc = IsZeroExt ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND;
55655       if (Op.getOpcode() == Opc &&
55656           Op.getOperand(0).getScalarValueSizeInBits() <= 32)
55657         return Op.getOperand(0);
55658       unsigned Ext = IsZeroExt ? ISD::ZEXTLOAD : ISD::EXTLOAD;
55659       if (auto *Ld = dyn_cast<LoadSDNode>(Op))
55660         if (Ld->getExtensionType() == Ext &&
55661             Ld->getMemoryVT().getScalarSizeInBits() <= 32)
55662           return Op;
55663       if (IsZeroExt) {
55664         KnownBits Known = DAG.computeKnownBits(Op);
55665         if (!Known.isConstant() && Known.countMinLeadingZeros() >= 32)
55666           return Op;
55667       }
55668       return SDValue();
55669     };
55670 
55671     if (SDValue AnyExt = IsExt64(peekThroughOneUseBitcasts(Src), false))
55672       return DAG.getBitcast(
55673           VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
55674                           DAG.getAnyExtOrTrunc(AnyExt, DL, MVT::i32)));
55675 
55676     if (SDValue ZeroExt = IsExt64(peekThroughOneUseBitcasts(Src), true))
55677       return DAG.getBitcast(
55678           VT,
55679           DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32,
55680                       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
55681                                   DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32))));
55682   }
55683 
55684   // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
55685   if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
55686       Src.getOperand(0).getValueType() == MVT::x86mmx)
55687     return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
55688 
55689   // See if we're broadcasting the scalar value, in which case just reuse that.
55690   // Ensure the same SDValue from the SDNode use is being used.
55691   if (VT.getScalarType() == Src.getValueType())
55692     for (SDNode *User : Src->uses())
55693       if (User->getOpcode() == X86ISD::VBROADCAST &&
55694           Src == User->getOperand(0)) {
55695         unsigned SizeInBits = VT.getFixedSizeInBits();
55696         unsigned BroadcastSizeInBits =
55697             User->getValueSizeInBits(0).getFixedValue();
55698         if (BroadcastSizeInBits == SizeInBits)
55699           return SDValue(User, 0);
55700         if (BroadcastSizeInBits > SizeInBits)
55701           return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
55702         // TODO: Handle BroadcastSizeInBits < SizeInBits when we have test
55703         // coverage.
55704       }
55705 
55706   return SDValue();
55707 }
55708 
55709 // Simplify PMULDQ and PMULUDQ operations.
55710 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
55711                              TargetLowering::DAGCombinerInfo &DCI,
55712                              const X86Subtarget &Subtarget) {
55713   SDValue LHS = N->getOperand(0);
55714   SDValue RHS = N->getOperand(1);
55715 
55716   // Canonicalize constant to RHS.
55717   if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) &&
55718       !DAG.isConstantIntBuildVectorOrConstantInt(RHS))
55719     return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS);
55720 
55721   // Multiply by zero.
55722   // Don't return RHS as it may contain UNDEFs.
55723   if (ISD::isBuildVectorAllZeros(RHS.getNode()))
55724     return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
55725 
55726   // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
55727   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55728   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(64), DCI))
55729     return SDValue(N, 0);
55730 
55731   // If the input is an extend_invec and the SimplifyDemandedBits call didn't
55732   // convert it to any_extend_invec, due to the LegalOperations check, do the
55733   // conversion directly to a vector shuffle manually. This exposes combine
55734   // opportunities missed by combineEXTEND_VECTOR_INREG not calling
55735   // combineX86ShufflesRecursively on SSE4.1 targets.
55736   // FIXME: This is basically a hack around several other issues related to
55737   // ANY_EXTEND_VECTOR_INREG.
55738   if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() &&
55739       (LHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
55740        LHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
55741       LHS.getOperand(0).getValueType() == MVT::v4i32) {
55742     SDLoc dl(N);
55743     LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
55744                                LHS.getOperand(0), { 0, -1, 1, -1 });
55745     LHS = DAG.getBitcast(MVT::v2i64, LHS);
55746     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
55747   }
55748   if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() &&
55749       (RHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
55750        RHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
55751       RHS.getOperand(0).getValueType() == MVT::v4i32) {
55752     SDLoc dl(N);
55753     RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
55754                                RHS.getOperand(0), { 0, -1, 1, -1 });
55755     RHS = DAG.getBitcast(MVT::v2i64, RHS);
55756     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
55757   }
55758 
55759   return SDValue();
55760 }
55761 
55762 // Simplify VPMADDUBSW/VPMADDWD operations.
55763 static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG,
55764                              TargetLowering::DAGCombinerInfo &DCI) {
55765   EVT VT = N->getValueType(0);
55766   SDValue LHS = N->getOperand(0);
55767   SDValue RHS = N->getOperand(1);
55768 
55769   // Multiply by zero.
55770   // Don't return LHS/RHS as it may contain UNDEFs.
55771   if (ISD::isBuildVectorAllZeros(LHS.getNode()) ||
55772       ISD::isBuildVectorAllZeros(RHS.getNode()))
55773     return DAG.getConstant(0, SDLoc(N), VT);
55774 
55775   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55776   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
55777   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
55778     return SDValue(N, 0);
55779 
55780   return SDValue();
55781 }
55782 
55783 static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
55784                                           TargetLowering::DAGCombinerInfo &DCI,
55785                                           const X86Subtarget &Subtarget) {
55786   EVT VT = N->getValueType(0);
55787   SDValue In = N->getOperand(0);
55788   unsigned Opcode = N->getOpcode();
55789   unsigned InOpcode = In.getOpcode();
55790   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55791   SDLoc DL(N);
55792 
55793   // Try to merge vector loads and extend_inreg to an extload.
55794   if (!DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(In.getNode()) &&
55795       In.hasOneUse()) {
55796     auto *Ld = cast<LoadSDNode>(In);
55797     if (Ld->isSimple()) {
55798       MVT SVT = In.getSimpleValueType().getVectorElementType();
55799       ISD::LoadExtType Ext = Opcode == ISD::SIGN_EXTEND_VECTOR_INREG
55800                                  ? ISD::SEXTLOAD
55801                                  : ISD::ZEXTLOAD;
55802       EVT MemVT = VT.changeVectorElementType(SVT);
55803       if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
55804         SDValue Load = DAG.getExtLoad(
55805             Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(),
55806             MemVT, Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags());
55807         DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
55808         return Load;
55809       }
55810     }
55811   }
55812 
55813   // Fold EXTEND_VECTOR_INREG(EXTEND_VECTOR_INREG(X)) -> EXTEND_VECTOR_INREG(X).
55814   if (Opcode == InOpcode)
55815     return DAG.getNode(Opcode, DL, VT, In.getOperand(0));
55816 
55817   // Fold EXTEND_VECTOR_INREG(EXTRACT_SUBVECTOR(EXTEND(X),0))
55818   // -> EXTEND_VECTOR_INREG(X).
55819   // TODO: Handle non-zero subvector indices.
55820   if (InOpcode == ISD::EXTRACT_SUBVECTOR && In.getConstantOperandVal(1) == 0 &&
55821       In.getOperand(0).getOpcode() == DAG.getOpcode_EXTEND(Opcode) &&
55822       In.getOperand(0).getOperand(0).getValueSizeInBits() ==
55823           In.getValueSizeInBits())
55824     return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0));
55825 
55826   // Fold EXTEND_VECTOR_INREG(BUILD_VECTOR(X,Y,?,?)) -> BUILD_VECTOR(X,0,Y,0).
55827   // TODO: Move to DAGCombine?
55828   if (!DCI.isBeforeLegalizeOps() && Opcode == ISD::ZERO_EXTEND_VECTOR_INREG &&
55829       In.getOpcode() == ISD::BUILD_VECTOR && In.hasOneUse() &&
55830       In.getValueSizeInBits() == VT.getSizeInBits()) {
55831     unsigned NumElts = VT.getVectorNumElements();
55832     unsigned Scale = VT.getScalarSizeInBits() / In.getScalarValueSizeInBits();
55833     EVT EltVT = In.getOperand(0).getValueType();
55834     SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT));
55835     for (unsigned I = 0; I != NumElts; ++I)
55836       Elts[I * Scale] = In.getOperand(I);
55837     return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts));
55838   }
55839 
55840   // Attempt to combine as a shuffle on SSE41+ targets.
55841   if (Subtarget.hasSSE41()) {
55842     SDValue Op(N, 0);
55843     if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType()))
55844       if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
55845         return Res;
55846   }
55847 
55848   return SDValue();
55849 }
55850 
55851 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
55852                              TargetLowering::DAGCombinerInfo &DCI) {
55853   EVT VT = N->getValueType(0);
55854 
55855   if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()))
55856     return DAG.getConstant(0, SDLoc(N), VT);
55857 
55858   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55859   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
55860   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
55861     return SDValue(N, 0);
55862 
55863   return SDValue();
55864 }
55865 
55866 // Optimize (fp16_to_fp (fp_to_fp16 X)) to VCVTPS2PH followed by VCVTPH2PS.
55867 // Done as a combine because the lowering for fp16_to_fp and fp_to_fp16 produce
55868 // extra instructions between the conversion due to going to scalar and back.
55869 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
55870                                  const X86Subtarget &Subtarget) {
55871   if (Subtarget.useSoftFloat() || !Subtarget.hasF16C())
55872     return SDValue();
55873 
55874   if (N->getOperand(0).getOpcode() != ISD::FP_TO_FP16)
55875     return SDValue();
55876 
55877   if (N->getValueType(0) != MVT::f32 ||
55878       N->getOperand(0).getOperand(0).getValueType() != MVT::f32)
55879     return SDValue();
55880 
55881   SDLoc dl(N);
55882   SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
55883                             N->getOperand(0).getOperand(0));
55884   Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
55885                     DAG.getTargetConstant(4, dl, MVT::i32));
55886   Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
55887   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
55888                      DAG.getIntPtrConstant(0, dl));
55889 }
55890 
55891 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
55892                                 const X86Subtarget &Subtarget) {
55893   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
55894     return SDValue();
55895 
55896   if (Subtarget.hasFP16())
55897     return SDValue();
55898 
55899   bool IsStrict = N->isStrictFPOpcode();
55900   EVT VT = N->getValueType(0);
55901   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
55902   EVT SrcVT = Src.getValueType();
55903 
55904   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16)
55905     return SDValue();
55906 
55907   if (VT.getVectorElementType() != MVT::f32 &&
55908       VT.getVectorElementType() != MVT::f64)
55909     return SDValue();
55910 
55911   unsigned NumElts = VT.getVectorNumElements();
55912   if (NumElts == 1 || !isPowerOf2_32(NumElts))
55913     return SDValue();
55914 
55915   SDLoc dl(N);
55916 
55917   // Convert the input to vXi16.
55918   EVT IntVT = SrcVT.changeVectorElementTypeToInteger();
55919   Src = DAG.getBitcast(IntVT, Src);
55920 
55921   // Widen to at least 8 input elements.
55922   if (NumElts < 8) {
55923     unsigned NumConcats = 8 / NumElts;
55924     SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
55925                                 : DAG.getConstant(0, dl, IntVT);
55926     SmallVector<SDValue, 4> Ops(NumConcats, Fill);
55927     Ops[0] = Src;
55928     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
55929   }
55930 
55931   // Destination is vXf32 with at least 4 elements.
55932   EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
55933                                std::max(4U, NumElts));
55934   SDValue Cvt, Chain;
55935   if (IsStrict) {
55936     Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
55937                       {N->getOperand(0), Src});
55938     Chain = Cvt.getValue(1);
55939   } else {
55940     Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src);
55941   }
55942 
55943   if (NumElts < 4) {
55944     assert(NumElts == 2 && "Unexpected size");
55945     Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt,
55946                       DAG.getIntPtrConstant(0, dl));
55947   }
55948 
55949   if (IsStrict) {
55950     // Extend to the original VT if necessary.
55951     if (Cvt.getValueType() != VT) {
55952       Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other},
55953                         {Chain, Cvt});
55954       Chain = Cvt.getValue(1);
55955     }
55956     return DAG.getMergeValues({Cvt, Chain}, dl);
55957   }
55958 
55959   // Extend to the original VT if necessary.
55960   return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt);
55961 }
55962 
55963 // Try to find a larger VBROADCAST_LOAD/SUBV_BROADCAST_LOAD that we can extract
55964 // from. Limit this to cases where the loads have the same input chain and the
55965 // output chains are unused. This avoids any memory ordering issues.
55966 static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
55967                                      TargetLowering::DAGCombinerInfo &DCI) {
55968   assert((N->getOpcode() == X86ISD::VBROADCAST_LOAD ||
55969           N->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) &&
55970          "Unknown broadcast load type");
55971 
55972   // Only do this if the chain result is unused.
55973   if (N->hasAnyUseOfValue(1))
55974     return SDValue();
55975 
55976   auto *MemIntrin = cast<MemIntrinsicSDNode>(N);
55977 
55978   SDValue Ptr = MemIntrin->getBasePtr();
55979   SDValue Chain = MemIntrin->getChain();
55980   EVT VT = N->getSimpleValueType(0);
55981   EVT MemVT = MemIntrin->getMemoryVT();
55982 
55983   // Look at other users of our base pointer and try to find a wider broadcast.
55984   // The input chain and the size of the memory VT must match.
55985   for (SDNode *User : Ptr->uses())
55986     if (User != N && User->getOpcode() == N->getOpcode() &&
55987         cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr &&
55988         cast<MemIntrinsicSDNode>(User)->getChain() == Chain &&
55989         cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() ==
55990             MemVT.getSizeInBits() &&
55991         !User->hasAnyUseOfValue(1) &&
55992         User->getValueSizeInBits(0).getFixedValue() > VT.getFixedSizeInBits()) {
55993       SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
55994                                          VT.getSizeInBits());
55995       Extract = DAG.getBitcast(VT, Extract);
55996       return DCI.CombineTo(N, Extract, SDValue(User, 1));
55997     }
55998 
55999   return SDValue();
56000 }
56001 
56002 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
56003                                const X86Subtarget &Subtarget) {
56004   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
56005     return SDValue();
56006 
56007   bool IsStrict = N->isStrictFPOpcode();
56008   EVT VT = N->getValueType(0);
56009   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
56010   EVT SrcVT = Src.getValueType();
56011 
56012   if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
56013       SrcVT.getVectorElementType() != MVT::f32)
56014     return SDValue();
56015 
56016   SDLoc dl(N);
56017 
56018   SDValue Cvt, Chain;
56019   unsigned NumElts = VT.getVectorNumElements();
56020   if (Subtarget.hasFP16()) {
56021     // Combine (v8f16 fp_round(concat_vectors(v4f32 (xint_to_fp v4i64), ..)))
56022     // into (v8f16 vector_shuffle(v8f16 (CVTXI2P v4i64), ..))
56023     if (NumElts == 8 && Src.getOpcode() == ISD::CONCAT_VECTORS) {
56024       SDValue Cvt0, Cvt1;
56025       SDValue Op0 = Src.getOperand(0);
56026       SDValue Op1 = Src.getOperand(1);
56027       bool IsOp0Strict = Op0->isStrictFPOpcode();
56028       if (Op0.getOpcode() != Op1.getOpcode() ||
56029           Op0.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64 ||
56030           Op1.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64) {
56031         return SDValue();
56032       }
56033       int Mask[8] = {0, 1, 2, 3, 8, 9, 10, 11};
56034       if (IsStrict) {
56035         assert(IsOp0Strict && "Op0 must be strict node");
56036         unsigned Opc = Op0.getOpcode() == ISD::STRICT_SINT_TO_FP
56037                            ? X86ISD::STRICT_CVTSI2P
56038                            : X86ISD::STRICT_CVTUI2P;
56039         Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
56040                            {Op0.getOperand(0), Op0.getOperand(1)});
56041         Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
56042                            {Op1.getOperand(0), Op1.getOperand(1)});
56043         Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
56044         return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl);
56045       }
56046       unsigned Opc = Op0.getOpcode() == ISD::SINT_TO_FP ? X86ISD::CVTSI2P
56047                                                         : X86ISD::CVTUI2P;
56048       Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0));
56049       Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0));
56050       return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
56051     }
56052     return SDValue();
56053   }
56054 
56055   if (NumElts == 1 || !isPowerOf2_32(NumElts))
56056     return SDValue();
56057 
56058   // Widen to at least 4 input elements.
56059   if (NumElts < 4)
56060     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
56061                       DAG.getConstantFP(0.0, dl, SrcVT));
56062 
56063   // Destination is v8i16 with at least 8 elements.
56064   EVT CvtVT =
56065       EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts));
56066   SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
56067   if (IsStrict) {
56068     Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},
56069                       {N->getOperand(0), Src, Rnd});
56070     Chain = Cvt.getValue(1);
56071   } else {
56072     Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd);
56073   }
56074 
56075   // Extract down to real number of elements.
56076   if (NumElts < 8) {
56077     EVT IntVT = VT.changeVectorElementTypeToInteger();
56078     Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt,
56079                       DAG.getIntPtrConstant(0, dl));
56080   }
56081 
56082   Cvt = DAG.getBitcast(VT, Cvt);
56083 
56084   if (IsStrict)
56085     return DAG.getMergeValues({Cvt, Chain}, dl);
56086 
56087   return Cvt;
56088 }
56089 
56090 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
56091   SDValue Src = N->getOperand(0);
56092 
56093   // Turn MOVDQ2Q+simple_load into an mmx load.
56094   if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
56095     LoadSDNode *LN = cast<LoadSDNode>(Src.getNode());
56096 
56097     if (LN->isSimple()) {
56098       SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
56099                                   LN->getBasePtr(),
56100                                   LN->getPointerInfo(),
56101                                   LN->getOriginalAlign(),
56102                                   LN->getMemOperand()->getFlags());
56103       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
56104       return NewLd;
56105     }
56106   }
56107 
56108   return SDValue();
56109 }
56110 
56111 static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG,
56112                            TargetLowering::DAGCombinerInfo &DCI) {
56113   unsigned NumBits = N->getSimpleValueType(0).getSizeInBits();
56114   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
56115   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBits), DCI))
56116     return SDValue(N, 0);
56117 
56118   return SDValue();
56119 }
56120 
56121 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
56122                                              DAGCombinerInfo &DCI) const {
56123   SelectionDAG &DAG = DCI.DAG;
56124   switch (N->getOpcode()) {
56125   default: break;
56126   case ISD::SCALAR_TO_VECTOR:
56127     return combineScalarToVector(N, DAG);
56128   case ISD::EXTRACT_VECTOR_ELT:
56129   case X86ISD::PEXTRW:
56130   case X86ISD::PEXTRB:
56131     return combineExtractVectorElt(N, DAG, DCI, Subtarget);
56132   case ISD::CONCAT_VECTORS:
56133     return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget);
56134   case ISD::INSERT_SUBVECTOR:
56135     return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget);
56136   case ISD::EXTRACT_SUBVECTOR:
56137     return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget);
56138   case ISD::VSELECT:
56139   case ISD::SELECT:
56140   case X86ISD::BLENDV:      return combineSelect(N, DAG, DCI, Subtarget);
56141   case ISD::BITCAST:        return combineBitcast(N, DAG, DCI, Subtarget);
56142   case X86ISD::CMOV:        return combineCMov(N, DAG, DCI, Subtarget);
56143   case X86ISD::CMP:         return combineCMP(N, DAG, Subtarget);
56144   case ISD::ADD:            return combineAdd(N, DAG, DCI, Subtarget);
56145   case ISD::SUB:            return combineSub(N, DAG, DCI, Subtarget);
56146   case X86ISD::ADD:
56147   case X86ISD::SUB:         return combineX86AddSub(N, DAG, DCI);
56148   case X86ISD::SBB:         return combineSBB(N, DAG);
56149   case X86ISD::ADC:         return combineADC(N, DAG, DCI);
56150   case ISD::MUL:            return combineMul(N, DAG, DCI, Subtarget);
56151   case ISD::SHL:            return combineShiftLeft(N, DAG);
56152   case ISD::SRA:            return combineShiftRightArithmetic(N, DAG, Subtarget);
56153   case ISD::SRL:            return combineShiftRightLogical(N, DAG, DCI, Subtarget);
56154   case ISD::AND:            return combineAnd(N, DAG, DCI, Subtarget);
56155   case ISD::OR:             return combineOr(N, DAG, DCI, Subtarget);
56156   case ISD::XOR:            return combineXor(N, DAG, DCI, Subtarget);
56157   case ISD::BITREVERSE:     return combineBITREVERSE(N, DAG, DCI, Subtarget);
56158   case X86ISD::BEXTR:
56159   case X86ISD::BEXTRI:      return combineBEXTR(N, DAG, DCI, Subtarget);
56160   case ISD::LOAD:           return combineLoad(N, DAG, DCI, Subtarget);
56161   case ISD::MLOAD:          return combineMaskedLoad(N, DAG, DCI, Subtarget);
56162   case ISD::STORE:          return combineStore(N, DAG, DCI, Subtarget);
56163   case ISD::MSTORE:         return combineMaskedStore(N, DAG, DCI, Subtarget);
56164   case X86ISD::VEXTRACT_STORE:
56165     return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget);
56166   case ISD::SINT_TO_FP:
56167   case ISD::STRICT_SINT_TO_FP:
56168     return combineSIntToFP(N, DAG, DCI, Subtarget);
56169   case ISD::UINT_TO_FP:
56170   case ISD::STRICT_UINT_TO_FP:
56171     return combineUIntToFP(N, DAG, Subtarget);
56172   case ISD::FADD:
56173   case ISD::FSUB:           return combineFaddFsub(N, DAG, Subtarget);
56174   case X86ISD::VFCMULC:
56175   case X86ISD::VFMULC:      return combineFMulcFCMulc(N, DAG, Subtarget);
56176   case ISD::FNEG:           return combineFneg(N, DAG, DCI, Subtarget);
56177   case ISD::TRUNCATE:       return combineTruncate(N, DAG, Subtarget);
56178   case X86ISD::VTRUNC:      return combineVTRUNC(N, DAG, DCI);
56179   case X86ISD::ANDNP:       return combineAndnp(N, DAG, DCI, Subtarget);
56180   case X86ISD::FAND:        return combineFAnd(N, DAG, Subtarget);
56181   case X86ISD::FANDN:       return combineFAndn(N, DAG, Subtarget);
56182   case X86ISD::FXOR:
56183   case X86ISD::FOR:         return combineFOr(N, DAG, DCI, Subtarget);
56184   case X86ISD::FMIN:
56185   case X86ISD::FMAX:        return combineFMinFMax(N, DAG);
56186   case ISD::FMINNUM:
56187   case ISD::FMAXNUM:        return combineFMinNumFMaxNum(N, DAG, Subtarget);
56188   case X86ISD::CVTSI2P:
56189   case X86ISD::CVTUI2P:     return combineX86INT_TO_FP(N, DAG, DCI);
56190   case X86ISD::CVTP2SI:
56191   case X86ISD::CVTP2UI:
56192   case X86ISD::STRICT_CVTTP2SI:
56193   case X86ISD::CVTTP2SI:
56194   case X86ISD::STRICT_CVTTP2UI:
56195   case X86ISD::CVTTP2UI:
56196                             return combineCVTP2I_CVTTP2I(N, DAG, DCI);
56197   case X86ISD::STRICT_CVTPH2PS:
56198   case X86ISD::CVTPH2PS:    return combineCVTPH2PS(N, DAG, DCI);
56199   case X86ISD::BT:          return combineBT(N, DAG, DCI);
56200   case ISD::ANY_EXTEND:
56201   case ISD::ZERO_EXTEND:    return combineZext(N, DAG, DCI, Subtarget);
56202   case ISD::SIGN_EXTEND:    return combineSext(N, DAG, DCI, Subtarget);
56203   case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget);
56204   case ISD::ANY_EXTEND_VECTOR_INREG:
56205   case ISD::SIGN_EXTEND_VECTOR_INREG:
56206   case ISD::ZERO_EXTEND_VECTOR_INREG:
56207     return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget);
56208   case ISD::SETCC:          return combineSetCC(N, DAG, DCI, Subtarget);
56209   case X86ISD::SETCC:       return combineX86SetCC(N, DAG, Subtarget);
56210   case X86ISD::BRCOND:      return combineBrCond(N, DAG, Subtarget);
56211   case X86ISD::PACKSS:
56212   case X86ISD::PACKUS:      return combineVectorPack(N, DAG, DCI, Subtarget);
56213   case X86ISD::HADD:
56214   case X86ISD::HSUB:
56215   case X86ISD::FHADD:
56216   case X86ISD::FHSUB:       return combineVectorHADDSUB(N, DAG, DCI, Subtarget);
56217   case X86ISD::VSHL:
56218   case X86ISD::VSRA:
56219   case X86ISD::VSRL:
56220     return combineVectorShiftVar(N, DAG, DCI, Subtarget);
56221   case X86ISD::VSHLI:
56222   case X86ISD::VSRAI:
56223   case X86ISD::VSRLI:
56224     return combineVectorShiftImm(N, DAG, DCI, Subtarget);
56225   case ISD::INSERT_VECTOR_ELT:
56226   case X86ISD::PINSRB:
56227   case X86ISD::PINSRW:      return combineVectorInsert(N, DAG, DCI, Subtarget);
56228   case X86ISD::SHUFP:       // Handle all target specific shuffles
56229   case X86ISD::INSERTPS:
56230   case X86ISD::EXTRQI:
56231   case X86ISD::INSERTQI:
56232   case X86ISD::VALIGN:
56233   case X86ISD::PALIGNR:
56234   case X86ISD::VSHLDQ:
56235   case X86ISD::VSRLDQ:
56236   case X86ISD::BLENDI:
56237   case X86ISD::UNPCKH:
56238   case X86ISD::UNPCKL:
56239   case X86ISD::MOVHLPS:
56240   case X86ISD::MOVLHPS:
56241   case X86ISD::PSHUFB:
56242   case X86ISD::PSHUFD:
56243   case X86ISD::PSHUFHW:
56244   case X86ISD::PSHUFLW:
56245   case X86ISD::MOVSHDUP:
56246   case X86ISD::MOVSLDUP:
56247   case X86ISD::MOVDDUP:
56248   case X86ISD::MOVSS:
56249   case X86ISD::MOVSD:
56250   case X86ISD::MOVSH:
56251   case X86ISD::VBROADCAST:
56252   case X86ISD::VPPERM:
56253   case X86ISD::VPERMI:
56254   case X86ISD::VPERMV:
56255   case X86ISD::VPERMV3:
56256   case X86ISD::VPERMIL2:
56257   case X86ISD::VPERMILPI:
56258   case X86ISD::VPERMILPV:
56259   case X86ISD::VPERM2X128:
56260   case X86ISD::SHUF128:
56261   case X86ISD::VZEXT_MOVL:
56262   case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget);
56263   case X86ISD::FMADD_RND:
56264   case X86ISD::FMSUB:
56265   case X86ISD::STRICT_FMSUB:
56266   case X86ISD::FMSUB_RND:
56267   case X86ISD::FNMADD:
56268   case X86ISD::STRICT_FNMADD:
56269   case X86ISD::FNMADD_RND:
56270   case X86ISD::FNMSUB:
56271   case X86ISD::STRICT_FNMSUB:
56272   case X86ISD::FNMSUB_RND:
56273   case ISD::FMA:
56274   case ISD::STRICT_FMA:     return combineFMA(N, DAG, DCI, Subtarget);
56275   case X86ISD::FMADDSUB_RND:
56276   case X86ISD::FMSUBADD_RND:
56277   case X86ISD::FMADDSUB:
56278   case X86ISD::FMSUBADD:    return combineFMADDSUB(N, DAG, DCI);
56279   case X86ISD::MOVMSK:      return combineMOVMSK(N, DAG, DCI, Subtarget);
56280   case X86ISD::TESTP:       return combineTESTP(N, DAG, DCI, Subtarget);
56281   case X86ISD::MGATHER:
56282   case X86ISD::MSCATTER:    return combineX86GatherScatter(N, DAG, DCI);
56283   case ISD::MGATHER:
56284   case ISD::MSCATTER:       return combineGatherScatter(N, DAG, DCI);
56285   case X86ISD::PCMPEQ:
56286   case X86ISD::PCMPGT:      return combineVectorCompare(N, DAG, Subtarget);
56287   case X86ISD::PMULDQ:
56288   case X86ISD::PMULUDQ:     return combinePMULDQ(N, DAG, DCI, Subtarget);
56289   case X86ISD::VPMADDUBSW:
56290   case X86ISD::VPMADDWD:    return combineVPMADD(N, DAG, DCI);
56291   case X86ISD::KSHIFTL:
56292   case X86ISD::KSHIFTR:     return combineKSHIFT(N, DAG, DCI);
56293   case ISD::FP16_TO_FP:     return combineFP16_TO_FP(N, DAG, Subtarget);
56294   case ISD::STRICT_FP_EXTEND:
56295   case ISD::FP_EXTEND:      return combineFP_EXTEND(N, DAG, Subtarget);
56296   case ISD::STRICT_FP_ROUND:
56297   case ISD::FP_ROUND:       return combineFP_ROUND(N, DAG, Subtarget);
56298   case X86ISD::VBROADCAST_LOAD:
56299   case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI);
56300   case X86ISD::MOVDQ2Q:     return combineMOVDQ2Q(N, DAG);
56301   case X86ISD::PDEP:        return combinePDEP(N, DAG, DCI);
56302   }
56303 
56304   return SDValue();
56305 }
56306 
56307 bool X86TargetLowering::preferABDSToABSWithNSW(EVT VT) const {
56308   return false;
56309 }
56310 
56311 // Prefer (non-AVX512) vector TRUNCATE(SIGN_EXTEND_INREG(X)) to use of PACKSS.
56312 bool X86TargetLowering::preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
56313                                                   EVT ExtVT) const {
56314   return Subtarget.hasAVX512() || !VT.isVector();
56315 }
56316 
56317 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
56318   if (!isTypeLegal(VT))
56319     return false;
56320 
56321   // There are no vXi8 shifts.
56322   if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)
56323     return false;
56324 
56325   // TODO: Almost no 8-bit ops are desirable because they have no actual
56326   //       size/speed advantages vs. 32-bit ops, but they do have a major
56327   //       potential disadvantage by causing partial register stalls.
56328   //
56329   // 8-bit multiply/shl is probably not cheaper than 32-bit multiply/shl, and
56330   // we have specializations to turn 32-bit multiply/shl into LEA or other ops.
56331   // Also, see the comment in "IsDesirableToPromoteOp" - where we additionally
56332   // check for a constant operand to the multiply.
56333   if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8)
56334     return false;
56335 
56336   // i16 instruction encodings are longer and some i16 instructions are slow,
56337   // so those are not desirable.
56338   if (VT == MVT::i16) {
56339     switch (Opc) {
56340     default:
56341       break;
56342     case ISD::LOAD:
56343     case ISD::SIGN_EXTEND:
56344     case ISD::ZERO_EXTEND:
56345     case ISD::ANY_EXTEND:
56346     case ISD::SHL:
56347     case ISD::SRA:
56348     case ISD::SRL:
56349     case ISD::SUB:
56350     case ISD::ADD:
56351     case ISD::MUL:
56352     case ISD::AND:
56353     case ISD::OR:
56354     case ISD::XOR:
56355       return false;
56356     }
56357   }
56358 
56359   // Any legal type not explicitly accounted for above here is desirable.
56360   return true;
56361 }
56362 
56363 SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc &dl,
56364                                                   SDValue Value, SDValue Addr,
56365                                                   int JTI,
56366                                                   SelectionDAG &DAG) const {
56367   const Module *M = DAG.getMachineFunction().getMMI().getModule();
56368   Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
56369   if (IsCFProtectionSupported) {
56370     // In case control-flow branch protection is enabled, we need to add
56371     // notrack prefix to the indirect branch.
56372     // In order to do that we create NT_BRIND SDNode.
56373     // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix.
56374     SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl);
56375     return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, JTInfo, Addr);
56376   }
56377 
56378   return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
56379 }
56380 
56381 TargetLowering::AndOrSETCCFoldKind
56382 X86TargetLowering::isDesirableToCombineLogicOpOfSETCC(
56383     const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const {
56384   using AndOrSETCCFoldKind = TargetLowering::AndOrSETCCFoldKind;
56385   EVT VT = LogicOp->getValueType(0);
56386   EVT OpVT = SETCC0->getOperand(0).getValueType();
56387   if (!VT.isInteger())
56388     return AndOrSETCCFoldKind::None;
56389 
56390   if (VT.isVector())
56391     return AndOrSETCCFoldKind(AndOrSETCCFoldKind::NotAnd |
56392                               (isOperationLegal(ISD::ABS, OpVT)
56393                                    ? AndOrSETCCFoldKind::ABS
56394                                    : AndOrSETCCFoldKind::None));
56395 
56396   // Don't use `NotAnd` as even though `not` is generally shorter code size than
56397   // `add`, `add` can lower to LEA which can save moves / spills. Any case where
56398   // `NotAnd` applies, `AddAnd` does as well.
56399   // TODO: Currently we lower (icmp eq/ne (and ~X, Y), 0) -> `test (not X), Y`,
56400   // if we change that to `andn Y, X` it may be worth prefering `NotAnd` here.
56401   return AndOrSETCCFoldKind::AddAnd;
56402 }
56403 
56404 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
56405   EVT VT = Op.getValueType();
56406   bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&
56407                              isa<ConstantSDNode>(Op.getOperand(1));
56408 
56409   // i16 is legal, but undesirable since i16 instruction encodings are longer
56410   // and some i16 instructions are slow.
56411   // 8-bit multiply-by-constant can usually be expanded to something cheaper
56412   // using LEA and/or other ALU ops.
56413   if (VT != MVT::i16 && !Is8BitMulByConstant)
56414     return false;
56415 
56416   auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
56417     if (!Op.hasOneUse())
56418       return false;
56419     SDNode *User = *Op->use_begin();
56420     if (!ISD::isNormalStore(User))
56421       return false;
56422     auto *Ld = cast<LoadSDNode>(Load);
56423     auto *St = cast<StoreSDNode>(User);
56424     return Ld->getBasePtr() == St->getBasePtr();
56425   };
56426 
56427   auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
56428     if (!Load.hasOneUse() || Load.getOpcode() != ISD::ATOMIC_LOAD)
56429       return false;
56430     if (!Op.hasOneUse())
56431       return false;
56432     SDNode *User = *Op->use_begin();
56433     if (User->getOpcode() != ISD::ATOMIC_STORE)
56434       return false;
56435     auto *Ld = cast<AtomicSDNode>(Load);
56436     auto *St = cast<AtomicSDNode>(User);
56437     return Ld->getBasePtr() == St->getBasePtr();
56438   };
56439 
56440   bool Commute = false;
56441   switch (Op.getOpcode()) {
56442   default: return false;
56443   case ISD::SIGN_EXTEND:
56444   case ISD::ZERO_EXTEND:
56445   case ISD::ANY_EXTEND:
56446     break;
56447   case ISD::SHL:
56448   case ISD::SRA:
56449   case ISD::SRL: {
56450     SDValue N0 = Op.getOperand(0);
56451     // Look out for (store (shl (load), x)).
56452     if (X86::mayFoldLoad(N0, Subtarget) && IsFoldableRMW(N0, Op))
56453       return false;
56454     break;
56455   }
56456   case ISD::ADD:
56457   case ISD::MUL:
56458   case ISD::AND:
56459   case ISD::OR:
56460   case ISD::XOR:
56461     Commute = true;
56462     [[fallthrough]];
56463   case ISD::SUB: {
56464     SDValue N0 = Op.getOperand(0);
56465     SDValue N1 = Op.getOperand(1);
56466     // Avoid disabling potential load folding opportunities.
56467     if (X86::mayFoldLoad(N1, Subtarget) &&
56468         (!Commute || !isa<ConstantSDNode>(N0) ||
56469          (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N1, Op))))
56470       return false;
56471     if (X86::mayFoldLoad(N0, Subtarget) &&
56472         ((Commute && !isa<ConstantSDNode>(N1)) ||
56473          (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
56474       return false;
56475     if (IsFoldableAtomicRMW(N0, Op) ||
56476         (Commute && IsFoldableAtomicRMW(N1, Op)))
56477       return false;
56478   }
56479   }
56480 
56481   PVT = MVT::i32;
56482   return true;
56483 }
56484 
56485 //===----------------------------------------------------------------------===//
56486 //                           X86 Inline Assembly Support
56487 //===----------------------------------------------------------------------===//
56488 
56489 // Helper to match a string separated by whitespace.
56490 static bool matchAsm(StringRef S, ArrayRef<const char *> Pieces) {
56491   S = S.substr(S.find_first_not_of(" \t")); // Skip leading whitespace.
56492 
56493   for (StringRef Piece : Pieces) {
56494     if (!S.starts_with(Piece)) // Check if the piece matches.
56495       return false;
56496 
56497     S = S.substr(Piece.size());
56498     StringRef::size_type Pos = S.find_first_not_of(" \t");
56499     if (Pos == 0) // We matched a prefix.
56500       return false;
56501 
56502     S = S.substr(Pos);
56503   }
56504 
56505   return S.empty();
56506 }
56507 
56508 static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
56509 
56510   if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
56511     if (llvm::is_contained(AsmPieces, "~{cc}") &&
56512         llvm::is_contained(AsmPieces, "~{flags}") &&
56513         llvm::is_contained(AsmPieces, "~{fpsr}")) {
56514 
56515       if (AsmPieces.size() == 3)
56516         return true;
56517       else if (llvm::is_contained(AsmPieces, "~{dirflag}"))
56518         return true;
56519     }
56520   }
56521   return false;
56522 }
56523 
56524 bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
56525   InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand());
56526 
56527   const std::string &AsmStr = IA->getAsmString();
56528 
56529   IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
56530   if (!Ty || Ty->getBitWidth() % 16 != 0)
56531     return false;
56532 
56533   // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
56534   SmallVector<StringRef, 4> AsmPieces;
56535   SplitString(AsmStr, AsmPieces, ";\n");
56536 
56537   switch (AsmPieces.size()) {
56538   default: return false;
56539   case 1:
56540     // FIXME: this should verify that we are targeting a 486 or better.  If not,
56541     // we will turn this bswap into something that will be lowered to logical
56542     // ops instead of emitting the bswap asm.  For now, we don't support 486 or
56543     // lower so don't worry about this.
56544     // bswap $0
56545     if (matchAsm(AsmPieces[0], {"bswap", "$0"}) ||
56546         matchAsm(AsmPieces[0], {"bswapl", "$0"}) ||
56547         matchAsm(AsmPieces[0], {"bswapq", "$0"}) ||
56548         matchAsm(AsmPieces[0], {"bswap", "${0:q}"}) ||
56549         matchAsm(AsmPieces[0], {"bswapl", "${0:q}"}) ||
56550         matchAsm(AsmPieces[0], {"bswapq", "${0:q}"})) {
56551       // No need to check constraints, nothing other than the equivalent of
56552       // "=r,0" would be valid here.
56553       return IntrinsicLowering::LowerToByteSwap(CI);
56554     }
56555 
56556     // rorw $$8, ${0:w}  -->  llvm.bswap.i16
56557     if (CI->getType()->isIntegerTy(16) &&
56558         IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
56559         (matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) ||
56560          matchAsm(AsmPieces[0], {"rolw", "$$8,", "${0:w}"}))) {
56561       AsmPieces.clear();
56562       StringRef ConstraintsStr = IA->getConstraintString();
56563       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
56564       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
56565       if (clobbersFlagRegisters(AsmPieces))
56566         return IntrinsicLowering::LowerToByteSwap(CI);
56567     }
56568     break;
56569   case 3:
56570     if (CI->getType()->isIntegerTy(32) &&
56571         IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
56572         matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) &&
56573         matchAsm(AsmPieces[1], {"rorl", "$$16,", "$0"}) &&
56574         matchAsm(AsmPieces[2], {"rorw", "$$8,", "${0:w}"})) {
56575       AsmPieces.clear();
56576       StringRef ConstraintsStr = IA->getConstraintString();
56577       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
56578       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
56579       if (clobbersFlagRegisters(AsmPieces))
56580         return IntrinsicLowering::LowerToByteSwap(CI);
56581     }
56582 
56583     if (CI->getType()->isIntegerTy(64)) {
56584       InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
56585       if (Constraints.size() >= 2 &&
56586           Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
56587           Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
56588         // bswap %eax / bswap %edx / xchgl %eax, %edx  -> llvm.bswap.i64
56589         if (matchAsm(AsmPieces[0], {"bswap", "%eax"}) &&
56590             matchAsm(AsmPieces[1], {"bswap", "%edx"}) &&
56591             matchAsm(AsmPieces[2], {"xchgl", "%eax,", "%edx"}))
56592           return IntrinsicLowering::LowerToByteSwap(CI);
56593       }
56594     }
56595     break;
56596   }
56597   return false;
56598 }
56599 
56600 static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
56601   X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
56602                            .Case("{@cca}", X86::COND_A)
56603                            .Case("{@ccae}", X86::COND_AE)
56604                            .Case("{@ccb}", X86::COND_B)
56605                            .Case("{@ccbe}", X86::COND_BE)
56606                            .Case("{@ccc}", X86::COND_B)
56607                            .Case("{@cce}", X86::COND_E)
56608                            .Case("{@ccz}", X86::COND_E)
56609                            .Case("{@ccg}", X86::COND_G)
56610                            .Case("{@ccge}", X86::COND_GE)
56611                            .Case("{@ccl}", X86::COND_L)
56612                            .Case("{@ccle}", X86::COND_LE)
56613                            .Case("{@ccna}", X86::COND_BE)
56614                            .Case("{@ccnae}", X86::COND_B)
56615                            .Case("{@ccnb}", X86::COND_AE)
56616                            .Case("{@ccnbe}", X86::COND_A)
56617                            .Case("{@ccnc}", X86::COND_AE)
56618                            .Case("{@ccne}", X86::COND_NE)
56619                            .Case("{@ccnz}", X86::COND_NE)
56620                            .Case("{@ccng}", X86::COND_LE)
56621                            .Case("{@ccnge}", X86::COND_L)
56622                            .Case("{@ccnl}", X86::COND_GE)
56623                            .Case("{@ccnle}", X86::COND_G)
56624                            .Case("{@ccno}", X86::COND_NO)
56625                            .Case("{@ccnp}", X86::COND_NP)
56626                            .Case("{@ccns}", X86::COND_NS)
56627                            .Case("{@cco}", X86::COND_O)
56628                            .Case("{@ccp}", X86::COND_P)
56629                            .Case("{@ccs}", X86::COND_S)
56630                            .Default(X86::COND_INVALID);
56631   return Cond;
56632 }
56633 
56634 /// Given a constraint letter, return the type of constraint for this target.
56635 X86TargetLowering::ConstraintType
56636 X86TargetLowering::getConstraintType(StringRef Constraint) const {
56637   if (Constraint.size() == 1) {
56638     switch (Constraint[0]) {
56639     case 'R':
56640     case 'q':
56641     case 'Q':
56642     case 'f':
56643     case 't':
56644     case 'u':
56645     case 'y':
56646     case 'x':
56647     case 'v':
56648     case 'l':
56649     case 'k': // AVX512 masking registers.
56650       return C_RegisterClass;
56651     case 'a':
56652     case 'b':
56653     case 'c':
56654     case 'd':
56655     case 'S':
56656     case 'D':
56657     case 'A':
56658       return C_Register;
56659     case 'I':
56660     case 'J':
56661     case 'K':
56662     case 'N':
56663     case 'G':
56664     case 'L':
56665     case 'M':
56666       return C_Immediate;
56667     case 'C':
56668     case 'e':
56669     case 'Z':
56670       return C_Other;
56671     default:
56672       break;
56673     }
56674   }
56675   else if (Constraint.size() == 2) {
56676     switch (Constraint[0]) {
56677     default:
56678       break;
56679     case 'Y':
56680       switch (Constraint[1]) {
56681       default:
56682         break;
56683       case 'z':
56684         return C_Register;
56685       case 'i':
56686       case 'm':
56687       case 'k':
56688       case 't':
56689       case '2':
56690         return C_RegisterClass;
56691       }
56692     }
56693   } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
56694     return C_Other;
56695   return TargetLowering::getConstraintType(Constraint);
56696 }
56697 
56698 /// Examine constraint type and operand type and determine a weight value.
56699 /// This object must already have been set up with the operand type
56700 /// and the current alternative constraint selected.
56701 TargetLowering::ConstraintWeight
56702 X86TargetLowering::getSingleConstraintMatchWeight(
56703     AsmOperandInfo &Info, const char *Constraint) const {
56704   ConstraintWeight Wt = CW_Invalid;
56705   Value *CallOperandVal = Info.CallOperandVal;
56706   // If we don't have a value, we can't do a match,
56707   // but allow it at the lowest weight.
56708   if (!CallOperandVal)
56709     return CW_Default;
56710   Type *Ty = CallOperandVal->getType();
56711   // Look at the constraint type.
56712   switch (*Constraint) {
56713   default:
56714     Wt = TargetLowering::getSingleConstraintMatchWeight(Info, Constraint);
56715     [[fallthrough]];
56716   case 'R':
56717   case 'q':
56718   case 'Q':
56719   case 'a':
56720   case 'b':
56721   case 'c':
56722   case 'd':
56723   case 'S':
56724   case 'D':
56725   case 'A':
56726     if (CallOperandVal->getType()->isIntegerTy())
56727       Wt = CW_SpecificReg;
56728     break;
56729   case 'f':
56730   case 't':
56731   case 'u':
56732     if (Ty->isFloatingPointTy())
56733       Wt = CW_SpecificReg;
56734     break;
56735   case 'y':
56736     if (Ty->isX86_MMXTy() && Subtarget.hasMMX())
56737       Wt = CW_SpecificReg;
56738     break;
56739   case 'Y':
56740     if (StringRef(Constraint).size() != 2)
56741       break;
56742     switch (Constraint[1]) {
56743     default:
56744       return CW_Invalid;
56745     // XMM0
56746     case 'z':
56747       if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
56748           ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()) ||
56749           ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512()))
56750         return CW_SpecificReg;
56751       return CW_Invalid;
56752     // Conditional OpMask regs (AVX512)
56753     case 'k':
56754       if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
56755         return CW_Register;
56756       return CW_Invalid;
56757     // Any MMX reg
56758     case 'm':
56759       if (Ty->isX86_MMXTy() && Subtarget.hasMMX())
56760         return Wt;
56761       return CW_Invalid;
56762     // Any SSE reg when ISA >= SSE2, same as 'x'
56763     case 'i':
56764     case 't':
56765     case '2':
56766       if (!Subtarget.hasSSE2())
56767         return CW_Invalid;
56768       break;
56769     }
56770     break;
56771   case 'v':
56772     if ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())
56773       Wt = CW_Register;
56774     [[fallthrough]];
56775   case 'x':
56776     if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
56777         ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()))
56778       Wt = CW_Register;
56779     break;
56780   case 'k':
56781     // Enable conditional vector operations using %k<#> registers.
56782     if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
56783       Wt = CW_Register;
56784     break;
56785   case 'I':
56786     if (auto *C = dyn_cast<ConstantInt>(Info.CallOperandVal))
56787       if (C->getZExtValue() <= 31)
56788         Wt = CW_Constant;
56789     break;
56790   case 'J':
56791     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56792       if (C->getZExtValue() <= 63)
56793         Wt = CW_Constant;
56794     break;
56795   case 'K':
56796     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56797       if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
56798         Wt = CW_Constant;
56799     break;
56800   case 'L':
56801     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56802       if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
56803         Wt = CW_Constant;
56804     break;
56805   case 'M':
56806     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56807       if (C->getZExtValue() <= 3)
56808         Wt = CW_Constant;
56809     break;
56810   case 'N':
56811     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56812       if (C->getZExtValue() <= 0xff)
56813         Wt = CW_Constant;
56814     break;
56815   case 'G':
56816   case 'C':
56817     if (isa<ConstantFP>(CallOperandVal))
56818       Wt = CW_Constant;
56819     break;
56820   case 'e':
56821     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56822       if ((C->getSExtValue() >= -0x80000000LL) &&
56823           (C->getSExtValue() <= 0x7fffffffLL))
56824         Wt = CW_Constant;
56825     break;
56826   case 'Z':
56827     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56828       if (C->getZExtValue() <= 0xffffffff)
56829         Wt = CW_Constant;
56830     break;
56831   }
56832   return Wt;
56833 }
56834 
56835 /// Try to replace an X constraint, which matches anything, with another that
56836 /// has more specific requirements based on the type of the corresponding
56837 /// operand.
56838 const char *X86TargetLowering::
56839 LowerXConstraint(EVT ConstraintVT) const {
56840   // FP X constraints get lowered to SSE1/2 registers if available, otherwise
56841   // 'f' like normal targets.
56842   if (ConstraintVT.isFloatingPoint()) {
56843     if (Subtarget.hasSSE1())
56844       return "x";
56845   }
56846 
56847   return TargetLowering::LowerXConstraint(ConstraintVT);
56848 }
56849 
56850 // Lower @cc targets via setcc.
56851 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
56852     SDValue &Chain, SDValue &Glue, const SDLoc &DL,
56853     const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
56854   X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
56855   if (Cond == X86::COND_INVALID)
56856     return SDValue();
56857   // Check that return type is valid.
56858   if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
56859       OpInfo.ConstraintVT.getSizeInBits() < 8)
56860     report_fatal_error("Glue output operand is of invalid type");
56861 
56862   // Get EFLAGS register. Only update chain when copyfrom is glued.
56863   if (Glue.getNode()) {
56864     Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue);
56865     Chain = Glue.getValue(1);
56866   } else
56867     Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
56868   // Extract CC code.
56869   SDValue CC = getSETCC(Cond, Glue, DL, DAG);
56870   // Extend to 32-bits
56871   SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
56872 
56873   return Result;
56874 }
56875 
56876 /// Lower the specified operand into the Ops vector.
56877 /// If it is invalid, don't add anything to Ops.
56878 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
56879                                                      StringRef Constraint,
56880                                                      std::vector<SDValue> &Ops,
56881                                                      SelectionDAG &DAG) const {
56882   SDValue Result;
56883 
56884   // Only support length 1 constraints for now.
56885   if (Constraint.size() > 1)
56886     return;
56887 
56888   char ConstraintLetter = Constraint[0];
56889   switch (ConstraintLetter) {
56890   default: break;
56891   case 'I':
56892     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56893       if (C->getZExtValue() <= 31) {
56894         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56895                                        Op.getValueType());
56896         break;
56897       }
56898     }
56899     return;
56900   case 'J':
56901     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56902       if (C->getZExtValue() <= 63) {
56903         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56904                                        Op.getValueType());
56905         break;
56906       }
56907     }
56908     return;
56909   case 'K':
56910     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56911       if (isInt<8>(C->getSExtValue())) {
56912         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56913                                        Op.getValueType());
56914         break;
56915       }
56916     }
56917     return;
56918   case 'L':
56919     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56920       if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
56921           (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) {
56922         Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
56923                                        Op.getValueType());
56924         break;
56925       }
56926     }
56927     return;
56928   case 'M':
56929     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56930       if (C->getZExtValue() <= 3) {
56931         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56932                                        Op.getValueType());
56933         break;
56934       }
56935     }
56936     return;
56937   case 'N':
56938     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56939       if (C->getZExtValue() <= 255) {
56940         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56941                                        Op.getValueType());
56942         break;
56943       }
56944     }
56945     return;
56946   case 'O':
56947     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56948       if (C->getZExtValue() <= 127) {
56949         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56950                                        Op.getValueType());
56951         break;
56952       }
56953     }
56954     return;
56955   case 'e': {
56956     // 32-bit signed value
56957     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56958       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
56959                                            C->getSExtValue())) {
56960         // Widen to 64 bits here to get it sign extended.
56961         Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
56962         break;
56963       }
56964     // FIXME gcc accepts some relocatable values here too, but only in certain
56965     // memory models; it's complicated.
56966     }
56967     return;
56968   }
56969   case 'Z': {
56970     // 32-bit unsigned value
56971     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56972       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
56973                                            C->getZExtValue())) {
56974         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56975                                        Op.getValueType());
56976         break;
56977       }
56978     }
56979     // FIXME gcc accepts some relocatable values here too, but only in certain
56980     // memory models; it's complicated.
56981     return;
56982   }
56983   case 'i': {
56984     // Literal immediates are always ok.
56985     if (auto *CST = dyn_cast<ConstantSDNode>(Op)) {
56986       bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
56987       BooleanContent BCont = getBooleanContents(MVT::i64);
56988       ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
56989                                     : ISD::SIGN_EXTEND;
56990       int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? CST->getZExtValue()
56991                                                   : CST->getSExtValue();
56992       Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);
56993       break;
56994     }
56995 
56996     // In any sort of PIC mode addresses need to be computed at runtime by
56997     // adding in a register or some sort of table lookup.  These can't
56998     // be used as immediates. BlockAddresses and BasicBlocks are fine though.
56999     if ((Subtarget.isPICStyleGOT() || Subtarget.isPICStyleStubPIC()) &&
57000         !(isa<BlockAddressSDNode>(Op) || isa<BasicBlockSDNode>(Op)))
57001       return;
57002 
57003     // If we are in non-pic codegen mode, we allow the address of a global (with
57004     // an optional displacement) to be used with 'i'.
57005     if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
57006       // If we require an extra load to get this address, as in PIC mode, we
57007       // can't accept it.
57008       if (isGlobalStubReference(
57009               Subtarget.classifyGlobalReference(GA->getGlobal())))
57010         return;
57011     break;
57012   }
57013   }
57014 
57015   if (Result.getNode()) {
57016     Ops.push_back(Result);
57017     return;
57018   }
57019   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
57020 }
57021 
57022 /// Check if \p RC is a general purpose register class.
57023 /// I.e., GR* or one of their variant.
57024 static bool isGRClass(const TargetRegisterClass &RC) {
57025   return RC.hasSuperClassEq(&X86::GR8RegClass) ||
57026          RC.hasSuperClassEq(&X86::GR16RegClass) ||
57027          RC.hasSuperClassEq(&X86::GR32RegClass) ||
57028          RC.hasSuperClassEq(&X86::GR64RegClass) ||
57029          RC.hasSuperClassEq(&X86::LOW32_ADDR_ACCESS_RBPRegClass);
57030 }
57031 
57032 /// Check if \p RC is a vector register class.
57033 /// I.e., FR* / VR* or one of their variant.
57034 static bool isFRClass(const TargetRegisterClass &RC) {
57035   return RC.hasSuperClassEq(&X86::FR16XRegClass) ||
57036          RC.hasSuperClassEq(&X86::FR32XRegClass) ||
57037          RC.hasSuperClassEq(&X86::FR64XRegClass) ||
57038          RC.hasSuperClassEq(&X86::VR128XRegClass) ||
57039          RC.hasSuperClassEq(&X86::VR256XRegClass) ||
57040          RC.hasSuperClassEq(&X86::VR512RegClass);
57041 }
57042 
57043 /// Check if \p RC is a mask register class.
57044 /// I.e., VK* or one of their variant.
57045 static bool isVKClass(const TargetRegisterClass &RC) {
57046   return RC.hasSuperClassEq(&X86::VK1RegClass) ||
57047          RC.hasSuperClassEq(&X86::VK2RegClass) ||
57048          RC.hasSuperClassEq(&X86::VK4RegClass) ||
57049          RC.hasSuperClassEq(&X86::VK8RegClass) ||
57050          RC.hasSuperClassEq(&X86::VK16RegClass) ||
57051          RC.hasSuperClassEq(&X86::VK32RegClass) ||
57052          RC.hasSuperClassEq(&X86::VK64RegClass);
57053 }
57054 
57055 std::pair<unsigned, const TargetRegisterClass *>
57056 X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
57057                                                 StringRef Constraint,
57058                                                 MVT VT) const {
57059   // First, see if this is a constraint that directly corresponds to an LLVM
57060   // register class.
57061   if (Constraint.size() == 1) {
57062     // GCC Constraint Letters
57063     switch (Constraint[0]) {
57064     default: break;
57065     // 'A' means [ER]AX + [ER]DX.
57066     case 'A':
57067       if (Subtarget.is64Bit())
57068         return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
57069       assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&
57070              "Expecting 64, 32 or 16 bit subtarget");
57071       return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
57072 
57073       // TODO: Slight differences here in allocation order and leaving
57074       // RIP in the class. Do they matter any more here than they do
57075       // in the normal allocation?
57076     case 'k':
57077       if (Subtarget.hasAVX512()) {
57078         if (VT == MVT::i1)
57079           return std::make_pair(0U, &X86::VK1RegClass);
57080         if (VT == MVT::i8)
57081           return std::make_pair(0U, &X86::VK8RegClass);
57082         if (VT == MVT::i16)
57083           return std::make_pair(0U, &X86::VK16RegClass);
57084       }
57085       if (Subtarget.hasBWI()) {
57086         if (VT == MVT::i32)
57087           return std::make_pair(0U, &X86::VK32RegClass);
57088         if (VT == MVT::i64)
57089           return std::make_pair(0U, &X86::VK64RegClass);
57090       }
57091       break;
57092     case 'q':   // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
57093       if (Subtarget.is64Bit()) {
57094         if (VT == MVT::i8 || VT == MVT::i1)
57095           return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
57096         if (VT == MVT::i16)
57097           return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
57098         if (VT == MVT::i32 || VT == MVT::f32)
57099           return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
57100         if (VT != MVT::f80 && !VT.isVector())
57101           return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
57102         break;
57103       }
57104       [[fallthrough]];
57105       // 32-bit fallthrough
57106     case 'Q':   // Q_REGS
57107       if (VT == MVT::i8 || VT == MVT::i1)
57108         return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
57109       if (VT == MVT::i16)
57110         return std::make_pair(0U, &X86::GR16_ABCDRegClass);
57111       if (VT == MVT::i32 || VT == MVT::f32 ||
57112           (!VT.isVector() && !Subtarget.is64Bit()))
57113         return std::make_pair(0U, &X86::GR32_ABCDRegClass);
57114       if (VT != MVT::f80 && !VT.isVector())
57115         return std::make_pair(0U, &X86::GR64_ABCDRegClass);
57116       break;
57117     case 'r':   // GENERAL_REGS
57118     case 'l':   // INDEX_REGS
57119       if (VT == MVT::i8 || VT == MVT::i1)
57120         return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
57121       if (VT == MVT::i16)
57122         return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
57123       if (VT == MVT::i32 || VT == MVT::f32 ||
57124           (!VT.isVector() && !Subtarget.is64Bit()))
57125         return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
57126       if (VT != MVT::f80 && !VT.isVector())
57127         return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
57128       break;
57129     case 'R':   // LEGACY_REGS
57130       if (VT == MVT::i8 || VT == MVT::i1)
57131         return std::make_pair(0U, &X86::GR8_NOREXRegClass);
57132       if (VT == MVT::i16)
57133         return std::make_pair(0U, &X86::GR16_NOREXRegClass);
57134       if (VT == MVT::i32 || VT == MVT::f32 ||
57135           (!VT.isVector() && !Subtarget.is64Bit()))
57136         return std::make_pair(0U, &X86::GR32_NOREXRegClass);
57137       if (VT != MVT::f80 && !VT.isVector())
57138         return std::make_pair(0U, &X86::GR64_NOREXRegClass);
57139       break;
57140     case 'f':  // FP Stack registers.
57141       // If SSE is enabled for this VT, use f80 to ensure the isel moves the
57142       // value to the correct fpstack register class.
57143       if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
57144         return std::make_pair(0U, &X86::RFP32RegClass);
57145       if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
57146         return std::make_pair(0U, &X86::RFP64RegClass);
57147       if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)
57148         return std::make_pair(0U, &X86::RFP80RegClass);
57149       break;
57150     case 'y':   // MMX_REGS if MMX allowed.
57151       if (!Subtarget.hasMMX()) break;
57152       return std::make_pair(0U, &X86::VR64RegClass);
57153     case 'v':
57154     case 'x':   // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
57155       if (!Subtarget.hasSSE1()) break;
57156       bool VConstraint = (Constraint[0] == 'v');
57157 
57158       switch (VT.SimpleTy) {
57159       default: break;
57160       // Scalar SSE types.
57161       case MVT::f16:
57162         if (VConstraint && Subtarget.hasFP16())
57163           return std::make_pair(0U, &X86::FR16XRegClass);
57164         break;
57165       case MVT::f32:
57166       case MVT::i32:
57167         if (VConstraint && Subtarget.hasVLX())
57168           return std::make_pair(0U, &X86::FR32XRegClass);
57169         return std::make_pair(0U, &X86::FR32RegClass);
57170       case MVT::f64:
57171       case MVT::i64:
57172         if (VConstraint && Subtarget.hasVLX())
57173           return std::make_pair(0U, &X86::FR64XRegClass);
57174         return std::make_pair(0U, &X86::FR64RegClass);
57175       case MVT::i128:
57176         if (Subtarget.is64Bit()) {
57177           if (VConstraint && Subtarget.hasVLX())
57178             return std::make_pair(0U, &X86::VR128XRegClass);
57179           return std::make_pair(0U, &X86::VR128RegClass);
57180         }
57181         break;
57182       // Vector types and fp128.
57183       case MVT::v8f16:
57184         if (!Subtarget.hasFP16())
57185           break;
57186         if (VConstraint)
57187           return std::make_pair(0U, &X86::VR128XRegClass);
57188         return std::make_pair(0U, &X86::VR128RegClass);
57189       case MVT::v8bf16:
57190         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57191           break;
57192         if (VConstraint)
57193           return std::make_pair(0U, &X86::VR128XRegClass);
57194         return std::make_pair(0U, &X86::VR128RegClass);
57195       case MVT::f128:
57196       case MVT::v16i8:
57197       case MVT::v8i16:
57198       case MVT::v4i32:
57199       case MVT::v2i64:
57200       case MVT::v4f32:
57201       case MVT::v2f64:
57202         if (VConstraint && Subtarget.hasVLX())
57203           return std::make_pair(0U, &X86::VR128XRegClass);
57204         return std::make_pair(0U, &X86::VR128RegClass);
57205       // AVX types.
57206       case MVT::v16f16:
57207         if (!Subtarget.hasFP16())
57208           break;
57209         if (VConstraint)
57210           return std::make_pair(0U, &X86::VR256XRegClass);
57211         return std::make_pair(0U, &X86::VR256RegClass);
57212       case MVT::v16bf16:
57213         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57214           break;
57215         if (VConstraint)
57216           return std::make_pair(0U, &X86::VR256XRegClass);
57217         return std::make_pair(0U, &X86::VR256RegClass);
57218       case MVT::v32i8:
57219       case MVT::v16i16:
57220       case MVT::v8i32:
57221       case MVT::v4i64:
57222       case MVT::v8f32:
57223       case MVT::v4f64:
57224         if (VConstraint && Subtarget.hasVLX())
57225           return std::make_pair(0U, &X86::VR256XRegClass);
57226         if (Subtarget.hasAVX())
57227           return std::make_pair(0U, &X86::VR256RegClass);
57228         break;
57229       case MVT::v32f16:
57230         if (!Subtarget.hasFP16())
57231           break;
57232         if (VConstraint)
57233           return std::make_pair(0U, &X86::VR512RegClass);
57234         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57235       case MVT::v32bf16:
57236         if (!Subtarget.hasBF16())
57237           break;
57238         if (VConstraint)
57239           return std::make_pair(0U, &X86::VR512RegClass);
57240         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57241       case MVT::v64i8:
57242       case MVT::v32i16:
57243       case MVT::v8f64:
57244       case MVT::v16f32:
57245       case MVT::v16i32:
57246       case MVT::v8i64:
57247         if (!Subtarget.hasAVX512()) break;
57248         if (VConstraint)
57249           return std::make_pair(0U, &X86::VR512RegClass);
57250         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57251       }
57252       break;
57253     }
57254   } else if (Constraint.size() == 2 && Constraint[0] == 'Y') {
57255     switch (Constraint[1]) {
57256     default:
57257       break;
57258     case 'i':
57259     case 't':
57260     case '2':
57261       return getRegForInlineAsmConstraint(TRI, "x", VT);
57262     case 'm':
57263       if (!Subtarget.hasMMX()) break;
57264       return std::make_pair(0U, &X86::VR64RegClass);
57265     case 'z':
57266       if (!Subtarget.hasSSE1()) break;
57267       switch (VT.SimpleTy) {
57268       default: break;
57269       // Scalar SSE types.
57270       case MVT::f16:
57271         if (!Subtarget.hasFP16())
57272           break;
57273         return std::make_pair(X86::XMM0, &X86::FR16XRegClass);
57274       case MVT::f32:
57275       case MVT::i32:
57276         return std::make_pair(X86::XMM0, &X86::FR32RegClass);
57277       case MVT::f64:
57278       case MVT::i64:
57279         return std::make_pair(X86::XMM0, &X86::FR64RegClass);
57280       case MVT::v8f16:
57281         if (!Subtarget.hasFP16())
57282           break;
57283         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57284       case MVT::v8bf16:
57285         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57286           break;
57287         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57288       case MVT::f128:
57289       case MVT::v16i8:
57290       case MVT::v8i16:
57291       case MVT::v4i32:
57292       case MVT::v2i64:
57293       case MVT::v4f32:
57294       case MVT::v2f64:
57295         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57296       // AVX types.
57297       case MVT::v16f16:
57298         if (!Subtarget.hasFP16())
57299           break;
57300         return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57301       case MVT::v16bf16:
57302         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57303           break;
57304         return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57305       case MVT::v32i8:
57306       case MVT::v16i16:
57307       case MVT::v8i32:
57308       case MVT::v4i64:
57309       case MVT::v8f32:
57310       case MVT::v4f64:
57311         if (Subtarget.hasAVX())
57312           return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57313         break;
57314       case MVT::v32f16:
57315         if (!Subtarget.hasFP16())
57316           break;
57317         return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57318       case MVT::v32bf16:
57319         if (!Subtarget.hasBF16())
57320           break;
57321         return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57322       case MVT::v64i8:
57323       case MVT::v32i16:
57324       case MVT::v8f64:
57325       case MVT::v16f32:
57326       case MVT::v16i32:
57327       case MVT::v8i64:
57328         if (Subtarget.hasAVX512())
57329           return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57330         break;
57331       }
57332       break;
57333     case 'k':
57334       // This register class doesn't allocate k0 for masked vector operation.
57335       if (Subtarget.hasAVX512()) {
57336         if (VT == MVT::i1)
57337           return std::make_pair(0U, &X86::VK1WMRegClass);
57338         if (VT == MVT::i8)
57339           return std::make_pair(0U, &X86::VK8WMRegClass);
57340         if (VT == MVT::i16)
57341           return std::make_pair(0U, &X86::VK16WMRegClass);
57342       }
57343       if (Subtarget.hasBWI()) {
57344         if (VT == MVT::i32)
57345           return std::make_pair(0U, &X86::VK32WMRegClass);
57346         if (VT == MVT::i64)
57347           return std::make_pair(0U, &X86::VK64WMRegClass);
57348       }
57349       break;
57350     }
57351   }
57352 
57353   if (parseConstraintCode(Constraint) != X86::COND_INVALID)
57354     return std::make_pair(0U, &X86::GR32RegClass);
57355 
57356   // Use the default implementation in TargetLowering to convert the register
57357   // constraint into a member of a register class.
57358   std::pair<Register, const TargetRegisterClass*> Res;
57359   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
57360 
57361   // Not found as a standard register?
57362   if (!Res.second) {
57363     // Only match x87 registers if the VT is one SelectionDAGBuilder can convert
57364     // to/from f80.
57365     if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {
57366       // Map st(0) -> st(7) -> ST0
57367       if (Constraint.size() == 7 && Constraint[0] == '{' &&
57368           tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
57369           Constraint[3] == '(' &&
57370           (Constraint[4] >= '0' && Constraint[4] <= '7') &&
57371           Constraint[5] == ')' && Constraint[6] == '}') {
57372         // st(7) is not allocatable and thus not a member of RFP80. Return
57373         // singleton class in cases where we have a reference to it.
57374         if (Constraint[4] == '7')
57375           return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
57376         return std::make_pair(X86::FP0 + Constraint[4] - '0',
57377                               &X86::RFP80RegClass);
57378       }
57379 
57380       // GCC allows "st(0)" to be called just plain "st".
57381       if (StringRef("{st}").equals_insensitive(Constraint))
57382         return std::make_pair(X86::FP0, &X86::RFP80RegClass);
57383     }
57384 
57385     // flags -> EFLAGS
57386     if (StringRef("{flags}").equals_insensitive(Constraint))
57387       return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
57388 
57389     // dirflag -> DF
57390     // Only allow for clobber.
57391     if (StringRef("{dirflag}").equals_insensitive(Constraint) &&
57392         VT == MVT::Other)
57393       return std::make_pair(X86::DF, &X86::DFCCRRegClass);
57394 
57395     // fpsr -> FPSW
57396     // Only allow for clobber.
57397     if (StringRef("{fpsr}").equals_insensitive(Constraint) && VT == MVT::Other)
57398       return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
57399 
57400     return Res;
57401   }
57402 
57403   // Make sure it isn't a register that requires 64-bit mode.
57404   if (!Subtarget.is64Bit() &&
57405       (isFRClass(*Res.second) || isGRClass(*Res.second)) &&
57406       TRI->getEncodingValue(Res.first) >= 8) {
57407     // Register requires REX prefix, but we're in 32-bit mode.
57408     return std::make_pair(0, nullptr);
57409   }
57410 
57411   // Make sure it isn't a register that requires AVX512.
57412   if (!Subtarget.hasAVX512() && isFRClass(*Res.second) &&
57413       TRI->getEncodingValue(Res.first) & 0x10) {
57414     // Register requires EVEX prefix.
57415     return std::make_pair(0, nullptr);
57416   }
57417 
57418   // Otherwise, check to see if this is a register class of the wrong value
57419   // type.  For example, we want to map "{ax},i32" -> {eax}, we don't want it to
57420   // turn into {ax},{dx}.
57421   // MVT::Other is used to specify clobber names.
57422   if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
57423     return Res;   // Correct type already, nothing to do.
57424 
57425   // Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
57426   // return "eax". This should even work for things like getting 64bit integer
57427   // registers when given an f64 type.
57428   const TargetRegisterClass *Class = Res.second;
57429   // The generic code will match the first register class that contains the
57430   // given register. Thus, based on the ordering of the tablegened file,
57431   // the "plain" GR classes might not come first.
57432   // Therefore, use a helper method.
57433   if (isGRClass(*Class)) {
57434     unsigned Size = VT.getSizeInBits();
57435     if (Size == 1) Size = 8;
57436     if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
57437       return std::make_pair(0, nullptr);
57438     Register DestReg = getX86SubSuperRegister(Res.first, Size);
57439     if (DestReg.isValid()) {
57440       bool is64Bit = Subtarget.is64Bit();
57441       const TargetRegisterClass *RC =
57442           Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
57443         : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
57444         : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
57445         : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
57446       if (Size == 64 && !is64Bit) {
57447         // Model GCC's behavior here and select a fixed pair of 32-bit
57448         // registers.
57449         switch (DestReg) {
57450         case X86::RAX:
57451           return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
57452         case X86::RDX:
57453           return std::make_pair(X86::EDX, &X86::GR32_DCRegClass);
57454         case X86::RCX:
57455           return std::make_pair(X86::ECX, &X86::GR32_CBRegClass);
57456         case X86::RBX:
57457           return std::make_pair(X86::EBX, &X86::GR32_BSIRegClass);
57458         case X86::RSI:
57459           return std::make_pair(X86::ESI, &X86::GR32_SIDIRegClass);
57460         case X86::RDI:
57461           return std::make_pair(X86::EDI, &X86::GR32_DIBPRegClass);
57462         case X86::RBP:
57463           return std::make_pair(X86::EBP, &X86::GR32_BPSPRegClass);
57464         default:
57465           return std::make_pair(0, nullptr);
57466         }
57467       }
57468       if (RC && RC->contains(DestReg))
57469         return std::make_pair(DestReg, RC);
57470       return Res;
57471     }
57472     // No register found/type mismatch.
57473     return std::make_pair(0, nullptr);
57474   } else if (isFRClass(*Class)) {
57475     // Handle references to XMM physical registers that got mapped into the
57476     // wrong class.  This can happen with constraints like {xmm0} where the
57477     // target independent register mapper will just pick the first match it can
57478     // find, ignoring the required type.
57479 
57480     // TODO: Handle f128 and i128 in FR128RegClass after it is tested well.
57481     if (VT == MVT::f16)
57482       Res.second = &X86::FR16XRegClass;
57483     else if (VT == MVT::f32 || VT == MVT::i32)
57484       Res.second = &X86::FR32XRegClass;
57485     else if (VT == MVT::f64 || VT == MVT::i64)
57486       Res.second = &X86::FR64XRegClass;
57487     else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT))
57488       Res.second = &X86::VR128XRegClass;
57489     else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT))
57490       Res.second = &X86::VR256XRegClass;
57491     else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))
57492       Res.second = &X86::VR512RegClass;
57493     else {
57494       // Type mismatch and not a clobber: Return an error;
57495       Res.first = 0;
57496       Res.second = nullptr;
57497     }
57498   } else if (isVKClass(*Class)) {
57499     if (VT == MVT::i1)
57500       Res.second = &X86::VK1RegClass;
57501     else if (VT == MVT::i8)
57502       Res.second = &X86::VK8RegClass;
57503     else if (VT == MVT::i16)
57504       Res.second = &X86::VK16RegClass;
57505     else if (VT == MVT::i32)
57506       Res.second = &X86::VK32RegClass;
57507     else if (VT == MVT::i64)
57508       Res.second = &X86::VK64RegClass;
57509     else {
57510       // Type mismatch and not a clobber: Return an error;
57511       Res.first = 0;
57512       Res.second = nullptr;
57513     }
57514   }
57515 
57516   return Res;
57517 }
57518 
57519 bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
57520   // Integer division on x86 is expensive. However, when aggressively optimizing
57521   // for code size, we prefer to use a div instruction, as it is usually smaller
57522   // than the alternative sequence.
57523   // The exception to this is vector division. Since x86 doesn't have vector
57524   // integer division, leaving the division as-is is a loss even in terms of
57525   // size, because it will have to be scalarized, while the alternative code
57526   // sequence can be performed in vector form.
57527   bool OptSize = Attr.hasFnAttr(Attribute::MinSize);
57528   return OptSize && !VT.isVector();
57529 }
57530 
57531 void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
57532   if (!Subtarget.is64Bit())
57533     return;
57534 
57535   // Update IsSplitCSR in X86MachineFunctionInfo.
57536   X86MachineFunctionInfo *AFI =
57537       Entry->getParent()->getInfo<X86MachineFunctionInfo>();
57538   AFI->setIsSplitCSR(true);
57539 }
57540 
57541 void X86TargetLowering::insertCopiesSplitCSR(
57542     MachineBasicBlock *Entry,
57543     const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
57544   const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
57545   const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
57546   if (!IStart)
57547     return;
57548 
57549   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
57550   MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
57551   MachineBasicBlock::iterator MBBI = Entry->begin();
57552   for (const MCPhysReg *I = IStart; *I; ++I) {
57553     const TargetRegisterClass *RC = nullptr;
57554     if (X86::GR64RegClass.contains(*I))
57555       RC = &X86::GR64RegClass;
57556     else
57557       llvm_unreachable("Unexpected register class in CSRsViaCopy!");
57558 
57559     Register NewVR = MRI->createVirtualRegister(RC);
57560     // Create copy from CSR to a virtual register.
57561     // FIXME: this currently does not emit CFI pseudo-instructions, it works
57562     // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
57563     // nounwind. If we want to generalize this later, we may need to emit
57564     // CFI pseudo-instructions.
57565     assert(
57566         Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) &&
57567         "Function should be nounwind in insertCopiesSplitCSR!");
57568     Entry->addLiveIn(*I);
57569     BuildMI(*Entry, MBBI, MIMetadata(), TII->get(TargetOpcode::COPY), NewVR)
57570         .addReg(*I);
57571 
57572     // Insert the copy-back instructions right before the terminator.
57573     for (auto *Exit : Exits)
57574       BuildMI(*Exit, Exit->getFirstTerminator(), MIMetadata(),
57575               TII->get(TargetOpcode::COPY), *I)
57576           .addReg(NewVR);
57577   }
57578 }
57579 
57580 bool X86TargetLowering::supportSwiftError() const {
57581   return Subtarget.is64Bit();
57582 }
57583 
57584 MachineInstr *
57585 X86TargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
57586                                  MachineBasicBlock::instr_iterator &MBBI,
57587                                  const TargetInstrInfo *TII) const {
57588   assert(MBBI->isCall() && MBBI->getCFIType() &&
57589          "Invalid call instruction for a KCFI check");
57590 
57591   MachineFunction &MF = *MBB.getParent();
57592   // If the call target is a memory operand, unfold it and use R11 for the
57593   // call, so KCFI_CHECK won't have to recompute the address.
57594   switch (MBBI->getOpcode()) {
57595   case X86::CALL64m:
57596   case X86::CALL64m_NT:
57597   case X86::TAILJMPm64:
57598   case X86::TAILJMPm64_REX: {
57599     MachineBasicBlock::instr_iterator OrigCall = MBBI;
57600     SmallVector<MachineInstr *, 2> NewMIs;
57601     if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true,
57602                                   /*UnfoldStore=*/false, NewMIs))
57603       report_fatal_error("Failed to unfold memory operand for a KCFI check");
57604     for (auto *NewMI : NewMIs)
57605       MBBI = MBB.insert(OrigCall, NewMI);
57606     assert(MBBI->isCall() &&
57607            "Unexpected instruction after memory operand unfolding");
57608     if (OrigCall->shouldUpdateCallSiteInfo())
57609       MF.moveCallSiteInfo(&*OrigCall, &*MBBI);
57610     MBBI->setCFIType(MF, OrigCall->getCFIType());
57611     OrigCall->eraseFromParent();
57612     break;
57613   }
57614   default:
57615     break;
57616   }
57617 
57618   MachineOperand &Target = MBBI->getOperand(0);
57619   Register TargetReg;
57620   switch (MBBI->getOpcode()) {
57621   case X86::CALL64r:
57622   case X86::CALL64r_NT:
57623   case X86::TAILJMPr64:
57624   case X86::TAILJMPr64_REX:
57625     assert(Target.isReg() && "Unexpected target operand for an indirect call");
57626     Target.setIsRenamable(false);
57627     TargetReg = Target.getReg();
57628     break;
57629   case X86::CALL64pcrel32:
57630   case X86::TAILJMPd64:
57631     assert(Target.isSymbol() && "Unexpected target operand for a direct call");
57632     // X86TargetLowering::EmitLoweredIndirectThunk always uses r11 for
57633     // 64-bit indirect thunk calls.
57634     assert(StringRef(Target.getSymbolName()).ends_with("_r11") &&
57635            "Unexpected register for an indirect thunk call");
57636     TargetReg = X86::R11;
57637     break;
57638   default:
57639     llvm_unreachable("Unexpected CFI call opcode");
57640     break;
57641   }
57642 
57643   return BuildMI(MBB, MBBI, MIMetadata(*MBBI), TII->get(X86::KCFI_CHECK))
57644       .addReg(TargetReg)
57645       .addImm(MBBI->getCFIType())
57646       .getInstr();
57647 }
57648 
57649 /// Returns true if stack probing through a function call is requested.
57650 bool X86TargetLowering::hasStackProbeSymbol(const MachineFunction &MF) const {
57651   return !getStackProbeSymbolName(MF).empty();
57652 }
57653 
57654 /// Returns true if stack probing through inline assembly is requested.
57655 bool X86TargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
57656 
57657   // No inline stack probe for Windows, they have their own mechanism.
57658   if (Subtarget.isOSWindows() ||
57659       MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
57660     return false;
57661 
57662   // If the function specifically requests inline stack probes, emit them.
57663   if (MF.getFunction().hasFnAttribute("probe-stack"))
57664     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
57665            "inline-asm";
57666 
57667   return false;
57668 }
57669 
57670 /// Returns the name of the symbol used to emit stack probes or the empty
57671 /// string if not applicable.
57672 StringRef
57673 X86TargetLowering::getStackProbeSymbolName(const MachineFunction &MF) const {
57674   // Inline Stack probes disable stack probe call
57675   if (hasInlineStackProbe(MF))
57676     return "";
57677 
57678   // If the function specifically requests stack probes, emit them.
57679   if (MF.getFunction().hasFnAttribute("probe-stack"))
57680     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString();
57681 
57682   // Generally, if we aren't on Windows, the platform ABI does not include
57683   // support for stack probes, so don't emit them.
57684   if (!Subtarget.isOSWindows() || Subtarget.isTargetMachO() ||
57685       MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
57686     return "";
57687 
57688   // We need a stack probe to conform to the Windows ABI. Choose the right
57689   // symbol.
57690   if (Subtarget.is64Bit())
57691     return Subtarget.isTargetCygMing() ? "___chkstk_ms" : "__chkstk";
57692   return Subtarget.isTargetCygMing() ? "_alloca" : "_chkstk";
57693 }
57694 
57695 unsigned
57696 X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
57697   // The default stack probe size is 4096 if the function has no stackprobesize
57698   // attribute.
57699   return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size",
57700                                                         4096);
57701 }
57702 
57703 Align X86TargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
57704   if (ML && ML->isInnermost() &&
57705       ExperimentalPrefInnermostLoopAlignment.getNumOccurrences())
57706     return Align(1ULL << ExperimentalPrefInnermostLoopAlignment);
57707   return TargetLowering::getPrefLoopAlignment();
57708 }
57709