Lines Matching defs:SDValue
2745 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2750 return SDValue(Node, 0);
2780 bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2799 bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2813 bool X86::mayFoldIntoStore(SDValue Op) {
2826 bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2899 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2988 bool isFP, SDValue &LHS, SDValue &RHS,
3255 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3315 SDValue C) const {
3360 bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
3460 bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
3476 bool X86TargetLowering::hasAndNot(SDValue Y) const {
3493 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
3499 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
3628 bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
3932 bool X86::isZeroNode(SDValue Elt) {
3939 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
3942 SmallVector<SDValue, 32> Ops;
3956 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
3963 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3969 static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
3973 SmallVector<SDValue, 32> Ops;
4006 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
4010 static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
4017 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
4026 SDValue Vec;
4046 static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute) {
4051 return SDValue();
4053 SDValue Src = LHS.getOperand(0);
4055 return SDValue();
4064 return SDValue();
4067 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
4094 SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
4104 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
4112 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
4118 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
4138 SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
4148 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
4156 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
4168 if (all_of(Hi, [&](SDValue V) {
4173 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
4181 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
4205 static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements,
4215 static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
4225 SDValue Src = N->getOperand(0);
4226 SDValue Sub = N->getOperand(1);
4244 SDValue Lo = Src.getOperand(1);
4245 SDValue Hi = Sub;
4246 SmallVector<SDValue, 2> LoOps, HiOps;
4279 static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL,
4281 SmallVector<SDValue> SubOps;
4283 return SDValue();
4289 ArrayRef<SDValue> UpperOps(SubOps.begin() + HalfNumSubOps, SubOps.end());
4290 if (any_of(UpperOps, [](SDValue Op) { return !Op.isUndef(); }))
4291 return SDValue();
4294 ArrayRef<SDValue> LowerOps(SubOps.begin(), SubOps.begin() + HalfNumSubOps);
4301 SmallVector<SDValue> Ops;
4305 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
4315 SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
4319 SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
4324 static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) {
4329 SmallVector<SDValue> LoOps(NumOps, SDValue());
4330 SmallVector<SDValue> HiOps(NumOps, SDValue());
4332 SDValue SrcOp = Op.getOperand(I);
4349 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG,
4365 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG,
4381 // SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
4383 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4384 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
4409 SmallVector<SDValue, 4> Subs;
4411 SmallVector<SDValue, 2> SubOps;
4412 for (SDValue Op : Ops) {
4425 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,
4426 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
4433 auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {
4439 return SDValue();
4442 return SDValue();
4452 return SDValue();
4462 SmallVector<SDValue> SrcOps(Ops);
4463 for (SDValue &Op : SrcOps) {
4470 if (SDValue BroadcastOp = MakeBroadcastOp(Op, OpVT, DstVT)) {
4480 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
4489 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4493 SDValue Vec = Op.getOperand(0);
4494 SDValue SubVec = Op.getOperand(1);
4495 SDValue Idx = Op.getOperand(2);
4507 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
4528 SDValue Undef = DAG.getUNDEF(WideOpVT);
4532 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
4559 [](SDValue V) { return V.isUndef(); })) {
4591 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
4613 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
4614 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
4634 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
4641 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
4654 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
4662 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
4669 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
4673 SDValue Vec = DAG.getAllOnesConstant(dl, MVT::getVectorVT(MVT::i32, NumElts));
4677 static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
4678 SDValue In, SelectionDAG &DAG) {
4703 static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
4704 SDValue Mask, SelectionDAG &DAG) {
4742 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
4743 SDValue V1, SDValue V2, ArrayRef<int> Mask) {
4746 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
4751 SDValue V = (M < NumElts) ? V1 : V2;
4763 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4764 SDValue V1, SDValue V2) {
4771 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4772 SDValue V1, SDValue V2) {
4781 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4782 const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
4822 SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
4828 SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
4848 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
4853 SDValue V1 = IsZero
4863 static ConstantPoolSDNode *getTargetConstantPoolFromBasePtr(SDValue Ptr) {
4871 static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
4884 static const Constant *getTargetConstantFromNode(SDValue Op) {
4896 static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
5067 SDValue Ptr = MemIntr->getBasePtr();
5088 SDValue Ptr = MemIntr->getBasePtr();
5227 bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
5254 static bool getTargetShuffleMaskIndices(SDValue MaskNode,
5272 static bool isConstantPowerOf2(SDValue V, unsigned EltSizeInBIts,
5288 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
5302 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
5336 SmallVector<SDValue, 2> CatOps;
5338 for (SDValue &CatOp : CatOps) {
5339 SDValue NotCat = IsNOT(CatOp, DAG);
5341 return SDValue();
5351 if (SDValue Op1 = IsNOT(V.getOperand(1), DAG))
5352 if (SDValue Op0 = IsNOT(V.getOperand(0), DAG))
5357 return SDValue();
5425 static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
5426 SmallVectorImpl<SDValue> &Ops,
5582 SDValue MaskNode = N.getOperand(1);
5595 SDValue MaskNode = N.getOperand(1);
5648 SDValue MaskNode = N.getOperand(2);
5649 SDValue CtrlNode = N.getOperand(3);
5665 SDValue MaskNode = N.getOperand(2);
5677 SDValue MaskNode = N.getOperand(0);
5692 SDValue MaskNode = N.getOperand(1);
5732 static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
5733 SmallVectorImpl<SDValue> &Ops,
5748 SDValue V1, SDValue V2,
5776 SDValue V = M < Size ? V1 : V2;
5787 SDValue Op = V.getOperand(M / Scale);
5813 SDValue Op = V.getOperand((M * Scale) + j);
5830 static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
5831 SmallVectorImpl<SDValue> &Ops,
5842 SDValue V1 = Ops[0];
5843 SDValue V2 = IsUnary ? V1 : Ops[1];
5879 SDValue V = M < Size ? V1 : V2;
5906 SDValue Vec = V.getOperand(0);
5966 SDValue Cond, bool IsBLENDV = false) {
5994 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
5995 SmallVectorImpl<SDValue> &Inputs,
6004 static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
6006 SmallVectorImpl<SDValue> &Ops,
6040 SDValue N0 = N.getOperand(0);
6041 SDValue N1 = N.getOperand(1);
6063 SDValue N0 = peekThroughBitcasts(N.getOperand(0));
6064 SDValue N1 = peekThroughBitcasts(N.getOperand(1));
6069 SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
6101 SDValue Src = N.getOperand(0);
6102 SDValue Sub = N.getOperand(1);
6107 SDValue SubBC = peekThroughBitcasts(Sub);
6113 SDValue SubBCSrc = SubBC.getOperand(0);
6150 SmallVector<SDValue, 2> SubInputs;
6151 SDValue SubSrc = peekThroughOneUseBitcasts(Sub);
6162 if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
6206 SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
6243 SDValue SrcExtract;
6253 SDValue SrcVec = SrcExtract.getOperand(0);
6284 SDValue N0 = N.getOperand(0);
6285 SDValue N1 = N.getOperand(1);
6342 SDValue Cond = N.getOperand(0);
6351 SDValue Src = N.getOperand(0);
6439 SDValue Src = N.getOperand(0);
6453 SDValue Src = N.getOperand(0);
6479 SDValue Src = N.getOperand(0);
6500 static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
6503 SmallVector<SDValue, 16> UsedInputs;
6544 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6545 SmallVectorImpl<SDValue> &Inputs,
6570 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6571 SmallVectorImpl<SDValue> &Inputs,
6580 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6595 static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT,
6604 return SDValue();
6606 SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
6609 SDValue Ops[] = {Mem->getChain(), Ptr};
6610 SDValue BcstLd = DAG.getMemIntrinsicNode(
6614 DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
6620 static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
6623 return SDValue(); // Limit search depth.
6636 SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
6646 SmallVector<SDValue, 16> ShuffleOps;
6648 return SDValue();
6658 SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
6664 SDValue Vec = Op.getOperand(0);
6665 SDValue Sub = Op.getOperand(1);
6685 SDValue Src = Op.getOperand(0);
6692 SDValue Src = Op.getOperand(0);
6696 return SDValue();
6717 return SDValue();
6721 static SDValue LowerBuildVectorAsInsert(SDValue Op, const SDLoc &DL,
6732 SDValue V;
6763 static SDValue LowerBuildVectorv16i8(SDValue Op, const SDLoc &DL,
6769 return SDValue();
6776 SDValue V;
6785 SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), DL, MVT::i32);
6802 SDValue Elt;
6811 SDValue NextElt = Op.getOperand(i + 1);
6845 static SDValue LowerBuildVectorv8i16(SDValue Op, const SDLoc &DL,
6851 return SDValue();
6859 static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
6874 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
6876 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
6877 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
6884 SDValue Elt = Op.getOperand(i);
6893 SDValue FirstNonZero;
6898 SDValue Elt = Op.getOperand(i);
6901 return SDValue();
6905 return SDValue();
6913 SDValue V1 = FirstNonZero.getOperand(0);
6917 SDValue Elt;
6937 SDValue VZeroOrUndef = (Zeroable == Undefs)
6947 return SDValue();
6949 SDValue V2 = Elt.getOperand(0);
6951 V1 = SDValue();
6958 SDValue Current = Op->getOperand(i);
6959 SDValue SrcVector = Current->getOperand(0);
6966 return SDValue();
6979 SDValue Result =
6986 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
6994 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
6998 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
7005 SDValue Ptr = LD->getBasePtr();
7007 return SDValue();
7010 return SDValue();
7023 return SDValue();
7029 SDValue Chain = LD->getChain();
7038 return SDValue();
7047 return SDValue();
7049 return SDValue();
7061 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
7069 return SDValue();
7073 static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
7099 SDValue Src = Elt.getOperand(0);
7120 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
7125 return SDValue();
7140 SDValue Elt = peekThroughBitcasts(Elts[i]);
7142 return SDValue();
7156 return SDValue();
7159 return SDValue();
7162 return SDValue();
7180 SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
7194 return SDValue();
7231 SDValue NewLd =
7254 return SDValue();
7260 return SDValue();
7282 SDValue V = CreateLoad(VT, LDBase);
7283 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
7296 SDValue HalfLD =
7319 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
7320 SDValue ResNode = DAG.getMemIntrinsicNode(
7346 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
7350 SDValue Elt = peekThroughBitcasts(Elts[i]);
7374 if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
7376 SDValue Broadcast = RepeatLoad;
7386 return SDValue();
7396 return SDValue();
7402 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
7406 SmallVector<SDValue, 64> Elts;
7408 if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
7412 return SDValue();
7501 /// or SDValue() otherwise.
7502 static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
7510 return SDValue();
7519 SDValue Ld;
7521 SmallVector<SDValue, 16> Sequence;
7540 [](SDValue V) { return !V || isNullConstantOrUndef(V); });
7541 SDValue Op0 = Sequence[0];
7545 SDValue BOperand = Op0.getOpcode() == ISD::BITCAST
7557 SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand);
7577 return SDValue();
7586 SDValue CP = DAG.getConstantPool(C, PVT);
7591 SDValue Ops[] = {DAG.getEntryNode(), CP};
7594 SDValue Brdcst =
7602 SDValue VCP = DAG.getConstantPool(VecC, PVT);
7607 SDValue Ops[] = {DAG.getEntryNode(), VCP};
7622 return SDValue();
7625 return SDValue();
7638 return SDValue();
7676 SDValue CP =
7681 SDValue Ops[] = {DAG.getEntryNode(), CP};
7696 return SDValue();
7700 return SDValue();
7706 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7707 SDValue BCast =
7710 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7720 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7721 SDValue BCast =
7724 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7732 return SDValue();
7740 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
7741 SDValue ExtIdx) {
7757 SDValue ShuffleVec = SVOp->getOperand(0);
7770 static SDValue buildFromShuffleMostly(SDValue Op, const SDLoc &DL,
7777 return SDValue();
7780 SDValue VecIn1;
7781 SDValue VecIn2;
7794 return SDValue();
7800 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
7801 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
7805 return SDValue();
7810 return SDValue();
7819 return SDValue();
7829 return SDValue();
7832 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
7842 static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG,
7847 SmallVector<SDValue, 16> NewOps;
7851 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
7856 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
7873 SDValue In = Op.getOperand(idx);
7894 SDValue Cond = Op.getOperand(SplatIdx);
7902 SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
7909 SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
7920 SDValue DstVec;
7923 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
7924 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
7930 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
7983 SDValue &V0, SDValue &V1) {
7999 SDValue Op = N->getOperand(i + BaseIdx);
8015 SDValue Op0 = Op.getOperand(0);
8016 SDValue Op1 = Op.getOperand(1);
8047 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
8094 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8103 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
8104 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
8105 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
8106 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
8109 SDValue LO = DAG.getUNDEF(NewVT);
8110 SDValue HI = DAG.getUNDEF(NewVT);
8137 SDValue &Opnd0, SDValue &Opnd1,
8146 SDValue InVec0 = DAG.getUNDEF(VT);
8147 SDValue InVec1 = DAG.getUNDEF(VT);
8157 SDValue Op = BV->getOperand(i);
8168 SDValue Op0 = Op.getOperand(0);
8169 SDValue Op1 = Op.getOperand(1);
8264 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8290 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
8294 SDValue Opnd0, Opnd1;
8299 return SDValue();
8304 SDValue Opnd2;
8312 return SDValue();
8322 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1);
8323 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1);
8331 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
8349 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
8371 SDValue Op0 = Op.getOperand(0);
8372 SDValue Op1 = Op.getOperand(1);
8390 SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
8419 static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
8421 unsigned HOpcode, SDValue V0, SDValue V1) {
8449 SDValue Half = DAG.getNode(HOpcode, DL, HalfVT, V0, V1);
8457 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV, const SDLoc &DL,
8462 count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
8464 return SDValue();
8475 SDValue V0, V1;
8482 return SDValue();
8497 SDValue InVec0, InVec1;
8499 SDValue InVec2, InVec3;
8523 return SDValue();
8528 SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
8529 SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
8554 return SDValue();
8559 return SDValue();
8569 return SDValue();
8572 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
8581 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op, const SDLoc &DL,
8593 return SDValue();
8599 return SDValue();
8611 return SDValue();
8613 return SDValue();
8617 SmallVector<SDValue, 4> LHSElts, RHSElts;
8618 for (SDValue Elt : Op->ops()) {
8619 SDValue LHS = Elt.getOperand(0);
8620 SDValue RHS = Elt.getOperand(1);
8624 return SDValue();
8629 return SDValue();
8640 if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
8641 return SDValue();
8643 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
8644 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
8645 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
8658 static SDValue materializeVectorConstant(SDValue Op, const SDLoc &DL,
8677 return SDValue();
8683 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
8718 SDValue NewSrcVec =
8722 return SDValue();
8727 return SDValue();
8730 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
8801 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
8802 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
8803 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
8804 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
8810 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
8811 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
8812 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
8813 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
8815 ArrayRef<SDValue> Ops) {
8819 SDValue Idx = Ops[2];
8826 SDValue Ops[] = {LoLo, HiHi, IndicesVec};
8849 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8851 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8859 SDValue Res = DAG.getSelectCC(
8876 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
8883 SDValue LoLo =
8885 SDValue HiHi =
8895 SDValue Res = DAG.getSelectCC(
8920 return SDValue();
8934 SDValue Res = Opcode == X86ISD::VPERMV
8951 static SDValue
8952 LowerBUILD_VECTORAsVariablePermute(SDValue V, const SDLoc &DL,
8955 SDValue SrcVec, IndicesVec;
8960 SDValue Op = V.getOperand(Idx);
8962 return SDValue();
8970 return SDValue();
8971 SDValue ExtractedIndex = Op->getOperand(1);
8977 return SDValue();
8985 return SDValue();
8989 return SDValue();
8996 SDValue
8997 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
9013 if (SDValue VectorCst = materializeVectorConstant(Op, dl, DAG, Subtarget))
9023 SmallSet<SDValue, 8> Values;
9026 SDValue Elt = Op.getOperand(i);
9068 SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT));
9080 SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts);
9081 SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT));
9082 SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt);
9103 SDValue NewBV =
9109 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, dl, Subtarget, DAG))
9111 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, dl, Subtarget, DAG))
9113 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, dl, Subtarget, DAG))
9115 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, dl, Subtarget, DAG))
9137 SDValue VarElt;
9138 SDValue InsIndex;
9140 SDValue Elt = Op.getOperand(i);
9153 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
9161 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
9164 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
9178 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
9185 SDValue Item = Op.getOperand(Idx);
9230 return SDValue();
9251 SDValue Item = Op.getOperand(Idx);
9255 return SDValue();
9261 return SDValue();
9263 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, dl, DAG, Subtarget))
9268 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
9269 if (SDValue LD =
9278 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
9280 auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
9291 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
9306 SDValue Lower =
9308 SDValue Upper = DAG.getBuildVector(
9320 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
9324 return SDValue();
9329 if (SDValue V = LowerBuildVectorv16i8(Op, dl, NonZeroMask, NumNonZero,
9334 if (SDValue V = LowerBuildVectorv8i16(Op, dl, NonZeroMask, NumNonZero,
9340 if (SDValue V = LowerBuildVectorv4x32(Op, dl, DAG, Subtarget))
9345 SmallVector<SDValue, 8> Ops(NumElems);
9386 if (SDValue Sh = buildFromShuffleMostly(Op, dl, DAG))
9391 SDValue Result;
9408 SmallVector<SDValue, 8> Ops(NumElems);
9438 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9452 SDValue SubVec = Op.getOperand(i);
9475 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9477 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9483 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
9504 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
9517 SDValue SubVec = Op.getOperand(i);
9535 SDValue SubVec = Op.getOperand(Idx);
9547 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
9551 SDValue SubVec = Op.getOperand(Idx);
9560 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9562 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9572 SDValue Vec =
9579 static SDValue LowerCONCAT_VECTORS(SDValue Op,
9785 static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
9846 SDValue V1 = SDValue(),
9847 SDValue V2 = SDValue()) {
9857 SDValue MaskV = MaskIdx < Size ? V1 : V2;
9858 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9880 SDValue V1 = SDValue(),
9881 SDValue V2 = SDValue()) {
9896 V1 = SDValue();
9899 V2 = SDValue();
9913 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9923 SDValue MaskV = MaskIdx < Size ? V1 : V2;
9924 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
10021 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
10058 static SDValue getSHUFPDImmForMask(ArrayRef<int> Mask, const SDLoc &DL,
10096 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
10097 ArrayRef<int> Mask, SDValue V1,
10098 SDValue V2, const APInt &Zeroable,
10110 SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
10112 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
10114 SDValue V;
10127 SDValue SrcV = (M >= Size ? V2 : V1);
10129 return SDValue();
10135 return SDValue();
10149 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
10154 static SDValue lowerShuffleWithEXPAND(const SDLoc &DL, MVT VT, SDValue V1,
10155 SDValue V2, ArrayRef<int> Mask,
10162 return SDValue();
10166 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
10170 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
10172 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
10173 SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
10177 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
10266 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1,
10267 SDValue V2, ArrayRef<int> Mask,
10288 return SDValue();
10293 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1,
10294 SDValue V2, ArrayRef<int> Mask,
10306 return SDValue();
10357 static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src,
10367 return SDValue();
10375 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10381 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10389 SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512);
10395 SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src);
10418 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,
10419 SDValue V2, ArrayRef<int> Mask,
10425 return SDValue();
10440 SDValue Src = peekThroughBitcasts(V1);
10452 return SDValue();
10454 return SDValue();
10458 return SDValue();
10464 return SDValue();
10468 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
10469 SDValue V2, ArrayRef<int> Mask,
10476 return SDValue();
10506 auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
10526 SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
10543 return SDValue();
10622 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
10632 auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
10693 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1,
10694 SDValue V2, ArrayRef<int> Mask,
10704 return SDValue();
10711 return SDValue();
10721 SDValue Res;
10743 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10744 SDValue V2, ArrayRef<int> Mask,
10750 SDValue Zero, AllOnes;
10769 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
10770 SDValue V;
10775 return SDValue(); // Not a blend.
10779 return SDValue(); // Can only let one input through the mask.
10784 return SDValue(); // No non-zeroable elements!
10786 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10789 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10798 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10799 SDValue V2, ArrayRef<int> Mask,
10803 SDValue Zero = DAG.getConstant(0, DL, EltVT);
10804 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10805 SmallVector<SDValue, 16> MaskOps;
10808 return SDValue(); // Shuffled input!
10812 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10816 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10817 SDValue PreservedSrc,
10821 static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
10905 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
10906 SDValue V2, ArrayRef<int> Original,
10915 return SDValue();
10961 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10963 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10978 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10984 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10990 if (SDValue BitBlend =
11021 SmallVector<SDValue, 32> VSELECTMask;
11045 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11053 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11066 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
11067 SDValue V1, SDValue V2,
11085 return SDValue(); // Can't blend in the needed input!
11094 return SDValue();
11096 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11105 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
11106 SDValue V1, SDValue V2,
11115 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
11125 SDValue &Op = Ops[Elt & 1];
11132 return SDValue();
11148 return SDValue();
11175 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11188 static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
11189 SDValue V1, SDValue V2,
11198 return SDValue();
11221 return SDValue();
11234 return SDValue();
11256 if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
11263 return SDValue();
11294 return SDValue();
11299 static SDValue lowerShuffleAsByteRotateAndPermute(
11300 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11305 return SDValue();
11309 return SDValue();
11348 return SDValue();
11351 return SDValue();
11354 auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
11356 SDValue Rotate = DAG.getBitcast(
11380 return SDValue();
11416 static SDValue lowerShuffleAsDecomposedShuffleMerge(
11417 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11449 &DAG](SDValue &Input,
11484 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11495 if (SDValue UnpackPerm =
11498 if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11502 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11506 if (SDValue PermUnpack = lowerShuffleAsPermuteAndUnpack(
11557 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
11566 return SDValue();
11572 return SDValue();
11579 return SDValue();
11584 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
11586 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
11588 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
11592 SDValue Rot =
11601 static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
11613 SDValue Lo, Hi;
11639 SDValue MaskV = M < NumElts ? V1 : V2;
11644 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11686 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11708 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11709 SDValue V2, ArrayRef<int> Mask,
11714 SDValue Lo = V1, Hi = V2;
11717 return SDValue();
11745 SDValue LoShift =
11748 SDValue HiShift =
11765 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
11766 SDValue V2, ArrayRef<int> Mask,
11777 SDValue Lo = V1, Hi = V2;
11792 return SDValue();
11795 SDValue Src = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11804 SDValue Src = Mask[0] < (int)NumElts ? V1 : V2;
11812 return SDValue();
11816 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
11817 SDValue V2, ArrayRef<int> Mask,
11829 return SDValue();
11834 return SDValue();
11840 return SDValue();
11842 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11875 return SDValue();
11965 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11966 SDValue V2, ArrayRef<int> Mask,
11974 SDValue V = V1;
11989 return SDValue();
11992 return SDValue();
12004 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
12025 SDValue Src;
12031 SDValue &V = (M < Size ? V1 : V2);
12060 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
12072 SDValue Base;
12088 SDValue Insert;
12126 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
12127 SDValue V2, ArrayRef<int> Mask,
12141 return SDValue();
12153 static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
12154 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
12174 auto ShuffleOffset = [&](SDValue V) {
12192 return SDValue();
12236 SDValue Lo = DAG.getBitcast(
12245 SDValue Hi = DAG.getBitcast(
12258 SDValue PSHUFBMask[16];
12294 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12317 static SDValue lowerShuffleAsZeroOrAnyExtend(
12318 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12331 auto Lower = [&](int Scale) -> SDValue {
12332 SDValue InputV;
12343 return SDValue();
12352 SDValue V = M < NumElements ? V1 : V2;
12358 return SDValue(); // Flip-flopping inputs.
12365 return SDValue();
12370 return SDValue();
12373 return SDValue(); // Non-consecutive strided elements.
12381 return SDValue();
12386 return SDValue();
12402 if (SDValue V = Lower(NumElements / NumExtElements))
12408 return SDValue();
12415 return SDValue();
12420 return SDValue();
12423 if (SDValue V = CanZExtLowHalf()) {
12430 return SDValue();
12436 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12446 return SDValue();
12452 SDValue S = V.getOperand(Idx);
12457 return SDValue();
12464 static bool isShuffleFoldableLoad(SDValue V) {
12480 static SDValue lowerShuffleAsElementInsertion(
12481 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12490 return SDValue();
12508 return SDValue();
12516 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12526 return SDValue();
12537 SDValue BitMask = getConstVector(Bits, VT, DAG, DL);
12549 return SDValue();
12557 return SDValue();
12559 return SDValue();
12576 return SDValue();
12606 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12621 return SDValue();
12628 return SDValue();
12639 return SDValue();
12641 SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12695 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12696 SDValue N1, ArrayRef<int> Mask,
12708 return SDValue();
12710 SDValue WideVec = N0.getOperand(0);
12713 return SDValue();
12724 return SDValue();
12731 return SDValue();
12737 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12749 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12750 SDValue V2, ArrayRef<int> Mask,
12757 return SDValue();
12770 return SDValue();
12781 SDValue V = V1;
12805 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12834 if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12846 return SDValue();
12855 SDValue BaseAddr = Ld->getOperand(1);
12859 SDValue NewAddr =
12867 SDValue Ops[] = {Ld->getChain(), NewAddr};
12882 return SDValue();
12888 return SDValue();
12892 return SDValue();
12907 return SDValue();
12956 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12967 auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
13044 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
13053 return SDValue();
13067 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13068 const APInt &Zeroable, SDValue V1, SDValue V2,
13077 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
13104 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13109 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13116 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13124 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
13132 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
13137 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask, DAG))
13151 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13152 const APInt &Zeroable, SDValue V1, SDValue V2,
13161 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
13184 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13188 if (SDValue Shift =
13195 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13201 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13209 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
13214 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask, DAG))
13221 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
13225 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
13251 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
13252 ArrayRef<int> Mask, SDValue V1,
13253 SDValue V2, SelectionDAG &DAG) {
13254 SDValue LowV = V1, HighV = V2;
13343 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13344 const APInt &Zeroable, SDValue V1, SDValue V2,
13352 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13360 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13395 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
13402 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13411 if (SDValue V = lowerShuffleAsElementInsertion(
13417 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13421 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13436 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask, DAG))
13447 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13448 const APInt &Zeroable, SDValue V1, SDValue V2,
13458 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13466 if (SDValue Shift =
13471 if (SDValue Rotate =
13479 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13501 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13505 if (SDValue Shift =
13512 if (SDValue V = lowerShuffleAsElementInsertion(
13520 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13524 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13529 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask, DAG))
13536 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
13540 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13556 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13566 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13567 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13568 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13588 static SDValue lowerV8I16GeneralSingleInputShuffle(
13589 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
14082 static SDValue lowerShuffleAsBlendOfPSHUFBs(
14083 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14092 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14093 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14123 SDValue V;
14145 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14146 const APInt &Zeroable, SDValue V1, SDValue V2,
14155 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
14160 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
14168 if (SDValue Shift =
14174 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
14179 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
14184 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
14188 if (SDValue V =
14193 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
14208 if (SDValue Shift =
14215 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
14221 if (SDValue V = lowerShuffleAsElementInsertion(
14229 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
14233 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
14238 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
14242 if (SDValue V =
14247 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
14252 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
14256 if (SDValue BitBlend =
14261 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
14274 SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL);
14283 SmallVector<SDValue, 4> DWordClearOps(4,
14287 SDValue DWordClearMask =
14295 SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8);
14306 SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2);
14330 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14349 static SDValue lowerV8F16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14350 const APInt &Zeroable, SDValue V1, SDValue V2,
14361 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f16, V1, V2,
14366 if (SDValue V = lowerShuffleAsElementInsertion(
14380 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14381 ArrayRef<int> OriginalMask, SDValue V1,
14382 SDValue V2, const X86Subtarget &Subtarget,
14393 SDValue MaskNode;
14413 SDValue Result;
14432 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14433 const APInt &Zeroable, SDValue V1, SDValue V2,
14441 if (SDValue Shift =
14447 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14452 if (SDValue V =
14457 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14462 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14466 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14472 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14481 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14486 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14490 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14508 auto tryToWidenViaDuplication = [&]() -> SDValue {
14510 return SDValue();
14543 return SDValue();
14585 if (SDValue V = tryToWidenViaDuplication())
14589 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14594 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14598 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
14626 SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14634 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14646 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14657 SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
14663 if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14673 if (SDValue V = lowerShuffleAsElementInsertion(
14677 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14695 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
14698 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
14706 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14736 SDValue V = V1;
14744 SDValue VLoHalf, VHiHalf;
14768 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14776 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14777 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14786 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14787 MVT VT, SDValue V1, SDValue V2,
14824 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14825 SDValue V2, ArrayRef<int> Mask,
14842 auto SplitVector = [&](SDValue V) {
14843 SDValue LoV, HiV;
14849 SDValue LoV1, HiV1, LoV2, HiV2;
14915 SDValue V1Blend, V2Blend;
14938 return SDValue();
14940 SDValue Lo = HalfBlend(LoMask);
14941 SDValue Hi = HalfBlend(HiMask);
14953 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14954 SDValue V2, ArrayRef<int> Mask,
15008 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
15009 SDValue V1, SDValue V2,
15030 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
15031 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
15044 static SDValue lowerShuffleAsLanePermuteAndPermute(
15045 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15057 auto getSublanePermute = [&](int NumSublanes) -> SDValue {
15092 return SDValue();
15114 return SDValue();
15121 return SDValue();
15129 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
15135 if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes))
15140 return SDValue();
15143 if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes * 2))
15149 return SDValue();
15175 static SDValue lowerShuffleAsLanePermuteAndShuffle(
15176 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15226 SDValue Flipped = DAG.getBitcast(PVT, V1);
15234 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
15235 SDValue V2, ArrayRef<int> Mask,
15248 if (SDValue BcstLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, DL,
15255 return SDValue();
15262 return SDValue();
15270 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
15282 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
15298 SDValue SubVec =
15354 static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
15355 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15360 return SDValue();
15388 return SDValue();
15438 return SDValue();
15458 return SDValue();
15462 return SDValue();
15468 return SDValue();
15481 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15487 return SDValue();
15498 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15504 return SDValue();
15578 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15592 SDValue V = (HalfIdx < 2 ? V1 : V2);
15599 SDValue Half1 = getHalfVector(HalfIdx1);
15600 SDValue Half2 = getHalfVector(HalfIdx2);
15601 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15603 SDValue Op0 = V;
15604 SDValue Op1 = DAG.getUNDEF(HalfVT);
15618 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15619 SDValue V2, ArrayRef<int> Mask,
15627 return SDValue();
15638 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15648 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15657 return SDValue();
15686 return SDValue();
15692 return SDValue();
15696 return SDValue();
15700 return SDValue();
15708 return SDValue();
15716 return SDValue();
15719 return SDValue();
15726 return SDValue();
15733 static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15734 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15772 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15783 return SDValue();
15792 return SDValue();
15796 return SDValue();
15824 return SDValue();
15870 return SDValue();
15902 return SDValue();
15904 SDValue RepeatedShuffle =
15926 if (SDValue Shuffle = ShuffleSubLanes(Scale))
15929 return SDValue();
15932 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15978 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15979 SDValue V2, ArrayRef<int> Mask,
15990 return SDValue();
16005 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
16006 SDValue V1, SDValue V2,
16014 return SDValue();
16018 return SDValue();
16028 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
16053 static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
16054 SDValue V1, SDValue V2,
16059 return SDValue();
16079 return SDValue();
16089 return SDValue();
16104 return SDValue();
16108 SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
16109 SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
16110 SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
16112 SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
16126 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16127 const APInt &Zeroable, SDValue V1, SDValue V2,
16134 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
16140 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
16164 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16169 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
16179 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask, DAG))
16182 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
16187 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
16212 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16221 if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
16227 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v4f64, V1, V2, Mask,
16246 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16247 const APInt &Zeroable, SDValue V1, SDValue V2,
16255 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
16259 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
16264 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
16270 if (SDValue Shift =
16296 if (SDValue Shift =
16303 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
16307 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v4i64, V1, V2, Mask,
16313 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
16318 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask, DAG))
16332 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16337 if (SDValue V =
16346 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16359 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16360 const APInt &Zeroable, SDValue V1, SDValue V2,
16367 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
16372 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
16381 if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG,
16385 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16407 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask, DAG))
16417 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16425 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16429 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16439 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16445 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8f32, V1, V2, Mask,
16455 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8f32, V1, V2,
16481 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16482 const APInt &Zeroable, SDValue V1, SDValue V2,
16495 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16502 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8i32, V1, V2,
16514 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16519 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16525 if (SDValue Shift =
16530 if (SDValue Rotate =
16548 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask, DAG))
16553 if (SDValue Shift =
16559 if (SDValue Rotate =
16565 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16569 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8i32, V1, V2, Mask,
16575 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16581 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16588 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask, DAG))
16593 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16601 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16602 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16603 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16610 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16623 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16624 const APInt &Zeroable, SDValue V1, SDValue V2,
16635 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16640 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16644 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16649 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask, DAG))
16653 if (SDValue V =
16658 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16663 if (SDValue Shift =
16669 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16675 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16681 if (SDValue Rotate =
16687 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
16693 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16711 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16721 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16726 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16733 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v16i16, V1, V2,
16746 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16747 const APInt &Zeroable, SDValue V1, SDValue V2,
16758 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16763 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16767 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16772 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask, DAG))
16776 if (SDValue V =
16781 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16786 if (SDValue Shift =
16792 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16798 if (SDValue Rotate =
16804 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16813 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
16816 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16824 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16834 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16839 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16847 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16854 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v32i8, V1, V2,
16868 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16869 SDValue V1, SDValue V2, const APInt &Zeroable,
16878 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16883 if (SDValue V =
16898 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16901 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16940 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16941 const APInt &Zeroable, SDValue V1, SDValue V2,
16954 return SDValue();
16962 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16975 SDValue SubVec =
17007 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
17023 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
17031 SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
17036 return SDValue();
17046 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17047 const APInt &Zeroable, SDValue V1, SDValue V2,
17076 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
17080 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask, DAG))
17084 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
17088 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8f64, V1, V2, Mask, Zeroable,
17092 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
17100 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17101 const APInt &Zeroable, SDValue V1, SDValue V2,
17125 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask, DAG))
17128 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
17136 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
17140 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17146 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17154 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
17159 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v16f32, V1, V2, Mask,
17167 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17168 const APInt &Zeroable, SDValue V1, SDValue V2,
17177 if (SDValue Shift =
17203 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
17208 if (SDValue Shift =
17214 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
17220 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
17224 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask, DAG))
17228 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
17232 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
17240 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17241 const APInt &Zeroable, SDValue V1, SDValue V2,
17253 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17259 if (SDValue Shift =
17264 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask,
17282 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask, DAG))
17287 if (SDValue Shift =
17293 if (SDValue Rotate =
17298 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
17304 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
17311 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
17312 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
17313 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
17320 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17325 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v16i32, V1, V2, Mask,
17329 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
17337 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17338 const APInt &Zeroable, SDValue V1, SDValue V2,
17349 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17354 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask, DAG))
17358 if (SDValue V =
17363 if (SDValue Shift =
17369 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
17375 if (SDValue Rotate =
17389 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
17393 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
17400 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17408 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17409 const APInt &Zeroable, SDValue V1, SDValue V2,
17420 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17425 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask, DAG))
17429 if (SDValue V =
17434 if (SDValue Shift =
17440 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17446 if (SDValue Rotate =
17451 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17455 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17461 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17465 if (SDValue Result = lowerShuffleAsLanePermuteAndPermute(
17469 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17476 if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v64i8, V1, V2,
17490 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17506 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17507 MVT VT, SDValue V1, SDValue V2,
17520 if (SDValue Insertion = lowerShuffleAsElementInsertion(
17525 if (SDValue V =
17530 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17537 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17540 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17580 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17581 MVT VT, SDValue V1, SDValue V2,
17586 return SDValue();
17602 return SDValue();
17606 return SDValue();
17611 SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL);
17655 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17656 MVT VT, SDValue V1, SDValue V2,
17691 SDValue Extract =
17700 if (SDValue Shift =
17706 for (SDValue V : {V1, V2}) {
17710 SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL);
17735 SDValue Op0 = V1.getOperand(0);
17736 SDValue Op1 = V1.getOperand(1);
17775 return SDValue();
17783 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17858 static bool canCombineAsMaskOperation(SDValue V,
17876 auto HasMaskOperation = [&](SDValue V) {
17911 static SDValue canonicalizeShuffleMaskWithHorizOp(
17912 MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
17923 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17927 SDValue V1 = Op.getOperand(0);
17928 SDValue V2 = Op.getOperand(1);
17992 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
18027 SmallVector<SDValue> Ops = {V1, V2};
18032 if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
18066 static SDValue lowerVECTOR_COMPRESS(SDValue Op, const X86Subtarget &Subtarget,
18072 SDValue Vec = Op.getOperand(0);
18073 SDValue Mask = Op.getOperand(1);
18074 SDValue Passthru = Op.getOperand(2);
18085 return SDValue();
18102 SDValue Compressed =
18118 SDValue Compressed =
18123 return SDValue();
18127 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
18130 SDValue Cond = Op.getOperand(0);
18131 SDValue LHS = Op.getOperand(1);
18132 SDValue RHS = Op.getOperand(2);
18143 return SDValue();
18146 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
18147 SDValue Cond = Op.getOperand(0);
18148 SDValue LHS = Op.getOperand(1);
18149 SDValue RHS = Op.getOperand(2);
18165 return SDValue();
18169 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
18181 return SDValue();
18188 return SDValue();
18196 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
18207 return SDValue();
18244 return SDValue();
18253 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
18259 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
18261 SDValue Vec = Op.getOperand(0);
18262 SDValue Idx = Op.getOperand(1);
18267 return SDValue();
18279 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,
18291 return SDValue();
18296 return SDValue();
18297 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18305 return SDValue();
18310 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
18312 SDValue Vec = Op.getOperand(0);
18315 SDValue Idx = Op.getOperand(1);
18335 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
18336 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
18389 SDValue
18390 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
18393 SDValue Vec = Op.getOperand(0);
18395 SDValue Idx = Op.getOperand(1);
18432 return SDValue();
18471 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
18477 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
18490 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18502 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
18541 return SDValue();
18546 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18549 SDValue Vec = Op.getOperand(0);
18550 SDValue Elt = Op.getOperand(1);
18551 SDValue Idx = Op.getOperand(2);
18560 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18567 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18571 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
18582 SDValue N0 = Op.getOperand(0);
18583 SDValue N1 = Op.getOperand(1);
18584 SDValue N2 = Op.getOperand(2);
18589 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
18602 return SDValue();
18607 return SDValue();
18609 SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT);
18610 SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt);
18611 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1);
18613 SmallVector<SDValue, 16> RawIndices;
18616 SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices);
18624 return SDValue();
18636 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType());
18637 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType());
18638 SmallVector<SDValue, 8> CstVectorElts(NumElts, ZeroCst);
18640 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts);
18650 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18667 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18684 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1);
18692 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18779 return SDValue();
18782 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
18813 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18821 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18828 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18834 SDValue Vec = Op.getOperand(0);
18877 SDValue
18878 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18886 SDValue Result = DAG.getTargetConstantPool(
18901 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18909 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18923 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18928 SDValue
18929 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18937 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18952 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18979 SDValue Result;
19026 SDValue
19027 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
19031 static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA,
19039 SDValue TGA;
19041 SDValue Chain = DAG.getEntryNode();
19042 SDValue Ret;
19059 Ret = SDValue(CopyFromRegOp, 0);
19073 SDValue InGlue;
19087 SDValue Glue = Chain.getValue(1);
19098 SDValue Offset =
19105 static SDValue
19113 static SDValue
19120 static SDValue
19126 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
19136 SDValue Base;
19154 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
19157 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
19164 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
19173 SDValue ThreadPointer =
19197 SDValue TGA =
19200 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
19218 SDValue
19219 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
19267 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
19270 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
19280 SDValue Chain = DAG.getEntryNode();
19283 SDValue Args[] = { Chain, Offset };
19310 SDValue Chain = DAG.getEntryNode();
19319 SDValue TlsArray = Subtarget.is64Bit()
19325 SDValue ThreadPointer =
19328 SDValue res;
19333 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
19341 SDValue Scale =
19351 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
19354 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
19391 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
19392 SDValue Lo, Hi;
19399 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, const SDLoc &dl,
19409 SDValue Src = Op.getOperand(OpNo);
19415 return SDValue();
19424 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
19426 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
19428 SDValue Chain = CvtVec.getValue(1);
19429 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19434 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
19441 static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
19449 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19454 return SDValue();
19460 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
19462 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},
19464 SDValue Chain = CvtVec.getValue(1);
19465 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19470 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);
19501 static SDValue vectorizeExtractedCast(SDValue Cast, const SDLoc &DL,
19506 SDValue Extract = Cast.getOperand(0);
19510 return SDValue();
19513 SDValue VecOp = Extract.getOperand(0);
19519 return SDValue();
19535 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
19543 static SDValue lowerFPToIntToFP(SDValue CastToFP, const SDLoc &DL,
19547 SDValue CastToInt = CastToFP.getOperand(0);
19550 return SDValue();
19553 SDValue X = CastToInt.getOperand(0);
19556 return SDValue();
19562 return SDValue();
19583 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
19584 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19585 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19586 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19590 static SDValue lowerINT_TO_FP_vXi64(SDValue Op, const SDLoc &DL,
19595 SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
19611 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19615 SDValue Res, Chain;
19635 return SDValue();
19637 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19638 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
19639 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19642 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19643 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19644 SmallVector<SDValue, 4> SignCvts(4);
19645 SmallVector<SDValue, 4> Chains(4);
19647 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19658 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19660 SDValue Slow, Chain;
19671 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19679 static SDValue promoteXINT_TO_FP(SDValue Op, const SDLoc &dl,
19682 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19683 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19687 SDValue Rnd = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
19718 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
19722 SDValue Src = Op.getOperand(OpNo);
19723 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19736 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
19739 if (SDValue R = lowerFPToIntToFP(Op, dl, DAG, Subtarget))
19758 return SDValue();
19773 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
19775 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
19780 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19789 return SDValue();
19791 SDValue ValueToStore = Src;
19805 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19807 std::pair<SDValue, SDValue> Tmp =
19816 std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
19817 EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19827 SDValue FILDOps[] = {Chain, Pointer};
19828 SDValue Result =
19839 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19841 SDValue FSTOps[] = {Chain, Result, StackSlot};
19868 static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
19894 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19904 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19907 SDValue XR1 =
19909 SDValue CLod0 = DAG.getLoad(
19912 SDValue Unpck1 =
19915 SDValue CLod1 = DAG.getLoad(
19918 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19920 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
19921 SDValue Result;
19927 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19936 static SDValue LowerUINT_TO_FP_i32(SDValue Op, const SDLoc &dl,
19941 SDValue Bias = DAG.getConstantFP(
19945 SDValue Load =
19952 SDValue Or = DAG.getNode(
19964 SDValue Chain = Op.getOperand(0);
19965 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19972 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19980 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19986 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, const SDLoc &DL,
19990 return SDValue();
19994 SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
20001 return SDValue();
20005 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
20007 SDValue Chain = Res.getValue(1);
20026 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
20027 SDValue VBias = DAG.getConstantFP(
20029 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
20039 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
20043 SDValue V = Op->getOperand(IsStrict ? 1 : 0);
20063 SDValue Tmp =
20067 SDValue Res, Chain;
20086 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
20091 SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
20093 SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
20094 SDValue VBias = DAG.getMemIntrinsicNode(
20099 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
20126 return SDValue();
20136 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
20138 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
20141 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
20142 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
20144 SDValue Low, High;
20148 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
20149 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
20156 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
20157 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
20163 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
20165 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
20173 SDValue VecCstFSub = DAG.getConstantFP(
20180 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
20183 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
20186 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
20192 SDValue FHigh =
20197 static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
20200 SDValue N0 = Op.getOperand(OpNo);
20217 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
20221 SDValue Src = Op.getOperand(OpNo);
20226 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
20230 return SDValue();
20243 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
20262 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
20264 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
20279 return SDValue();
20282 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
20288 SDValue OffsetSlot =
20290 SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign);
20291 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
20293 std::pair<SDValue, SDValue> Tmp =
20302 SDValue ValueToStore = Src;
20309 SDValue Store =
20315 SDValue Ops[] = {Store, StackSlot};
20316 SDValue Fild =
20322 SDValue SignSet = DAG.getSetCC(
20328 SDValue FudgePtr =
20333 SDValue Zero = DAG.getIntPtrConstant(0, dl);
20334 SDValue Four = DAG.getIntPtrConstant(4, dl);
20335 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
20339 SDValue Fudge = DAG.getExtLoad(
20352 SDValue Add =
20366 SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge);
20373 // just return an SDValue().
20377 SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
20379 SDValue &Chain) const {
20384 SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
20391 return SDValue();
20418 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
20422 SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
20456 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
20460 SDValue Cmp;
20480 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);
20481 SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);
20484 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal,
20503 SDValue Ops[] = { Chain, StackSlot };
20516 SDValue Ops[] = { Chain, Value, StackSlot };
20517 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
20521 SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
20531 static SDValue LowerAVXExtend(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
20534 SDValue In = Op.getOperand(0);
20575 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20583 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20584 SDValue Undef = DAG.getUNDEF(InVT);
20586 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20593 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
20596 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20598 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20602 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20606 static SDValue LowerZERO_EXTEND_Mask(SDValue Op, const SDLoc &DL,
20610 SDValue In = Op->getOperand(0);
20618 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20643 SDValue One = DAG.getConstant(1, DL, WideVT);
20644 SDValue Zero = DAG.getConstant(0, DL, WideVT);
20646 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20662 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
20664 SDValue In = Op.getOperand(0);
20680 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
20689 return SDValue();
20699 return SDValue();
20725 SDValue LHS = DAG.getBitcast(InVT, In);
20726 SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS;
20727 SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS);
20734 SDValue Lo, Hi;
20740 if (SDValue Res =
20753 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20762 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20786 SDValue Res =
20794 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20802 static SDValue truncateVectorWithPACKUS(EVT DstVT, SDValue In, const SDLoc &DL,
20810 static SDValue truncateVectorWithPACKSS(EVT DstVT, SDValue In, const SDLoc &DL,
20822 static SDValue matchTruncateWithPACK(unsigned &PackOpcode, EVT DstVT,
20823 SDValue In, const SDLoc &DL,
20829 return SDValue();
20840 return SDValue();
20851 return SDValue();
20858 return SDValue();
20862 return SDValue();
20889 return SDValue();
20908 return SDValue();
20914 static SDValue LowerTruncateVecPackWithSignBits(
20915 MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget,
20922 return SDValue();
20927 SmallVector<SDValue> LowerOps;
20928 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20930 if (SDValue Res = LowerTruncateVecPackWithSignBits(DstHalfVT, Lo, DL,
20938 if (SDValue Src = matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG,
20942 return SDValue();
20947 static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL,
20957 return SDValue();
20962 return SDValue();
20964 return SDValue();
20970 SmallVector<SDValue> LowerOps;
20971 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20973 if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG))
20991 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In);
20995 return SDValue();
20998 static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
21002 SDValue In = Op.getOperand(0);
21038 SDValue Lo, Hi;
21077 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
21080 SDValue In = Op.getOperand(0);
21095 SDValue Lo, Hi;
21109 if (SDValue SignPack = LowerTruncateVecPackWithSignBits(
21118 return SDValue();
21127 if (SDValue SignPack = LowerTruncateVecPackWithSignBits(
21160 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
21162 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
21201 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl,
21210 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src);
21211 SDValue Big =
21226 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big);
21230 SDValue IsOverflown =
21237 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
21242 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21243 SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue();
21247 SDValue Res;
21279 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
21309 SDValue Tmp =
21311 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
21383 SDValue Tmp =
21413 SDValue Tmp =
21439 return SDValue();
21441 SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
21442 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
21446 SDValue Chain = Tmp.getValue(1);
21453 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
21473 return SDValue();
21491 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT,
21498 SDValue Small =
21501 SDValue Big = DAG.getNode(
21512 SDValue IsOverflown = DAG.getNode(
21520 return SDValue();
21544 return SDValue();
21578 std::pair<SDValue, SDValue> Tmp =
21588 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
21597 SDValue X86TargetLowering::LowerLRINT_LLRINT(SDValue Op,
21599 SDValue Src = Op.getOperand(0);
21604 return DstVT.getScalarType() == MVT::i32 ? Op : SDValue();
21607 return SDValue();
21616 SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
21619 SDValue Src = N->getOperand(0);
21625 return SDValue();
21629 SDValue Chain = DAG.getEntryNode();
21636 SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
21645 SDValue Ops[] = { Chain, StackPtr };
21653 SDValue StoreOps[] = { Chain, Src, StackPtr };
21661 SDValue
21662 X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
21668 SDLoc dl(SDValue(Node, 0));
21669 SDValue Src = Node->getOperand(0);
21682 return SDValue();
21731 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
21732 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
21739 SDValue MinClamped = DAG.getNode(
21742 SDValue BothClamped = DAG.getNode(
21745 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped);
21753 SDValue MinClamped = DAG.getNode(
21756 SDValue BothClamped = DAG.getNode(
21759 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
21768 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21773 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
21774 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
21777 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src);
21785 SDValue Select = FpToInt;
21807 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21812 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21817 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21818 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21825 return SDValue();
21848 return SDValue();
21865 SDValue Callee = DAG.getExternalSymbol(
21872 SDValue Res;
21884 SDValue Res;
21910 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,
21922 SDValue Res =
21930 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21934 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21935 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21940 return SDValue();
21945 return SDValue();
21959 SDValue Callee = DAG.getExternalSymbol(
21967 SDValue Res;
21983 return SDValue();
21988 return SDValue();
21993 SDValue Res;
21994 SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
22022 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
22024 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
22029 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
22033 SDValue Chain;
22051 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
22053 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
22058 SDValue Res, Chain;
22083 SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
22090 SDValue Res;
22100 SDValue Res =
22107 static SDValue lowerAddSubToHorizontalOp(SDValue Op, const SDLoc &DL,
22111 SDValue LHS = Op.getOperand(0);
22112 SDValue RHS = Op.getOperand(1);
22154 SDValue X = LHS.getOperand(0);
22174 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
22181 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
22191 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
22192 SDValue N0 = Op.getOperand(0);
22203 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
22213 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
22254 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
22256 SDValue Op0 = Op.getOperand(0);
22261 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
22269 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
22274 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
22275 SDValue Mag = Op.getOperand(0);
22276 SDValue Sign = Op.getOperand(1);
22312 SDValue SignMask = DAG.getConstantFP(
22314 SDValue MagMask = DAG.getConstantFP(
22320 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
22325 SDValue MagBits;
22338 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
22344 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
22345 SDValue N0 = Op.getOperand(0);
22355 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
22363 static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
22374 return SDValue();
22404 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
22412 static bool isOrXorXorTree(SDValue X, bool Root = true) {
22424 static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG,
22426 SDValue Op0 = X.getOperand(0);
22427 SDValue Op1 = X.getOperand(1);
22429 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22430 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22438 SDValue A = SToV(Op0);
22439 SDValue B = SToV(Op1);
22451 static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y,
22462 return SDValue();
22471 return SDValue();
22474 auto IsVectorBitCastCheap = [](SDValue X) {
22481 return SDValue();
22525 auto ScalarToVector = [&](SDValue X) -> SDValue {
22529 SDValue OrigX = X.getOperand(0);
22551 SDValue Cmp;
22559 SDValue VecX = ScalarToVector(X);
22560 SDValue VecY = ScalarToVector(Y);
22578 SDValue BCCmp =
22580 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
22582 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
22590 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
22591 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
22595 return SDValue();
22602 static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
22603 SmallVectorImpl<SDValue> &SrcOps,
22605 SmallVector<SDValue, 8> Opnds;
22606 DenseMap<SDValue, APInt> SrcOpMap;
22617 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
22636 SDValue Src = I->getOperand(0);
22637 DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
22658 for (SDValue &SrcOp : SrcOps)
22671 static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS,
22679 return SDValue();
22684 return SDValue();
22688 return SDValue();
22695 auto MaskBits = [&](SDValue Src) {
22699 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
22708 return SDValue();
22713 SDValue Lo =
22715 SDValue Hi =
22731 return SDValue();
22740 return SDValue();
22765 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ);
22778 SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
22794 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE);
22802 SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS);
22810 SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS);
22819 static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS,
22829 return SDValue();
22831 SDValue Op = LHS;
22833 return SDValue();
22842 SDValue Src = Op.getOperand(0);
22862 SmallVector<SDValue, 8> VecIns;
22866 [VT](SDValue V) { return VT == V.getValueType(); }) &&
22871 return SDValue();
22879 SDValue LHS = VecIns[Slot];
22880 SDValue RHS = VecIns[Slot + 1];
22894 if (SDValue Match =
22907 SDValue Src = peekThroughBitcasts(Op);
22913 SDValue LHS = Src.getOperand(0);
22914 SDValue RHS = Src.getOperand(1);
22928 SDValue Inner = Src.getOperand(0);
22942 return SDValue();
22946 static bool hasNonFlagsUse(SDValue Op) {
22966 static bool isProfitableToUseFlagOp(SDValue Op) {
22978 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
23022 SDValue ArithOp = Op;
23061 return SDValue(Op.getNode(), 1);
23079 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
23081 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
23082 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
23083 return SDValue(New.getNode(), 1);
23088 static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
23147 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
23156 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
23162 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
23172 SDNode *N, SDValue, SDValue IntPow2) const {
23190 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
23208 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
23235 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
23248 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
23249 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
23257 return SDValue();
23262 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
23283 return SDValue();
23299 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
23300 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
23308 return SDValue();
23321 SDValue
23327 return SDValue(N,0); // Lower SDIV as SDIV
23335 return SDValue();
23342 return SDValue();
23347 return SDValue();
23354 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl,
23357 SDValue Op0 = And.getOperand(0);
23358 SDValue Op1 = And.getOperand(1);
23364 SDValue Src, BitNo;
23376 return SDValue();
23384 SDValue AndLHS = Op0;
23404 return SDValue();
23413 if (SDValue BT = getBT(Src, BitNo, dl, DAG)) {
23418 return SDValue();
23428 static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
23429 SDValue &Op1, bool &IsAlwaysSignaling) {
23491 static SDValue splitVSETCC(EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond,
23495 SDValue CC = DAG.getCondCode(Cond);
23498 SDValue LHS1, LHS2;
23502 SDValue RHS1, RHS2;
23513 static SDValue LowerIntVSETCC_AVX512(SDValue Op, const SDLoc &dl,
23515 SDValue Op0 = Op.getOperand(0);
23516 SDValue Op1 = Op.getOperand(1);
23517 SDValue CC = Op.getOperand(2);
23537 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc,
23541 return SDValue();
23546 SmallVector<SDValue, 8> NewVecC;
23551 return SDValue();
23556 return SDValue();
23559 return SDValue();
23571 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
23576 return SDValue();
23580 return SDValue();
23584 return SDValue();
23592 return SDValue();
23593 SDValue ULEOp1 =
23596 return SDValue();
23605 SDValue UGEOp1 =
23608 return SDValue();
23621 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
23626 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
23630 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23631 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23632 SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
23643 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23646 return SDValue();
23668 SDValue Cmp = DAG.getNode(Op.getOpcode(), dl, DVT,
23698 SDValue Cmp;
23710 return SDValue();
23714 SDValue SignalCmp = DAG.getNode(
23742 SDValue Cmp0, Cmp1;
23857 SDValue BC0 = peekThroughBitcasts(Op0);
23874 SDValue Result = Op0.getOperand(0);
23924 if (SDValue UGTOp1 =
23932 if (SDValue ULTOp1 =
23950 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23962 if (SDValue V =
23991 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23993 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
24002 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24004 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
24016 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24018 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
24026 SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL
24038 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24039 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
24044 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
24045 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
24046 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
24048 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
24067 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
24071 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
24085 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
24091 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
24101 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
24104 SDValue &X86CC) {
24109 return SDValue();
24116 return SDValue();
24125 return SDValue();
24136 SDValue LHS = Op0.getOperand(0);
24137 SDValue RHS = Op0.getOperand(1);
24143 SDValue LHS = Op0;
24144 SDValue RHS = Op0;
24156 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
24159 SDValue &X86CC) const {
24169 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) {
24176 if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG,
24183 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
24213 SDValue Neg = DAG.getNode(X86ISD::SUB, dl, CmpVTs,
24215 return SDValue(Neg.getNode(), 1);
24226 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
24228 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
24231 return SDValue(New.getNode(), 1);
24240 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
24245 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
24254 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
24255 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
24256 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
24262 return SDValue();
24308 SDValue X86CC;
24309 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
24310 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
24326 return SDValue();
24328 SDValue EFLAGS;
24339 SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
24340 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
24344 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
24345 SDValue LHS = Op.getOperand(0);
24346 SDValue RHS = Op.getOperand(1);
24347 SDValue Carry = Op.getOperand(2);
24348 SDValue Cond = Op.getOperand(3);
24360 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
24368 static std::pair<SDValue, SDValue>
24369 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
24371 SDValue Value, Overflow;
24372 SDValue LHS = Op.getOperand(0);
24373 SDValue RHS = Op.getOperand(1);
24414 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
24421 SDValue Value, Overflow;
24424 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
24430 static bool isX86LogicalCmp(SDValue Op) {
24444 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
24448 SDValue VOp0 = V.getOperand(0);
24455 static SDValue LowerSELECTWithCmpZero(SDValue CmpVal, SDValue LHS, SDValue RHS,
24462 return SDValue();
24469 SDValue Neg = CmpVal;
24487 SDValue Mask = SplatLSB(VT);
24488 SDValue Diff = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
24489 SDValue Flip = DAG.getNode(ISD::AND, DL, VT, Mask, Diff);
24493 SDValue Src1, Src2;
24545 SDValue Mask = SplatLSB(Src1.getValueType());
24546 SDValue And = DAG.getNode(ISD::AND, DL, Src1.getValueType(), Mask,
24552 SDValue Mask = SplatLSB(VT);
24553 SDValue Or = DAG.getNode(ISD::OR, DL, VT, Mask, Src1); // Mask | z
24560 SDValue Y = isAllOnesConstant(RHS) ? LHS : RHS;
24570 SDValue Sub;
24572 SDValue Zero = DAG.getConstant(0, DL, CmpVT);
24575 SDValue One = DAG.getConstant(1, DL, CmpVT);
24578 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
24584 return SDValue();
24587 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
24589 SDValue Cond = Op.getOperand(0);
24590 SDValue Op1 = Op.getOperand(1);
24591 SDValue Op2 = Op.getOperand(2);
24594 SDValue CC;
24608 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
24615 SDValue Cmp =
24623 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
24643 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
24644 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
24645 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
24650 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
24655 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
24656 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
24663 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
24669 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
24690 SDValue Cmp = Cond.getOperand(1);
24691 SDValue CmpOp0 = Cmp.getOperand(0);
24699 auto MatchFFSMinus1 = [&](SDValue Op1, SDValue Op2) {
24707 } else if (SDValue R = LowerSELECTWithCmpZero(CmpOp0, Op1, Op2, CondCode,
24721 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT);
24722 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt);
24738 if (SDValue R = LowerSELECTWithCmpZero(Cond, Op2, Op1, X86::COND_E, DL, DAG,
24749 SDValue Cmp = Cond.getOperand(1);
24763 SDValue Value;
24780 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
24803 SDValue Res =
24817 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
24821 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
24838 SDValue Ops[] = { Op2, Op1, CC, Cond };
24839 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
24845 SDValue Ops[] = { Op2, Op1, CC, Cond };
24849 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
24853 SDValue In = Op->getOperand(0);
24879 SDValue V;
24885 SDValue NegOne = DAG.getAllOnesConstant(dl, WideVT);
24886 SDValue Zero = DAG.getConstant(0, dl, WideVT);
24904 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24906 SDValue In = Op->getOperand(0);
24921 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
24924 SDValue In = Op->getOperand(0);
24933 return SDValue();
24935 return SDValue();
24939 return SDValue();
24983 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
24984 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
25007 SDValue Curr = In;
25008 SDValue SignExt = Curr;
25035 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
25036 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
25044 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
25047 SDValue In = Op->getOperand(0);
25083 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
25090 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
25097 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
25098 SDValue StoredVal = Store->getValue();
25109 return SDValue();
25112 SDValue Value0, Value1;
25115 SDValue Ptr0 = Store->getBasePtr();
25116 SDValue Ptr1 =
25118 SDValue Ch0 =
25122 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
25131 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
25133 SDValue StoredVal = Store->getValue();
25142 return SDValue();
25149 SmallVector<SDValue, 4> Stores;
25152 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(),
25154 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
25156 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
25165 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
25169 SDValue StoredVal = St->getValue();
25197 return SDValue();
25209 return SDValue();
25213 return SDValue();
25240 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
25252 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
25269 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
25276 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
25283 return SDValue();
25288 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
25298 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
25299 SDValue Chain = Op.getOperand(0);
25300 SDValue Cond = Op.getOperand(1);
25301 SDValue Dest = Op.getOperand(2);
25308 SDValue LHS = Cond.getOperand(0);
25309 SDValue RHS = Cond.getOperand(1);
25318 SDValue Value, Overflow;
25325 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25331 SDValue CCVal;
25332 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
25349 SDValue FalseBB = User->getOperand(1);
25356 SDValue Cmp =
25358 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25370 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
25371 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25380 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
25381 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25388 SDValue Value, Overflow;
25392 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25408 SDValue LHS = Cond;
25409 SDValue RHS = DAG.getConstant(0, dl, CondVT);
25411 SDValue CCVal;
25412 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
25422 SDValue
25423 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
25434 SDValue Chain = Op.getOperand(0);
25435 SDValue Size = Op.getOperand(1);
25446 SDValue Result;
25460 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
25491 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
25504 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
25506 SDValue Ops[2] = {Result, Chain};
25510 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
25522 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25532 SmallVector<SDValue, 8> MemOps;
25533 SDValue FIN = Op.getOperand(1);
25535 SDValue Store = DAG.getStore(
25551 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25559 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
25567 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
25577 SDValue Chain = Op.getOperand(0);
25578 SDValue SrcPtr = Op.getOperand(1);
25609 SDValue InstOps[] = {Chain, SrcPtr,
25614 SDValue VAARG = DAG.getMemIntrinsicNode(
25625 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
25635 SDValue Chain = Op.getOperand(0);
25636 SDValue DstPtr = Op.getOperand(1);
25637 SDValue SrcPtr = Op.getOperand(2);
25671 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
25672 SDValue SrcOp, uint64_t ShiftAmt,
25713 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
25714 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
25723 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
25724 SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
25767 SmallVector<SDValue> MaskElts(
25771 SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
25796 SDValue ByteShift = DAG.getTargetConstant(
25820 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
25835 SDValue Lo, Hi;
25854 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
25855 SDValue PreservedSrc,
25866 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25880 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
25881 SDValue PreservedSrc,
25893 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
25929 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
25930 SDValue EntryEBP) {
25947 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
25948 SDValue ParentFrameOffset =
25960 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
25965 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
25968 auto isRoundModeCurDirection = [](SDValue Rnd) {
25974 auto isRoundModeSAE = [](SDValue Rnd) {
25988 auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
26020 SDValue Rnd = Op.getOperand(2);
26027 return SDValue();
26033 SDValue Sae = Op.getOperand(2);
26041 return SDValue();
26046 SDValue Src2 = Op.getOperand(2);
26053 SDValue Rnd = Op.getOperand(3);
26060 return SDValue();
26067 SDValue Sae = Op.getOperand(3);
26075 return SDValue();
26082 SDValue Src1 = Op.getOperand(1);
26083 SDValue Src2 = Op.getOperand(2);
26084 SDValue Src3 = Op.getOperand(3);
26096 SDValue Rnd = Op.getOperand(4);
26103 return SDValue();
26111 SDValue Src4 = Op.getOperand(4);
26121 SDValue Src = Op.getOperand(1);
26122 SDValue PassThru = Op.getOperand(2);
26123 SDValue Mask = Op.getOperand(3);
26129 SDValue Rnd = Op.getOperand(4);
26137 return SDValue();
26144 SDValue Src = Op.getOperand(1);
26145 SDValue PassThru = Op.getOperand(2);
26146 SDValue Mask = Op.getOperand(3);
26147 SDValue Rnd = Op.getOperand(4);
26155 return SDValue();
26161 SDValue Src1 = Op.getOperand(1);
26162 SDValue Src2 = Op.getOperand(2);
26163 SDValue passThru = Op.getOperand(3);
26164 SDValue Mask = Op.getOperand(4);
26172 SDValue Rnd = Op.getOperand(5);
26180 return SDValue();
26189 SDValue RoundingMode = Op.getOperand(5);
26192 SDValue Sae = Op.getOperand(6);
26196 return SDValue();
26203 SDValue Src1 = Op.getOperand(1);
26204 SDValue Src2 = Op.getOperand(2);
26205 SDValue passThru = Op.getOperand(3);
26206 SDValue Mask = Op.getOperand(4);
26207 SDValue Rnd = Op.getOperand(5);
26209 SDValue NewOp;
26217 return SDValue();
26222 SDValue Src1 = Op.getOperand(1);
26223 SDValue Src2 = Op.getOperand(2);
26224 SDValue passThru = Op.getOperand(3);
26225 SDValue Mask = Op.getOperand(4);
26226 SDValue Sae = Op.getOperand(5);
26233 return SDValue();
26239 SDValue Src1 = Op.getOperand(1);
26240 SDValue Src2 = Op.getOperand(2);
26241 SDValue PassThru = Op.getOperand(3);
26242 SDValue Mask = Op.getOperand(4);
26243 SDValue NewOp;
26245 SDValue Rnd = Op.getOperand(5);
26251 return SDValue();
26258 SDValue Src1 = Op.getOperand(1);
26259 SDValue Src2 = Op.getOperand(2);
26260 SDValue PassThru = Op.getOperand(3);
26261 SDValue Mask = Op.getOperand(4);
26265 SDValue Sae = Op.getOperand(5);
26269 return SDValue();
26276 SDValue Src1 = Op.getOperand(1);
26277 SDValue Src2 = Op.getOperand(2);
26278 SDValue Src3 = Op.getOperand(3);
26279 SDValue PassThru = Op.getOperand(4);
26280 SDValue Mask = Op.getOperand(5);
26281 SDValue Sae = Op.getOperand(6);
26288 return SDValue();
26294 SDValue Src1 = Op.getOperand(1);
26295 SDValue Src2 = Op.getOperand(2);
26296 SDValue Src3 = Op.getOperand(3);
26297 SDValue PassThru = Op.getOperand(4);
26298 SDValue Mask = Op.getOperand(5);
26302 SDValue Sae = Op.getOperand(6);
26306 return SDValue();
26312 SDValue Src1 = Op.getOperand(1);
26313 SDValue Src2 = Op.getOperand(2);
26314 SDValue Src3 = Op.getOperand(3);
26323 SDValue Src1 = Op.getOperand(1);
26324 SDValue Src2 = Op.getOperand(2);
26331 SDValue Src1 = Op.getOperand(1);
26332 SDValue Src2 = Op.getOperand(2);
26333 SDValue Src3 = Op.getOperand(3);
26334 SDValue Mask = Op.getOperand(4);
26337 SDValue PassThru = Src3;
26344 SDValue NewOp;
26346 SDValue Rnd = Op.getOperand(5);
26352 return SDValue();
26367 SDValue Src1 = Op.getOperand(1);
26368 SDValue Imm = Op.getOperand(2);
26369 SDValue Mask = Op.getOperand(3);
26370 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
26371 SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
26375 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
26383 SDValue CC = Op.getOperand(3);
26384 SDValue Mask = Op.getOperand(4);
26389 SDValue Sae = Op.getOperand(5);
26394 return SDValue();
26401 SDValue Src1 = Op.getOperand(1);
26402 SDValue Src2 = Op.getOperand(2);
26403 SDValue CC = Op.getOperand(3);
26404 SDValue Mask = Op.getOperand(4);
26406 SDValue Cmp;
26408 SDValue Sae = Op.getOperand(5);
26412 return SDValue();
26418 SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
26422 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
26429 SDValue LHS = Op.getOperand(1);
26430 SDValue RHS = Op.getOperand(2);
26449 SDValue Comi = DAG.getNode(ComiOpCode, dl, MVT::i32, LHS, RHS);
26451 SDValue SetCC;
26458 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
26467 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
26486 SDValue LHS = Op.getOperand(1);
26487 SDValue RHS = Op.getOperand(2);
26489 SDValue Sae = Op.getOperand(4);
26491 SDValue FCmp;
26499 return SDValue();
26502 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
26509 SDValue SrcOp = Op.getOperand(1);
26510 SDValue ShAmt = Op.getOperand(2);
26525 SDValue Mask = Op.getOperand(3);
26526 SDValue DataToCompress = Op.getOperand(1);
26527 SDValue PassThru = Op.getOperand(2);
26540 SDValue Src1 = Op.getOperand(1);
26541 SDValue Src2 = Op.getOperand(2);
26542 SDValue Src3 = Op.getOperand(3);
26543 SDValue Imm = Op.getOperand(4);
26544 SDValue Mask = Op.getOperand(5);
26545 SDValue Passthru = (IntrData->Type == FIXUPIMM)
26551 SDValue Sae = Op.getOperand(6);
26555 return SDValue();
26558 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
26570 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26579 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26587 SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
26597 SDValue Res;
26604 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
26609 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
26610 SDValue Results[] = { SetCC, Res };
26617 SDValue Src = Op.getOperand(1);
26618 SDValue PassThru = Op.getOperand(2);
26619 SDValue Mask = Op.getOperand(3);
26631 SDValue Src = Op.getOperand(1);
26632 SDValue Src2 = Op.getOperand(2);
26633 SDValue PassThru = Op.getOperand(3);
26634 SDValue Mask = Op.getOperand(4);
26646 SDValue Src = Op.getOperand(1);
26647 SDValue Rnd = Op.getOperand(2);
26648 SDValue PassThru = Op.getOperand(3);
26649 SDValue Mask = Op.getOperand(4);
26673 SDValue Src = Op.getOperand(1);
26674 SDValue PassThru = Op.getOperand(2);
26675 SDValue Mask = Op.getOperand(3);
26693 default: return SDValue(); // Don't custom lower most intrinsics.
26778 SDValue LHS = Op.getOperand(1);
26779 SDValue RHS = Op.getOperand(2);
26780 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
26781 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
26840 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26842 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
26843 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
26855 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26868 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26887 SDValue Op1 = Op.getOperand(1);
26894 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
26899 SDValue FnOp = Op.getOperand(1);
26900 SDValue IncomingFPOp = Op.getOperand(2);
26937 SDValue Operation =
26941 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26943 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
26956 SDValue ShAmt = Op.getOperand(2);
27024 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27025 SDValue Src, SDValue Mask, SDValue Base,
27026 SDValue Index, SDValue ScaleOp, SDValue Chain,
27032 return SDValue();
27034 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27049 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
27050 SDValue Res =
27056 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
27057 SDValue Src, SDValue Mask, SDValue Base,
27058 SDValue Index, SDValue ScaleOp, SDValue Chain,
27065 return SDValue();
27067 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27087 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
27088 SDValue Res =
27094 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27095 SDValue Src, SDValue Mask, SDValue Base,
27096 SDValue Index, SDValue ScaleOp, SDValue Chain,
27102 return SDValue();
27104 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27118 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
27119 SDValue Res =
27125 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27126 SDValue Mask, SDValue Base, SDValue Index,
27127 SDValue ScaleOp, SDValue Chain,
27133 return SDValue();
27135 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27137 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
27138 SDValue Segment = DAG.getRegister(0, MVT::i32);
27141 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27142 SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
27144 return SDValue(Res, 0);
27155 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
27160 SmallVectorImpl<SDValue> &Results) {
27161 SDValue Chain = N->getOperand(0);
27162 SDValue Glue;
27171 SDValue N1Ops[] = {Chain, Glue};
27173 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
27174 Chain = SDValue(N1, 0);
27177 SDValue LO, HI;
27179 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
27183 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
27192 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
27200 SDValue Ops[] = { LO, HI };
27201 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
27213 SmallVectorImpl<SDValue> &Results) {
27217 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
27223 SDValue Chain = Results[1];
27226 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
27231 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
27233 SmallVector<SDValue, 3> Results;
27240 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
27242 SDValue Chain = Op.getOperand(0);
27243 SDValue RegNode = Op.getOperand(2);
27258 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
27260 SDValue Chain = Op.getOperand(0);
27261 SDValue EHGuard = Op.getOperand(2);
27277 static SDValue
27278 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
27279 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
27282 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
27283 SDValue Ops[] = { Chain, Val, Ptr, Undef };
27289 static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
27291 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
27294 SDValue Ops[] = { Chain, Val, Ptr, Mask };
27308 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
27322 SDValue Chain = Op->getOperand(0);
27323 SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
27324 SDValue Result =
27325 SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
27338 SDValue Result =
27390 SDValue Chain = Op->getOperand(0);
27408 SDValue Operation =
27411 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27418 SDValue Chain = Op.getOperand(0);
27430 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
27432 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
27442 SDValue Chain = Op.getOperand(0);
27464 SDValue Operation = DAG.getMemIntrinsicNode(
27467 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
27480 SDValue Chain = Op.getOperand(0);
27502 SDValue Operation = DAG.getMemIntrinsicNode(
27508 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
27519 SDValue Chain = Op.getOperand(0);
27521 SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain);
27522 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27570 SDValue Ops[] = {Op.getOperand(2), // Row
27581 SDValue Res0 = DAG.getTargetExtractSubreg(X86::sub_t0, DL, MVT::x86amx,
27582 SDValue(Res, 0));
27583 SDValue Res1 = DAG.getTargetExtractSubreg(X86::sub_t1, DL, MVT::x86amx,
27584 SDValue(Res, 0));
27585 return DAG.getMergeValues({Res0, Res1, SDValue(Res, 1)}, DL);
27592 SDValue Chain = Op.getOperand(0);
27593 SDValue Op1 = Op.getOperand(2);
27594 SDValue Op2 = Op.getOperand(3);
27599 SDValue Res =
27611 SDValue Chain = Op.getOperand(0);
27612 SDValue Op1 = Op.getOperand(2);
27613 SDValue Op2 = Op.getOperand(3);
27617 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);
27619 SDValue Res =
27633 SDValue Chain = Op.getOperand(0);
27634 SDValue Addr = Op.getOperand(2);
27635 SDValue Src1 = Op.getOperand(3);
27636 SDValue Src2 = Op.getOperand(4);
27637 SDValue CC = Op.getOperand(5);
27639 SDValue Operation = DAG.getMemIntrinsicNode(
27653 SDValue Chain = Op.getOperand(0);
27654 SDValue Op1 = Op.getOperand(2);
27655 SDValue Op2 = Op.getOperand(3);
27688 SDValue Chain = Op.getOperand(0);
27689 SDValue Op1 = Op.getOperand(2);
27690 SDValue Op2 = Op.getOperand(3);
27714 SDValue LockArith =
27721 return SDValue();
27731 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27735 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
27738 SDValue(Result.getNode(), 1)};
27739 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
27743 SDValue(Result.getNode(), 2));
27746 SDValue Chain = Op.getOperand(0);
27747 SDValue Src = Op.getOperand(2);
27748 SDValue Base = Op.getOperand(3);
27749 SDValue Index = Op.getOperand(4);
27750 SDValue Mask = Op.getOperand(5);
27751 SDValue Scale = Op.getOperand(6);
27757 SDValue Chain = Op.getOperand(0);
27758 SDValue Src = Op.getOperand(2);
27759 SDValue Base = Op.getOperand(3);
27760 SDValue Index = Op.getOperand(4);
27761 SDValue Mask = Op.getOperand(5);
27762 SDValue Scale = Op.getOperand(6);
27768 SDValue Chain = Op.getOperand(0);
27769 SDValue Base = Op.getOperand(2);
27770 SDValue Mask = Op.getOperand(3);
27771 SDValue Index = Op.getOperand(4);
27772 SDValue Src = Op.getOperand(5);
27773 SDValue Scale = Op.getOperand(6);
27782 SDValue Chain = Op.getOperand(0);
27783 SDValue Mask = Op.getOperand(2);
27784 SDValue Index = Op.getOperand(3);
27785 SDValue Base = Op.getOperand(4);
27786 SDValue Scale = Op.getOperand(5);
27792 SmallVector<SDValue, 2> Results;
27803 SmallVector<SDValue, 2> Results;
27816 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27818 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
27819 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
27821 Ret, SDValue(InTrans.getNode(), 1));
27826 SDValue Mask = Op.getOperand(4);
27827 SDValue DataToTruncate = Op.getOperand(3);
27828 SDValue Addr = Op.getOperand(2);
27829 SDValue Chain = Op.getOperand(0);
27844 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27845 SDValue Offset = DAG.getUNDEF(VMask.getValueType());
27859 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27869 return SDValue(); // handled in combineINTRINSIC_*
27873 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
27879 return SDValue();
27886 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
27888 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
27895 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
27900 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
27906 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
27937 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
27979 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
28005 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
28006 SDValue Chain = Op.getOperand(0);
28007 SDValue Offset = Op.getOperand(1);
28008 SDValue Handler = Op.getOperand(2);
28017 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
28020 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
28031 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
28049 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
28056 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
28063 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
28067 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
28069 SDValue Root = Op.getOperand(0);
28070 SDValue Trmp = Op.getOperand(1); // trampoline
28071 SDValue FPtr = Op.getOperand(2); // nested function
28072 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
28079 SDValue OutChains[6];
28092 SDValue Addr = Trmp;
28177 SDValue OutChains[4];
28178 SDValue Addr, Disp;
28212 SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
28242 SDValue StackSlot =
28247 SDValue Chain = Op.getOperand(0);
28248 SDValue Ops[] = {Chain, StackSlot};
28254 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
28258 SDValue Shift =
28265 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
28266 SDValue RetVal =
28276 SDValue X86TargetLowering::LowerSET_ROUNDING(SDValue Op,
28280 SDValue Chain = Op.getNode()->getOperand(0);
28285 SDValue StackSlot =
28292 SDValue Ops[] = {Chain, StackSlot};
28297 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);
28303 SDValue NewRM = Op.getNode()->getOperand(1);
28304 SDValue RMBits;
28332 SDValue ShiftValue =
28338 SDValue Shifted =
28350 SDValue OpsLD[] = {Chain, StackSlot};
28366 SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);
28394 SDValue X86TargetLowering::LowerGET_FPENV_MEM(SDValue Op,
28398 SDValue Chain = Op->getOperand(0);
28399 SDValue Ptr = Op->getOperand(1);
28425 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
28437 static SDValue createSetFPEnvNodes(SDValue Ptr, SDValue Chain, const SDLoc &DL,
28450 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
28461 SDValue X86TargetLowering::LowerSET_FPENV_MEM(SDValue Op,
28464 SDValue Chain = Op->getOperand(0);
28465 SDValue Ptr = Op->getOperand(1);
28473 SDValue X86TargetLowering::LowerRESET_FPENV(SDValue Op,
28477 SDValue Chain = Op.getNode()->getOperand(0);
28497 SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT);
28530 SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL,
28535 SmallVector<SDValue> MaskBits;
28549 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
28571 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
28572 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
28573 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
28579 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
28593 SmallVector<SDValue, 64> LUTVec;
28596 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
28603 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
28604 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
28606 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
28607 SDValue Lo = Op0;
28608 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
28609 SDValue HiZ;
28621 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
28633 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
28650 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
28651 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
28652 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
28661 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
28683 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
28702 SDValue PassThru = DAG.getUNDEF(OpVT);
28714 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
28729 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
28733 SDValue N0 = Op.getOperand(0);
28741 SDValue PassThru = DAG.getUNDEF(VT);
28754 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
28760 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
28777 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
28780 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
28806 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT);
28807 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT);
28808 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask);
28809 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt);
28815 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
28816 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
28829 SDValue Zero = DAG.getConstant(0, DL, VT);
28830 SDValue Result =
28833 SDValue SumDiff = Result.getValue(0);
28834 SDValue Overflow = Result.getValue(1);
28835 SDValue SatMin = DAG.getConstant(MinVal, DL, VT);
28836 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT);
28837 SDValue SumNeg =
28844 return SDValue();
28847 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
28855 SDValue N0 = Op.getOperand(0);
28856 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
28858 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
28859 SDValue(Neg.getNode(), 1)};
28865 SDValue Src = Op.getOperand(0);
28866 SDValue Neg = DAG.getNegative(Src, DL, VT);
28880 return SDValue();
28883 static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget,
28896 return SDValue();
28899 static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
28912 return SDValue();
28915 static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
28919 SDValue X = Op.getOperand(0);
28920 SDValue Y = Op.getOperand(1);
28934 SDValue Imm =
28972 auto MatchesZero = [](SDValue Op, APInt Zero) {
28980 for (const SDValue &OpVal : Op->op_values()) {
29002 SDValue NewX, NewY;
29018 SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
29022 SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
29024 SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
29025 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
29028 SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
29033 SDValue IsXSigned;
29035 SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
29036 SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
29040 SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
29043 SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
29044 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
29047 SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
29070 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
29075 SDValue IsNaN =
29081 static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
29104 SDValue LHS = DAG.getFreeze(Op.getOperand(0));
29105 SDValue RHS = DAG.getFreeze(Op.getOperand(1));
29106 SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, VTs, LHS, RHS);
29107 SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, VTs, RHS, LHS);
29119 SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
29120 SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
29121 SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, WideVTs, LHS, RHS);
29122 SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, WideVTs, RHS, LHS);
29123 SDValue AbsDiff = DAG.getNode(X86ISD::CMOV, dl, WideVT, Diff1, Diff0,
29131 return SDValue();
29134 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
29146 SDValue A = Op.getOperand(0);
29147 SDValue B = Op.getOperand(1);
29183 SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT));
29184 SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B);
29185 SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B);
29186 SDValue RLo = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BLo);
29187 SDValue RHi = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BHi);
29199 SDValue Undef = DAG.getUNDEF(VT);
29200 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
29201 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
29203 SDValue BLo, BHi;
29206 SmallVector<SDValue, 16> LoOps, HiOps;
29224 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
29225 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
29236 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
29237 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
29240 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
29244 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
29281 SDValue Zero = DAG.getConstant(0, dl, VT);
29284 SDValue AloBlo = Zero;
29288 SDValue AloBhi = Zero;
29290 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
29294 SDValue AhiBlo = Zero;
29296 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
29300 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
29306 static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
29310 SDValue *Low = nullptr) {
29326 SDValue Zero = DAG.getConstant(0, dl, VT);
29328 SDValue ALo, AHi;
29337 SDValue BLo, BHi;
29340 SmallVector<SDValue, 16> LoOps, HiOps;
29343 SDValue LoOp = B.getOperand(i + j);
29344 SDValue HiOp = B.getOperand(i + j + 8);
29376 SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
29377 SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
29385 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
29391 SDValue A = Op.getOperand(0);
29392 SDValue B = Op.getOperand(1);
29421 SDValue Odd0 =
29424 SDValue Odd1 =
29434 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
29439 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
29448 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
29453 SDValue Zero = DAG.getConstant(0, dl, VT);
29454 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
29456 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
29459 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
29481 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
29482 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
29483 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
29492 static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
29502 SDValue A = Op.getOperand(0);
29503 SDValue B = Op.getOperand(1);
29509 SDValue LHSLo, LHSHi;
29513 SDValue RHSLo, RHSHi;
29522 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo);
29523 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi);
29526 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29527 SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1),
29542 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
29543 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
29544 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
29546 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
29548 SDValue Ovf;
29550 SDValue High, LowSign;
29577 SDValue High =
29602 SDValue Low;
29603 SDValue High =
29606 SDValue Ovf;
29609 SDValue LowSign =
29623 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
29630 SmallVector<SDValue> Result;
29648 SDValue InChain = DAG.getEntryNode();
29656 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29669 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
29683 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
29687 SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
29689 SDValue &Chain) const {
29694 SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
29712 SDValue Result;
29721 SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
29727 SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
29743 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
29746 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29752 SDValue Result;
29823 static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
29827 SDValue R = Op.getOperand(0);
29828 SDValue Amt = Op.getOperand(1);
29835 SDValue Ex = DAG.getBitcast(ExVT, R);
29846 SDValue Upper =
29848 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
29857 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
29859 SDValue Lower =
29874 return SDValue();
29909 SDValue Mask = DAG.getAllOnesConstant(dl, VT);
29932 SDValue Zeros = DAG.getConstant(0, dl, VT);
29935 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
29943 return SDValue();
29946 SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt);
29953 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
29962 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
29971 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29973 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
29981 return SDValue();
29984 static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
29988 SDValue R = Op.getOperand(0);
29989 SDValue Amt = Op.getOperand(1);
29994 if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) {
30012 SDValue BitMask = DAG.getAllOnesConstant(dl, ExtVT);
30022 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
30031 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
30044 return SDValue();
30048 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
30058 return SDValue();
30068 SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
30090 SDValue Z = DAG.getConstant(0, dl, VT);
30091 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
30092 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
30100 return SDValue();
30103 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
30107 SDValue R = Op.getOperand(0);
30108 SDValue Amt = Op.getOperand(1);
30120 if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
30123 if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
30135 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
30136 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
30159 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
30160 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
30161 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
30162 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
30170 SDValue A = Amt.getOperand(I);
30213 SDValue Shift1 =
30215 SDValue Shift2 =
30234 SmallVector<SDValue, 32> AmtWideElts(Amt->op_begin(), Amt->op_end());
30235 SmallVector<SDValue, 32> TmpAmtWideElts;
30314 SDValue RWide = DAG.getBitcast(WideVT, R);
30316 SDValue AmtWide = DAG.getBuildVector(
30321 SDValue ShiftedR = DAG.getNode(LogicalOpc, dl, WideVT, RWide, AmtWide);
30332 SDValue SplatFullMask = DAG.getAllOnesConstant(dl, VT);
30333 SDValue Mask = DAG.getNode(LogicalOpc, dl, VT, SplatFullMask, Amt);
30336 SDValue Masked = DAG.getNode(ISD::AND, dl, WideVT, ShiftedR, Mask);
30365 SDValue SplatHighBit =
30368 SDValue SignBitMask = DAG.getNode(LogicalOpc, dl, VT, SplatHighBit, Amt);
30369 SDValue FlippedSignBit =
30371 SDValue Subtraction =
30382 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
30389 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
30390 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
30391 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
30392 SDValue Zero = DAG.getConstant(0, dl, VT);
30393 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
30394 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
30408 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
30409 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
30410 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
30411 SDValue Amt0 =
30413 SDValue Amt1 =
30415 SDValue Sra1 =
30417 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
30429 SDValue Amt0, Amt1, Amt2, Amt3;
30441 SDValue Z = DAG.getConstant(0, dl, VT);
30447 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
30448 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
30450 SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG);
30451 SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG);
30460 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
30461 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
30462 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
30463 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
30468 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
30469 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
30472 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
30473 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
30504 SDValue Mask = getConstVector(LUT, Undefs, VT, DAG, dl);
30536 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
30555 SmallVector<SDValue, 16> LoAmt, HiAmt;
30563 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
30564 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
30566 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
30567 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
30582 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
30606 SDValue Z = DAG.getConstant(0, dl, SelVT);
30607 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
30620 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
30643 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
30644 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
30645 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
30646 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
30653 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
30654 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
30688 SDValue Z = DAG.getConstant(0, dl, VT);
30689 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
30690 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
30691 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
30692 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
30697 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
30698 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
30710 auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
30724 SDValue C =
30742 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
30775 return SDValue();
30778 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
30785 SDValue Op0 = Op.getOperand(0);
30786 SDValue Op1 = Op.getOperand(1);
30787 SDValue Amt = Op.getOperand(2);
30802 SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);
30835 SDValue ShX =
30838 SDValue ShY =
30848 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0,
30850 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1,
30855 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
30856 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30861 return SDValue();
30881 if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) {
30884 return SDValue();
30886 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30887 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30902 return SDValue();
30914 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1);
30925 SDValue Z = DAG.getConstant(0, DL, VT);
30926 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30927 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30928 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
30929 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
30930 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
30931 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
30936 return SDValue();
30950 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
30951 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
30955 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
30967 return SDValue();
30980 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
30986 SDValue R = Op.getOperand(0);
30987 SDValue Amt = Op.getOperand(1);
31023 SDValue Z = DAG.getConstant(0, DL, VT);
31028 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
31041 SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt);
31075 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R,
31077 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R,
31096 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
31097 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
31104 if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) {
31110 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
31111 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
31130 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
31131 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
31132 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
31133 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
31134 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
31135 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
31153 return SDValue();
31167 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
31180 SDValue Z = DAG.getConstant(0, DL, SelVT);
31181 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
31202 SDValue M;
31238 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
31240 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
31241 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
31258 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
31260 return SDValue();
31264 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
31265 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
31274 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
31275 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
31277 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
31280 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
31820 static SDValue emitLockedStackOp(SelectionDAG &DAG,
31821 const X86Subtarget &Subtarget, SDValue Chain,
31852 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31853 SDValue Ops[] = {
31863 return SDValue(Res, 1);
31866 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31867 SDValue Ops[] = {
31878 return SDValue(Res, 1);
31881 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
31896 SDValue Chain = Op.getOperand(0);
31904 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
31920 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
31921 Op.getOperand(2), SDValue());
31922 SDValue Ops[] = { cpIn.getValue(0),
31929 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
31932 SDValue cpOut =
31934 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
31936 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
31943 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
31948 SDValue Lo, Hi;
31959 SDValue Lo, Hi;
31971 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
31973 SDValue Src = Op.getOperand(0);
31983 SDValue Lo, Hi;
31995 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
32007 return SDValue();
32039 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
32055 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
32067 SDValue Zeros = DAG.getConstant(0, DL, VT);
32068 SDValue V32 = DAG.getBitcast(VT, V);
32069 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
32070 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
32096 SDValue ShifterV = DAG.getConstant(8, DL, VT);
32097 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
32103 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
32127 SmallVector<SDValue, 64> LUTVec;
32130 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
32131 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
32134 SDValue FourV = DAG.getConstant(4, DL, VT);
32135 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
32138 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
32143 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
32144 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
32150 static SDValue LowerVectorCTPOP(SDValue Op, const SDLoc &DL,
32156 SDValue Op0 = Op.getOperand(0);
32182 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
32183 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
32189 return SDValue();
32194 static SDValue LowerCTPOP(SDValue N, const X86Subtarget &Subtarget,
32197 SDValue Op = N.getOperand(0);
32243 SDValue LUT = DAG.getConstant(0x4332322132212110ULL, DL, MVT::i64);
32259 SDValue Mask11 = DAG.getConstant(0x11111111U, DL, MVT::i32);
32275 return SDValue(); // fallback to generic expansion.
32283 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
32285 SDValue In = Op.getOperand(0);
32292 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
32312 SmallVector<SDValue, 16> MaskElts;
32321 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
32322 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
32328 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
32337 SDValue In = Op.getOperand(0);
32354 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
32368 SDValue Res = DAG.getNode(ISD::BSWAP, DL, VT, In);
32380 SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT);
32388 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
32389 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
32390 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
32403 SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
32409 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
32410 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
32416 static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget,
32419 SDValue X = Op.getOperand(0);
32426 SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
32429 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32436 return SDValue();
32440 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
32443 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
32449 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,
32459 SDValue Hi = DAG.getNode(
32462 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
32464 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
32467 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32472 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
32504 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
32507 SDValue Chain = N->getOperand(0);
32508 SDValue LHS = N->getOperand(1);
32509 SDValue RHS = N->getOperand(2);
32550 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
32557 SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
32564 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
32572 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
32590 SDValue Chain;
32594 SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal());
32603 SDValue SclToVec =
32608 SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()};
32614 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
32621 SDValue LdOps[] = {Chain, StackPtr};
32622 SDValue Value = DAG.getMemIntrinsicNode(
32628 SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()};
32648 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
32654 static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
32661 return SDValue();
32667 SDValue Carry = Op.getOperand(2);
32673 SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
32678 SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
32686 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
32694 SDValue Arg = Op.getOperand(0);
32714 SDValue Callee =
32725 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
32732 SDValue SinVal =
32735 SDValue CosVal =
32744 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
32764 SDValue N1 = InOp.getOperand(1);
32775 SDValue FillVal =
32777 SmallVector<SDValue, 16> Ops(InOp->op_begin(), InOp->op_end());
32781 SDValue FillVal =
32787 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
32793 SDValue Src = N->getValue();
32798 SDValue Scale = N->getScale();
32799 SDValue Index = N->getIndex();
32800 SDValue Mask = N->getMask();
32801 SDValue Chain = N->getChain();
32802 SDValue BasePtr = N->getBasePtr();
32812 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
32816 return SDValue();
32824 return SDValue();
32845 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
32850 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
32856 SDValue Mask = N->getMask();
32858 SDValue PassThru = N->getPassThru();
32867 SDValue NewLoad = DAG.getMaskedLoad(
32873 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
32904 SDValue NewLoad = DAG.getMaskedLoad(
32909 SDValue Extract =
32912 SDValue RetOps[] = {Extract, NewLoad.getValue(1)};
32916 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
32919 SDValue DataToStore = N->getValue();
32922 SDValue Mask = N->getMask();
32958 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
32966 SDValue Index = N->getIndex();
32967 SDValue Mask = N->getMask();
32968 SDValue PassThru = N->getPassThru();
32975 return SDValue();
33001 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
33003 SDValue NewGather = DAG.getMemIntrinsicNode(
33006 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT, NewGather,
33011 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
33013 SDValue Src = Op.getOperand(0);
33034 SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
33042 SmallVector<SDValue, 2> Ops;
33048 return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
33052 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
33055 SDValue Lo, Hi;
33059 SDValue RC = Op.getOperand(1);
33065 static SDValue LowerPREFETCH(SDValue Op, const X86Subtarget &Subtarget,
33077 static SDValue LowerFCanonicalize(SDValue Op, SelectionDAG &DAG) {
33079 SDValue Operand = N->getOperand(0);
33083 SDValue One = DAG.getConstantFP(1.0, dl, VT);
33089 SDValue Chain = DAG.getEntryNode();
33090 SDValue StrictFmul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other},
33149 static SDValue getFlagsOfCmpZeroFori1(SelectionDAG &DAG, const SDLoc &DL,
33150 SDValue Mask) {
33157 return SDValue(CmpZero.getNode(), 1);
33160 SDValue X86TargetLowering::visitMaskedLoad(
33161 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO,
33162 SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const {
33175 SDValue Ops[] = {Chain, Ptr, ScalarPassThru, COND_NE, Flags};
33180 SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL,
33181 SDValue Chain,
33182 MachineMemOperand *MMO, SDValue Ptr,
33183 SDValue Val, SDValue Mask) const {
33193 SDValue Ops[] = {Chain, ScalarVal, Ptr, COND_NE, Flags};
33198 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
33363 SmallVectorImpl<SDValue>&Results,
33376 SDValue Lo, Hi;
33382 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33388 SDValue Lo, Hi;
33396 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
33398 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33404 Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG));
33414 SDValue Op = DAG.getNode(ISD::SRL, dl, MVT::i64, N->getOperand(0),
33427 SDValue Wide =
33447 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
33448 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
33449 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
33452 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
33465 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
33466 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
33467 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);
33470 SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);
33478 SDValue HiCmp;
33486 SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
33514 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
33516 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
33518 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
33520 SDValue Res = DAG.getNode(Opc, dl, WideVT, InVec0, InVec1);
33534 SDValue UNDEF = DAG.getUNDEF(VT);
33535 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
33537 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
33539 SDValue Res;
33564 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
33567 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
33568 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
33569 SDValue Res = DAG.getNode(Opc, dl, ResVT, N0, N1);
33575 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
33588 SDValue In = N->getOperand(0);
33598 if (SDValue Src = matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG,
33600 if (SDValue Res =
33616 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl, 128);
33648 SDValue Lo, Hi;
33653 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
33667 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl,
33684 SDValue In = N->getOperand(0);
33698 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
33699 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
33703 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
33706 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
33710 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33736 SDValue Lo = getEXTEND_VECTOR_INREG(Opc, dl, LoVT, In, DAG);
33745 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
33748 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33760 SDValue Op = N->getOperand(0);
33762 SDValue Res;
33780 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
33781 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
33784 SDValue Res;
33810 SDValue Tmp =
33812 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
33835 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT));
33855 SDValue Res;
33856 SDValue Chain;
33885 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
33903 SDValue Res =
33927 SDValue Res;
33928 SDValue Chain;
33947 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
33978 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
33979 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
33982 SDValue Chain;
33997 SDValue Chain;
33998 SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain);
34005 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
34014 if (SDValue V = LRINT_LLRINTHelper(N, DAG))
34026 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
34039 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
34056 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
34068 SDValue Zero = DAG.getConstant(0, dl, SrcVT);
34069 SDValue One = DAG.getConstant(1, dl, SrcVT);
34070 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
34073 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
34074 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
34075 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
34077 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
34086 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
34087 SDValue Slow, Chain;
34100 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
34118 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
34126 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
34127 SDValue VBias = DAG.getConstantFP(
34129 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
34133 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
34135 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
34142 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
34150 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
34151 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
34152 SDValue Rnd = N->getOperand(IsStrict ? 2 : 1);
34155 SDValue V;
34157 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)
34199 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
34202 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)
34204 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);
34250 SDValue cpInL, cpInH;
34254 Regs64bit ? X86::RAX : X86::EAX, cpInL, SDValue());
34258 SDValue swapInL, swapInH;
34271 SDValue Result;
34275 SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL,
34282 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
34288 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
34291 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
34294 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
34296 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
34298 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
34318 SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(),
34320 SDValue ResL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34322 SDValue ResH = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34336 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
34337 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
34340 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34349 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
34360 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
34361 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
34364 SDValue Chain = Result.getValue(1);
34370 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
34374 SDValue StoreOps[] = { Chain, Result, StackPtr };
34417 SDValue Lo, Hi;
34421 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
34432 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
34446 SDValue Index = Gather->getIndex();
34452 SDValue Mask = Gather->getMask();
34454 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
34464 SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
34466 SDValue Res = DAG.getMemIntrinsicNode(
34488 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
34491 SDValue Chain = Res.getValue(1);
34502 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
34503 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
34510 SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
34519 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
34527 SDValue VecOp = N->getOperand(0);
34529 SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0));
35153 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
35183 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
38008 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
38020 auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
38042 SDValue NewC =
38045 SDValue NewOp =
38094 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
38095 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
38099 static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS,
38116 static void computeKnownBitsForPMADDWD(SDValue LHS, SDValue RHS,
38138 static void computeKnownBitsForPMADDUBSW(SDValue LHS, SDValue RHS,
38162 const SDValue Op, const APInt &DemandedElts, unsigned Depth,
38172 [&DAG, Depth, KnownBitsFunc](SDValue Op, APInt &DemandedEltsOp) {
38187 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
38251 SDValue Src = Op.getOperand(0);
38316 SDValue Src = Op.getOperand(0);
38317 SDValue Idx = Op.getOperand(1);
38329 SDValue Src = Op.getOperand(0);
38364 SDValue LHS = Op.getOperand(0);
38365 SDValue RHS = Op.getOperand(1);
38391 SDValue LHS = Op.getOperand(0);
38392 SDValue RHS = Op.getOperand(1);
38401 SDValue LHS = Op.getOperand(0);
38402 SDValue RHS = Op.getOperand(1);
38433 SDValue Op0 = Op.getOperand(0);
38434 SDValue Op1 = Op.getOperand(1);
38562 SDValue LHS = Op.getOperand(1);
38563 SDValue RHS = Op.getOperand(2);
38574 SDValue LHS = Op.getOperand(1);
38575 SDValue RHS = Op.getOperand(2);
38586 SDValue LHS = Op.getOperand(1);
38587 SDValue RHS = Op.getOperand(2);
38604 SmallVector<SDValue, 2> Ops;
38651 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
38662 SDValue Src = Op.getOperand(0);
38681 auto NumSignBitsPACKSS = [&](SDValue V, const APInt &Elts) -> unsigned {
38682 SDValue BC = peekThroughBitcasts(V);
38686 SDValue BC0 = peekThroughBitcasts(BC.getOperand(0));
38687 SDValue BC1 = peekThroughBitcasts(BC.getOperand(1));
38710 SDValue Src = Op.getOperand(0);
38717 SDValue Src = Op.getOperand(0);
38728 SDValue Src = Op.getOperand(0);
38773 SmallVector<SDValue, 2> Ops;
38819 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
38826 // specified VT and memory VT. Returns SDValue() on failure.
38827 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,
38831 return SDValue();
38834 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
38845 SDValue V1, const SelectionDAG &DAG,
39148 SDValue &V1, SDValue &V2, const SDLoc &DL,
39295 auto computeKnownBitsElementWise = [&DAG](SDValue V) {
39344 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
39474 return SDValue();
39478 SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
39479 SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
39504 static SDValue combineX86ShuffleChainWithExtract(
39505 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
39519 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
39536 auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) {
39547 SDValue V1 = peekThroughBitcasts(Inputs[0]);
39548 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
39556 SDValue Res;
39616 return SDValue(); // Nothing to do!
39633 ArrayRef<int> ScaledMask, SDValue V1, SDValue V2,
39637 SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
39643 SDValue Op = ScaledMask[i] >= 4 ? V2 : V1;
39648 return SDValue();
39673 return SDValue(); // Nothing to do!
39675 if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG))
39687 return SDValue(); // Nothing to do!
39701 return SDValue(); // Nothing to do!
39702 SDValue Lo = CanonicalizeShuffleInput(RootVT, V1);
39703 SDValue Hi = CanonicalizeShuffleInput(RootVT, BaseMask[1] == 0 ? V1 : V2);
39709 return SDValue(); // Nothing to do!
39726 return SDValue(); // Nothing to do!
39737 SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
39738 SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
39780 return SDValue();
39809 return SDValue(); // Nothing to do!
39816 return SDValue(); // Nothing to do!
39829 return SDValue(); // Nothing to do!
39841 return SDValue(); // Nothing to do!
39856 SDValue SrcV1 = V1, SrcV2 = V2;
39861 return SDValue(); // Nothing to do!
39874 return SDValue(); // Nothing to do!
39884 SDValue NewV1 = V1; // Save operands in case early exit happens.
39885 SDValue NewV2 = V2;
39891 return SDValue(); // Nothing to do!
39905 return SDValue(); // Nothing to do!
39923 return SDValue(); // Nothing to do!
39933 return SDValue(); // Nothing to do!
39953 return SDValue(); // Nothing to do!
39972 return SDValue(); // Nothing to do!
39988 return SDValue();
40015 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
40059 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
40082 return SDValue();
40104 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
40118 SmallVector<SDValue, 16> VPermIdx;
40120 SDValue Idx =
40124 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
40159 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
40174 SmallVector<SDValue, 16> PSHUFBMask;
40193 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
40206 SmallVector<SDValue, 16> VPPERMMask;
40225 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
40232 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
40259 return SDValue();
40270 static SDValue combineX86ShuffleChainWithExtract(
40271 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
40278 return SDValue();
40288 for (SDValue Input : Inputs) {
40311 return SDValue();
40325 SmallVector<SDValue, 4> WideInputs(Inputs);
40327 SDValue &Input = WideInputs[I];
40366 return SDValue();
40392 SDValue WideRoot = WideInputs.front().getValueSizeInBits() >
40399 if (SDValue WideShuffle =
40408 return SDValue();
40413 static SDValue canonicalizeShuffleMaskWithHorizOp(
40414 MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
40418 return SDValue();
40420 SmallVector<SDValue> BC;
40421 for (SDValue Op : Ops)
40425 SDValue BC0 = BC[0];
40428 if (VT0.getSizeInBits() != RootSizeInBits || llvm::any_of(BC, [&](SDValue V) {
40431 return SDValue();
40437 return SDValue();
40440 bool OneUseOps = llvm::all_of(Ops, [](SDValue Op) {
40468 SDValue Src0 = BC[M / 4];
40469 SDValue Src1 = Src0.getOperand((M % 4) >= 2);
40472 return SDValue();
40474 SDValue M0 = GetHOpSrc(ScaledMask[0]);
40475 SDValue M1 = GetHOpSrc(ScaledMask[1]);
40476 SDValue M2 = GetHOpSrc(ScaledMask[2]);
40477 SDValue M3 = GetHOpSrc(ScaledMask[3]);
40479 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1);
40480 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3);
40486 SDValue LHS, RHS;
40491 SDValue Src = BC[M / 4].getOperand((M % 4) >= 2);
40511 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
40524 return SDValue();
40526 SDValue BC1 = BC[BC.size() - 1];
40532 auto ContainsOps = [](SDValue HOp, SDValue Op) {
40581 SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1;
40582 SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1;
40586 SDValue Undef = DAG.getUNDEF(SrcVT);
40587 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
40609 SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL);
40610 SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL);
40611 SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1);
40616 return SDValue();
40622 static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef<SDValue> Ops,
40640 return SDValue();
40647 llvm::none_of(Ops, [](SDValue SrcOp) { return SrcOp->hasOneUse(); }))
40648 return SDValue();
40701 return SDValue();
40703 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
40744 static SDValue combineX86ShufflesRecursively(
40745 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
40761 return SDValue();
40764 SDValue Op = SrcOps[SrcOpIndex];
40769 return SDValue(); // Bail if we hit a non-simple non-vector.
40773 return SDValue();
40801 SmallVector<SDValue, 2> OpInputs;
40808 if (llvm::any_of(OpInputs, [VT](SDValue OpInput) {
40811 return SDValue();
40815 SDValue SrcVec = Op.getOperand(0);
40823 return SDValue();
40847 SmallVector<SDValue, 16> Ops;
40879 auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
40881 SDValue InputBC = peekThroughBitcasts(Input);
40896 for (SDValue OpInput : OpInputs)
40983 SDValue &Op = Ops[I];
40999 for (SDValue &Op : Ops)
41045 if (SDValue Res = combineX86ShufflesRecursively(
41054 if (SDValue Cst = combineX86ShufflesConstants(
41060 if (Depth == 0 && llvm::all_of(Ops, [&](SDValue Op) {
41069 return SDValue();
41074 if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
41081 SDValue &Op = I.value();
41113 if (SDValue NewOp =
41123 if (any_of(Ops, [RootSizeInBits](SDValue Op) {
41126 for (SDValue &Op : Ops)
41156 if (SDValue Shuffle = combineX86ShuffleChain(
41163 SDValue LHS = peekThroughBitcasts(Ops.front());
41164 SDValue RHS = peekThroughBitcasts(Ops.back());
41170 return SDValue();
41181 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
41194 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
41197 SmallVector<SDValue, 2> Ops;
41236 static SDValue combineRedundantDWordShuffle(SDValue N,
41246 SmallVector<SDValue, 8> Chain;
41247 SDValue V = N.getOperand(0);
41251 return SDValue(); // Nothing combined!
41267 return SDValue();
41277 return SDValue();
41288 return SDValue();
41295 return SDValue();
41301 return SDValue(); // Nothing to combine.
41325 return SDValue();
41336 SDValue W = Chain.pop_back_val();
41366 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
41370 return SDValue();
41373 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
41375 return SDValue();
41376 SDValue N0 = V.getOperand(0);
41377 SDValue N1 = V.getOperand(1);
41382 return SDValue();
41390 if (SDValue NewSHUFP = commuteSHUFP(N, N.getOperand(0))) {
41397 SDValue N0 = N.getOperand(0);
41398 SDValue N1 = N.getOperand(1);
41401 if (SDValue NewSHUFP = commuteSHUFP(N, N0))
41404 } else if (SDValue NewSHUFP = commuteSHUFP(N, N0)) {
41407 } else if (SDValue NewSHUFP = commuteSHUFP(N, N1)) {
41415 return SDValue();
41420 static SDValue
41421 combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef<int> BlendMask,
41426 return SDValue();
41429 SDValue BC0 = peekThroughOneUseBitcasts(N0);
41430 SDValue BC1 = peekThroughOneUseBitcasts(N1);
41435 SmallVector<SDValue, 2> Ops0, Ops1;
41441 return SDValue();
41453 return SDValue();
41459 return SDValue();
41493 return SDValue();
41495 return SDValue();
41504 return SDValue();
41506 SDValue NewBlend =
41526 static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
41532 auto IsMergeableWithShuffle = [Opc, &DAG](SDValue Op, bool FoldShuf = true,
41549 auto IsSafeToMoveShuffle = [ShuffleVT](SDValue Op, unsigned BinOp) {
41561 SmallVector<SDValue> Ops;
41575 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
41579 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
41580 SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
41585 SDValue LHS, RHS;
41604 SDValue Op00 = DAG.getBitcast(ShuffleVT, N0.getOperand(0));
41605 SDValue Res =
41632 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
41633 SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
41639 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
41640 SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
41641 SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
41642 SDValue Op11 = peekThroughOneUseBitcasts(N1.getOperand(1));
41649 SDValue LHS, RHS;
41672 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
41673 SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
41674 SDValue Res;
41697 SDValue Res =
41706 return SDValue();
41710 static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
41716 SDValue Src0 = peekThroughBitcasts(V.getOperand(0));
41717 SDValue Src1 = peekThroughBitcasts(V.getOperand(1));
41724 return SDValue();
41728 SDValue LHS = Src0.getOperand(0);
41729 SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
41730 SDValue Res =
41748 SDValue LHS = Src0.getOperand(0);
41749 SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
41750 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS,
41758 return SDValue();
41761 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
41762 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
41767 static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
41779 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
41785 SDValue Src = N.getOperand(0);
41790 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
41791 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
41793 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
41799 return SDValue();
41802 SDValue Src = N.getOperand(0);
41803 SDValue BC = peekThroughBitcasts(Src);
41816 if (SDValue Res = combineX86ShufflesRecursively(
41871 // Ensure the same SDValue from the SDNode use is being used.
41877 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
41887 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
41888 SDValue BcastLd =
41895 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41898 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
41910 SDValue TruncIn = Src.getOperand(0);
41919 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
41920 SDValue BcastLd = DAG.getMemIntrinsicNode(
41925 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41937 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
41938 SDValue BcastLd =
41942 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41962 SDValue Ptr = DAG.getMemBasePlusOffset(
41964 SDValue Ops[] = { LN->getChain(), Ptr };
41965 SDValue BcastLd = DAG.getMemIntrinsicNode(
41971 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41983 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
41984 SDValue BcastLd =
41988 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
42002 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
42003 SDValue BcastLd = DAG.getMemIntrinsicNode(
42008 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
42014 return SDValue();
42017 SDValue N0 = N.getOperand(0);
42023 if (SDValue VZLoad =
42026 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
42039 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
42040 SDValue VZLoad =
42044 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
42056 SDValue In = N0.getOperand(0);
42059 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
42061 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
42062 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
42081 SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
42095 SDValue V = peekThroughOneUseBitcasts(N0);
42099 SDValue In = V.getOperand(1);
42104 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In);
42111 return SDValue();
42114 SDValue N0 = N.getOperand(0);
42115 SDValue N1 = N.getOperand(1);
42141 SmallVector<SDValue> Ops;
42142 SDValue LHS = peekThroughOneUseBitcasts(N0);
42143 SDValue RHS = peekThroughOneUseBitcasts(N1);
42153 SDValue MaskLHS = LHS.getOperand(1);
42154 SDValue MaskRHS = RHS.getOperand(1);
42156 if (SDValue NewMask = combineX86ShufflesConstants(
42159 SDValue NewLHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
42161 SDValue NewRHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
42170 return SDValue();
42179 SmallVector<SDValue> Ops;
42182 SmallVector<SDValue> SubOps;
42184 SDValue Sub = peekThroughBitcasts(Ops[i]);
42204 return SDValue();
42209 SDValue N0 = N.getOperand(0);
42210 SDValue N1 = N.getOperand(1);
42214 SDValue Src = N0.getOperand(0);
42216 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
42219 return SDValue();
42227 SDValue LHS = N->getOperand(0);
42228 SDValue RHS = N->getOperand(1);
42230 SmallVector<SDValue> LHSOps, RHSOps;
42231 SDValue NewLHS, NewRHS;
42247 return SDValue();
42251 SDValue LHS = N->getOperand(0);
42252 SDValue RHS = N->getOperand(1);
42265 if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
42272 return SDValue();
42273 SDValue Src = peekThroughBitcasts(N.getOperand(Idx < 2 ? 0 : 1));
42274 SmallVector<SDValue> SubOps;
42283 return SDValue();
42286 if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) {
42287 if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) {
42294 return SDValue();
42299 SDValue N0 = N.getOperand(0);
42300 SDValue N1 = N.getOperand(1);
42302 SDValue V = peekThroughOneUseBitcasts(N0);
42314 SDValue Res = DAG.getNode(Opcode, DL, VT,
42332 SDValue N0 = N.getOperand(0);
42333 SDValue N1 = N.getOperand(1);
42341 SDValue N10 = N1.getOperand(0);
42342 SDValue N11 = N1.getOperand(1);
42348 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
42351 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
42352 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
42357 return SDValue();
42361 SDValue Op0 = N.getOperand(0);
42362 SDValue Op1 = N.getOperand(1);
42380 SmallVector<SDValue, 2> Ops1;
42401 SmallVector<SDValue, 2> Ops0;
42424 return SDValue();
42452 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
42455 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
42459 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
42464 return SDValue();
42471 SDValue Ops[] = {N.getOperand(0), N.getOperand(2)};
42473 if (SDValue ConcatSrc =
42475 SDValue Mask = widenSubVector(N.getOperand(1), false, Subtarget, DAG,
42477 SDValue Perm = DAG.getNode(X86ISD::VPERMV, DL, WideVT, Mask, ConcatSrc);
42482 SmallVector<SDValue, 2> Ops;
42486 SDValue V1 = peekThroughBitcasts(N.getOperand(0));
42487 SDValue V2 = peekThroughBitcasts(N.getOperand(2));
42493 SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL,
42499 SDValue Ops[2];
42506 if (SDValue ConcatSrc =
42510 SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL,
42519 SDValue NewMask =
42525 return SDValue();
42528 return SDValue();
42536 SDValue V = N.getOperand(0);
42567 SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
42596 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DL, DAG))
42602 return SDValue();
42646 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
42661 SDValue V1 = N->getOperand(0);
42662 SDValue V2 = N->getOperand(1);
42676 SDValue LHS, RHS;
42705 static SDValue combineShuffleToFMAddSub(SDNode *N, const SDLoc &DL,
42712 return SDValue();
42717 return SDValue();
42720 SDValue Op0 = N->getOperand(0);
42721 SDValue Op1 = N->getOperand(1);
42722 SDValue FMAdd = Op0, FMSub = Op1;
42730 return SDValue();
42736 return SDValue();
42747 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
42750 if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG))
42753 SDValue Opnd0, Opnd1;
42756 return SDValue();
42761 SDValue Opnd2;
42768 return SDValue();
42774 return SDValue();
42780 return SDValue();
42788 static SDValue combineShuffleOfConcatUndef(SDNode *N, const SDLoc &DL,
42792 return SDValue();
42798 return SDValue();
42804 return SDValue();
42806 SDValue N0 = N->getOperand(0);
42807 SDValue N1 = N->getOperand(1);
42814 return SDValue();
42825 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
42833 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
42836 return SDValue();
42838 return SDValue();
42843 return SDValue();
42851 return SDValue();
42863 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
42867 if (SDValue V = narrowShuffle(Shuf, DAG))
42876 if (SDValue AddSub =
42881 if (SDValue LD = combineToConsecutiveLoads(
42882 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
42891 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, dl, DAG, Subtarget))
42895 SDValue Op(N, 0);
42896 if (SDValue Shuffle = combineTargetShuffle(Op, dl, DAG, DCI, Subtarget))
42904 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42911 return SDValue(N, 0);
42917 if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl))
42921 return SDValue();
42927 SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
42934 SDValue Mask = Op.getOperand(MaskIndex);
42946 SDValue BC = peekThroughOneUseBitcasts(Mask);
42984 SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
42985 SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
42986 SDValue NewMask = TLO.DAG.getLoad(
42994 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
43006 SDValue LHS = Op.getOperand(0);
43007 SDValue RHS = Op.getOperand(1);
43022 SDValue LHS = Op.getOperand(0);
43023 SDValue RHS = Op.getOperand(1);
43047 SDValue LHS = Op.getOperand(0);
43048 SDValue RHS = Op.getOperand(1);
43058 SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts(
43060 SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts(
43075 SDValue Amt = Op.getOperand(1);
43099 SDValue Src = Op.getOperand(0);
43112 if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
43125 SDValue LHS = Op.getOperand(0);
43126 SDValue RHS = Op.getOperand(1);
43147 SDValue LHS = Op.getOperand(0);
43148 SDValue RHS = Op.getOperand(1);
43158 SDValue Src = Op.getOperand(0);
43180 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
43197 SDValue Src = Op.getOperand(0);
43219 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
43237 SDValue LHS = Op.getOperand(0);
43238 SDValue RHS = Op.getOperand(1);
43240 auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
43283 SDValue NewLHS = SimplifyMultipleUseDemandedBits(LHS, BitsLHS, EltsLHS,
43285 SDValue NewRHS = SimplifyMultipleUseDemandedBits(RHS, BitsRHS, EltsRHS,
43300 SDValue Src = Op.getOperand(0);
43311 SDValue N0 = Op.getOperand(0);
43312 SDValue N1 = Op.getOperand(1);
43331 SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
43333 SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
43348 SDValue N0 = Op.getOperand(0);
43349 SDValue N1 = Op.getOperand(1);
43368 SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
43370 SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
43384 SDValue Src = Op.getOperand(0);
43398 if (SDValue R = combineBlendOfPermutes(
43427 SDValue Src = Op.getOperand(0);
43441 SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(),
43443 SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt);
43449 SDValue Src = Op.getOperand(0);
43469 if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
43510 SDValue Src = Op.getOperand(0);
43515 SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src);
43525 SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)};
43526 SDValue Bcst = TLO.DAG.getMemIntrinsicNode(
43529 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
43540 SDValue Ld =
43543 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
43551 if (SDValue BcstLd =
43571 SDValue Ext0 =
43573 SDValue ExtOp =
43575 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43576 SDValue Insert =
43588 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
43589 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43590 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
43605 SDValue ExtOp =
43607 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43608 SDValue Insert =
43619 SDValue SrcOp = Op.getOperand(0);
43625 SDValue ExtOp = TLO.DAG.getNode(
43627 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43628 SDValue Insert =
43671 SmallVector<SDValue, 4> Ops;
43673 SDValue SrcOp = Op.getOperand(i);
43684 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
43685 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43686 SDValue Insert =
43702 SmallVector<SDValue, 2> OpInputs;
43709 llvm::any_of(OpInputs, [VT](SDValue V) {
43774 SDValue NewShuffle = combineX86ShufflesRecursively(
43787 SDValue Op, const APInt &OriginalDemandedBits,
43796 SDValue Src = Op.getOperand(0);
43810 SDValue LHS = Op.getOperand(0);
43811 SDValue RHS = Op.getOperand(1);
43837 SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT);
43842 SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
43844 SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
43856 SDValue Op0 = Op.getOperand(0);
43857 SDValue Op1 = Op.getOperand(1);
43878 SDValue Op0 = Op.getOperand(0);
43879 SDValue Op1 = Op.getOperand(1);
43899 SDValue NewShift = TLO.DAG.getNode(
43925 if (SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
43927 SDValue NewOp =
43935 SDValue Op0 = Op.getOperand(0);
43936 SDValue Op1 = Op.getOperand(1);
43956 if (SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
43958 SDValue NewOp =
43966 SDValue Op0 = Op.getOperand(0);
43967 SDValue Op1 = Op.getOperand(1);
43981 SDValue Op00 = Op0.getOperand(0);
44013 if (SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
44015 SDValue NewOp =
44023 SDValue Sel = Op.getOperand(0);
44024 SDValue LHS = Op.getOperand(1);
44025 SDValue RHS = Op.getOperand(2);
44028 SDValue NewSel = SimplifyMultipleUseDemandedBits(
44030 SDValue NewLHS = SimplifyMultipleUseDemandedBits(
44032 SDValue NewRHS = SimplifyMultipleUseDemandedBits(
44046 SDValue Vec = Op.getOperand(0);
44072 if (SDValue V = SimplifyMultipleUseDemandedBits(
44084 SDValue Vec = Op.getOperand(0);
44085 SDValue Scl = Op.getOperand(1);
44131 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
44133 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
44136 SDValue Op0 = DemandedOp0 ? DemandedOp0 : Op.getOperand(0);
44137 SDValue Op1 = DemandedOp1 ? DemandedOp1 : Op.getOperand(1);
44144 SDValue Src = Op.getOperand(0);
44158 SDValue NewSrc =
44161 SDValue NewBcst =
44175 SDValue Src = Op.getOperand(0);
44187 SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src));
44214 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
44220 SDValue Op0 = Op.getOperand(0);
44221 SDValue Op1 = Op.getOperand(1);
44251 SDValue Op0 = Op.getOperand(0);
44252 SDValue Op1 = Op.getOperand(1);
44300 SDValue Op0 = Op.getOperand(0);
44301 SDValue Op1 = Op.getOperand(1);
44333 SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
44334 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
44344 SDValue Vec = Op.getOperand(0);
44355 SDValue Op0 = Op.getOperand(0);
44379 SDValue Cond = Op.getOperand(0);
44380 SDValue LHS = Op.getOperand(1);
44381 SDValue RHS = Op.getOperand(2);
44392 SDValue LHS = Op.getOperand(0);
44393 SDValue RHS = Op.getOperand(1);
44409 SmallVector<SDValue, 2> ShuffleOps;
44416 llvm::all_of(ShuffleOps, [VT](SDValue V) {
44454 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
44463 SmallVector<SDValue, 2> Ops;
44491 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
44528 bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
44549 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
44591 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
44595 return SDValue();
44602 SDValue Op0 = Src.getOperand(0);
44613 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
44614 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
44621 return SDValue();
44625 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
44626 SDValue Src, const SDLoc &DL) {
44656 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
44661 return SDValue();
44666 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
44697 return SDValue();
44701 SmallVector<SDValue, 4> SubSrcOps;
44704 SDValue LowerOp = SubSrcOps[0];
44705 ArrayRef<SDValue> UpperOps(std::next(SubSrcOps.begin()), SubSrcOps.end());
44707 all_of(UpperOps, [](SDValue Op) { return Op.isUndef(); })) {
44710 if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) {
44731 return SDValue();
44773 return SDValue();
44782 return SDValue();
44785 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
44805 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
44814 SDValue In = Op.getOperand(Idx);
44822 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
44828 return SDValue();
44832 return SDValue();
44835 SDValue Op = N->getOperand(0);
44839 return SDValue();
44845 return SDValue();
44852 return SDValue();
44854 SDValue LHS = Op.getOperand(0);
44855 SDValue RHS = Op.getOperand(1);
44875 return SDValue();
44878 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
44882 SDValue Splat = BV->getSplatValue();
44885 auto CreateMMXElement = [&](SDValue V) {
44902 SmallVector<SDValue, 8> Ops;
44943 SDValue Intrin = DAG.getTargetConstant(
44958 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
44963 return SDValue(); // Limit search depth.
44970 SDValue Src = V.getOperand(0);
44986 SDValue Src = V.getOperand(0);
44990 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
44999 SDValue Src = V.getOperand(0);
45003 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
45015 if (SDValue N0 = combineBitcastToBoolVector(VT, V.getOperand(0), DL, DAG,
45017 if (SDValue N1 = combineBitcastToBoolVector(VT, V.getOperand(1), DL, DAG,
45024 SDValue Src0 = V.getOperand(0);
45030 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget,
45042 return SDValue(Alt, 0);
45044 return SDValue();
45047 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
45050 SDValue N0 = N->getOperand(0);
45063 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
45088 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
45092 SmallVector<SDValue, 4> Ops(N0->ops());
45101 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
45112 if (SDValue V =
45147 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
45148 SDValue ResNode =
45151 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
45183 SDValue Op = N0.getOperand(i);
45188 SDValue N00 = N0.getOperand(0);
45208 SDValue N00 = N0.getOperand(0);
45217 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
45249 SDValue Src = N0;
45256 SDValue MovmskIn = Src.getOperand(0);
45268 SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
45276 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
45285 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
45299 default: return SDValue();
45309 return SDValue();
45311 SDValue LogicOp0 = N0.getOperand(0);
45312 SDValue LogicOp1 = N0.getOperand(1);
45320 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
45329 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
45334 return SDValue();
45338 static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
45339 SDValue &Op1) {
45347 auto IsFreeTruncation = [](SDValue &Op) -> bool {
45371 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
45372 SDValue AbsOp1 = Abs->getOperand(0);
45389 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
45412 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT));
45415 SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45417 SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45422 ArrayRef<SDValue> Ops) {
45427 SDValue Zero = DAG.getConstant(0, DL, DpVT);
45435 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
45436 const SDValue &Zext1, const SDLoc &DL,
45445 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
45448 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45450 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45454 ArrayRef<SDValue> Ops) {
45465 static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
45469 return SDValue();
45473 return SDValue();
45477 SDValue Src = DAG.matchBinOpReduction(
45480 return SDValue();
45485 return SDValue();
45488 SDValue MinPos = Src;
45492 SDValue Lo, Hi;
45503 SDValue Mask;
45520 SDValue Upper = DAG.getVectorShuffle(
45539 static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
45543 return SDValue();
45549 return SDValue();
45553 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
45557 return SDValue();
45562 return SDValue();
45564 SDValue Movmsk;
45575 return SDValue();
45583 SDValue LHS = DAG.getFreeze(Match.getOperand(0));
45584 SDValue RHS = DAG.getFreeze(Match.getOperand(1));
45586 if (SDValue V = LowerVectorAllEqual(DL, LHS, RHS, CC, Mask, Subtarget,
45599 SDValue Lo, Hi;
45608 return SDValue();
45615 return SDValue();
45622 return SDValue();
45626 return SDValue();
45629 SDValue Lo, Hi;
45643 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
45653 SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk);
45657 SDValue CmpC;
45673 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
45674 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
45678 static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
45681 return SDValue();
45687 return SDValue();
45691 return SDValue();
45695 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
45709 return SDValue();
45712 SDValue LHS, RHS;
45714 return SDValue();
45719 SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
45734 SDValue Shuffle =
45749 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
45753 return SDValue();
45759 return SDValue();
45763 return SDValue();
45767 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
45785 return SDValue();
45788 SDValue Zext0, Zext1;
45790 return SDValue();
45794 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
45808 SDValue Shuffle =
45831 static SDValue
45832 combineExtractFromVectorLoad(SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx,
45852 SDValue NewPtr = TLI.getVectorElementPointer(
45857 SDValue Load =
45864 return SDValue();
45869 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
45873 return SDValue();
45876 SDValue Src = N->getOperand(0);
45877 SDValue Idx = N->getOperand(1);
45887 return SDValue();
45891 return SDValue();
45893 SDValue SrcBC = peekThroughBitcasts(Src);
45897 SDValue SrcOp = SrcBC.getOperand(0);
45919 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
45924 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
45939 SDValue Scl = SrcBC.getOperand(0);
45965 auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT,
45991 return SDValue();
45996 SmallVector<SDValue, 2> Ops;
45998 return SDValue();
46001 if (llvm::any_of(Ops, [SrcVT](SDValue Op) {
46004 return SDValue();
46039 return SDValue();
46042 return SDValue();
46058 SDValue SrcOp = Ops[ExtractIdx / Mask.size()];
46060 if (SDValue V = GetLegalExtract(SrcOp, ExtractVT, ExtractIdx))
46064 if (SDValue V = combineExtractFromVectorLoad(
46068 return SDValue();
46073 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
46077 SDValue Vec = ExtElt->getOperand(0);
46078 SDValue Index = ExtElt->getOperand(1);
46085 return SDValue();
46092 return SDValue();
46096 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
46098 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
46105 return SDValue();
46119 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
46122 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
46124 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
46164 SmallVector<SDValue, 4> ExtOps;
46165 for (SDValue Op : Vec->ops())
46170 return SDValue();
46177 static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
46183 return SDValue();
46186 SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
46189 return SDValue();
46191 SDValue Index = ExtElt->getOperand(1);
46198 return SDValue();
46205 auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
46225 return SDValue();
46228 SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
46229 SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
46267 return SDValue();
46272 SDValue Lo, Hi;
46279 SDValue Hi = DAG.getVectorShuffle(
46309 ArrayRef<SDValue> Ops) {
46311 SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
46319 SDValue Lo, Hi;
46327 SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
46338 return SDValue();
46349 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
46350 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
46356 return SDValue();
46370 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
46373 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
46376 SDValue InputVector = N->getOperand(0);
46377 SDValue EltIdx = N->getOperand(1);
46408 SDValue Src = peekThroughBitcasts(InputVector);
46412 SDValue Sub = DAG.getNode(
46421 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumEltBits),
46423 return SDValue(N, 0);
46431 SDValue Scl = InputVector.getOperand(1);
46439 return SDValue();
46460 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
46463 if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
46467 if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
46471 if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
46475 if (SDValue V = combineArithReduction(N, DAG, Subtarget))
46478 if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget, DCI))
46482 if (SDValue V = combineExtractFromVectorLoad(
46515 if (SDValue BC =
46520 SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);
46521 SDValue MaskBit = DAG.getConstant(1, dl, BCVT);
46522 SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx);
46523 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
46527 return SDValue(N, 0);
46534 SDValue TruncSrc = InputVector.getOperand(0);
46537 SDValue NewExt =
46543 return SDValue();
46548 static SDValue combineToExtendBoolVectorInReg(
46549 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG,
46553 return SDValue();
46555 return SDValue();
46557 return SDValue();
46566 return SDValue();
46568 return SDValue();
46570 return SDValue();
46572 SDValue N00 = N0.getOperand(0);
46575 return SDValue();
46577 SDValue Vec;
46621 SmallVector<SDValue, 32> Bits;
46627 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
46646 static SDValue
46650 SDValue Cond = N->getOperand(0);
46651 SDValue LHS = N->getOperand(1);
46652 SDValue RHS = N->getOperand(2);
46658 return SDValue();
46682 return SDValue();
46696 SDValue CC = Cond.getOperand(2);
46709 return SDValue();
46716 return SDValue();
46720 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
46721 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
46727 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
46728 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
46734 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
46735 SDValue AndN;
46745 return SDValue();
46752 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
46756 return SDValue();
46761 return SDValue();
46764 SDValue Cond = N->getOperand(0);
46765 SDValue TVal = N->getOperand(1);
46766 SDValue FVal = N->getOperand(2);
46770 return SDValue();
46773 ArrayRef<SDValue> Ops) {
46780 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG,
46782 SDValue Cond = N->getOperand(0);
46783 SDValue LHS = N->getOperand(1);
46784 SDValue RHS = N->getOperand(2);
46789 return SDValue();
46794 return SDValue();
46800 return SDValue();
46814 return SDValue();
46820 return SDValue();
46836 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
46844 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
46849 return SDValue();
46857 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
46861 SDValue Cond = N->getOperand(0);
46865 return SDValue();
46882 return SDValue();
46887 return SDValue();
46890 return SDValue();
46893 return SDValue();
46896 return SDValue();
46901 return SDValue();
46903 auto OnlyUsedAsSelectCond = [](SDValue Cond) {
46920 return SDValue();
46931 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
46933 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
46937 return SDValue(N, 0);
46941 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
46945 return SDValue();
46964 static SDValue combineLogicBlendIntoConditionalNegate(
46965 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
46973 return SDValue();
46975 return SDValue();
46977 auto IsNegV = [](SDNode *N, SDValue V) {
46982 SDValue V;
46988 return SDValue();
46990 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
46991 SDValue SubOp2 = Mask;
47006 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
47010 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
47013 return SDValue();
47015 return SDValue();
47017 SDValue Cond = N->getOperand(0);
47018 SDValue LHS = N->getOperand(1);
47019 SDValue RHS = N->getOperand(2);
47022 return SDValue();
47025 return SDValue();
47028 return SDValue();
47041 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
47045 SDValue Cond = N->getOperand(0);
47046 SDValue LHS = N->getOperand(1);
47047 SDValue RHS = N->getOperand(2);
47051 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
47058 if (SDValue V = commuteSelect(N, DAG, DL, Subtarget))
47073 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
47093 SmallVector<SDValue, 1> LHSOps, RHSOps;
47132 SDValue Op0 = Cond.getOperand(IsStrict ? 1 : 0);
47133 SDValue Op1 = Cond.getOperand(IsStrict ? 2 : 1);
47272 SDValue Ret = DAG.getNode(Opcode == X86ISD::FMIN ? X86ISD::STRICT_FMIN
47291 SDValue AndNode = Cond.getOperand(0);
47323 auto SelectableOp = [&TLI](SDValue Op, SDValue Alt) {
47346 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
47351 if (SDValue V = combineSelectOfTwoConstants(N, DAG, DL))
47357 SDValue Cond0 = Cond.getOperand(0);
47358 SDValue Cond1 = Cond.getOperand(1);
47403 SDValue InnerSetCC = RHS.getOperand(0);
47435 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
47446 if (SDValue ExtCond = combineToExtendBoolVectorInReg(
47491 return SDValue();
47493 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget))
47496 if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
47499 if (SDValue V = narrowVectorSelect(N, DAG, DL, Subtarget))
47504 if (SDValue CondNot = IsNOT(Cond, DAG))
47559 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
47576 SDValue And = Cond.getOperand(0);
47580 SDValue NotCond =
47598 SDValue Mask = And.getOperand(1);
47606 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
47607 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
47608 SDValue NewCond =
47614 return SDValue();
47623 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
47629 return SDValue();
47635 return SDValue();
47649 SDValue CmpLHS = Cmp.getOperand(0);
47650 SDValue CmpRHS = Cmp.getOperand(1);
47654 return SDValue();
47658 return SDValue();
47660 SDValue OpRHS = CmpLHS.getOperand(2);
47663 return SDValue();
47671 return SDValue();
47721 return SDValue();
47732 return SDValue();
47734 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
47742 static SDValue checkSignTestSetCCCombine(SDValue Cmp, X86::CondCode &CC,
47745 return SDValue();
47748 return SDValue();
47750 SDValue Src;
47754 return SDValue();
47760 return SDValue();
47770 return SDValue();
47772 return SDValue();
47789 SDValue Mask = DAG.getNode(ISD::AND, DL, SrcVT, Src,
47811 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
47815 return SDValue();
47819 return SDValue();
47823 SDValue Op1 = Cmp.getOperand(0);
47824 SDValue Op2 = Cmp.getOperand(1);
47826 SDValue SetCC;
47836 return SDValue();
47843 return SDValue();
47887 return SDValue();
47890 SDValue Op = SetCC.getOperand(0);
47899 return SDValue();
47905 return SDValue();
47912 return SDValue();
47914 return SDValue();
47922 return SDValue();
47929 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
47930 X86::CondCode &CC1, SDValue &Flags,
47941 SDValue SetCC0, SetCC1;
47970 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
47974 SDValue Carry = EFLAGS.getOperand(0);
47986 SDValue CarryOp1 = Carry.getOperand(1);
48000 SDValue SubCommute =
48003 return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
48014 SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType());
48024 return SDValue();
48029 static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
48035 return SDValue();
48042 SDValue Op0 = EFLAGS.getOperand(0);
48043 SDValue Op1 = EFLAGS.getOperand(1);
48048 if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
48086 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
48098 SDValue BC0 = peekThroughBitcasts(Op0);
48103 SDValue X = DAG.getBitcast(OpVT, BC0.getOperand(0));
48111 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
48118 SDValue BC = peekThroughBitcasts(Op0);
48146 if (SDValue Res =
48184 SDValue Src0 = peekThroughBitcasts(Op0);
48185 SDValue Src1 = peekThroughBitcasts(Op1);
48201 return SDValue();
48205 static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
48211 return SDValue();
48213 return SDValue();
48216 return SDValue();
48219 return SDValue();
48222 SDValue CmpOp = EFLAGS.getOperand(0);
48232 return SDValue();
48234 SDValue Vec = CmpOp.getOperand(0);
48245 return SDValue();
48262 SDValue BC = peekThroughBitcasts(Vec);
48282 SmallVector<SDValue> Ops;
48288 SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT,
48304 SDValue BC = peekThroughBitcasts(Vec);
48308 SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(),
48317 SDValue LHS = BC.getOperand(0);
48318 SDValue RHS = BC.getOperand(1);
48325 SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS);
48337 SDValue VecOp0 = Vec.getOperand(0);
48338 SDValue VecOp1 = Vec.getOperand(1);
48344 SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
48358 if (SDValue Src = getSplitVectorSrc(VecOp0, VecOp1, true)) {
48360 SDValue Result = peekThroughBitcasts(Src);
48363 SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(),
48399 SmallVector<SDValue, 2> ShuffleInputs;
48407 SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
48426 SDValue LHS = Vec;
48427 SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT);
48434 return SDValue();
48440 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
48444 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
48447 if (SDValue R = checkSignTestSetCCCombine(EFLAGS, CC, DAG))
48450 if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
48453 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
48456 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
48463 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
48468 SDValue FalseOp = N->getOperand(0);
48469 SDValue TrueOp = N->getOperand(1);
48471 SDValue Cond = N->getOperand(3);
48479 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
48484 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
48527 SDValue(FalseC, 0));
48567 SDValue(FalseC, 0));
48604 SDValue Ops[] = {FalseOp, Cond.getOperand(0),
48620 SDValue Cond0 = Cond.getOperand(0);
48627 SDValue NewSub =
48630 SDValue EFLAGS(NewSub.getNode(), 1);
48654 SDValue Flags;
48664 SDValue LOps[] = {FalseOp, TrueOp,
48666 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, VT, LOps);
48667 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
48669 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
48680 SDValue Add = TrueOp;
48681 SDValue Const = FalseOp;
48698 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
48699 SDValue CMov =
48706 return SDValue();
48721 SDValue Opd = N->getOperand(i);
48774 static SDValue reduceVMULWidth(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
48779 return SDValue();
48787 return SDValue();
48791 return SDValue();
48793 SDValue N0 = N->getOperand(0);
48794 SDValue N1 = N->getOperand(1);
48798 return SDValue();
48803 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
48804 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
48808 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
48817 SDValue MulHi =
48829 SDValue ResLo =
48837 SDValue ResHi =
48843 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
48847 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48857 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48918 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48920 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48926 return SDValue();
48932 static SDValue combineMulToPMADDWD(SDNode *N, const SDLoc &DL,
48936 return SDValue();
48939 return SDValue();
48945 return SDValue();
48951 return SDValue();
48955 return SDValue();
48957 SDValue N0 = N->getOperand(0);
48958 SDValue N1 = N->getOperand(1);
48971 return SDValue();
48980 return SDValue();
48985 return SDValue();
48989 auto GetZeroableOp = [&](SDValue Op) {
48997 SDValue Src = Op.getOperand(0);
49012 SDValue Src = Op.getOperand(0);
49022 return SDValue();
49024 SDValue ZeroN0 = GetZeroableOp(N0);
49025 SDValue ZeroN1 = GetZeroableOp(N1);
49027 return SDValue();
49033 ArrayRef<SDValue> Ops) {
49043 static SDValue combineMulToPMULDQ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
49046 return SDValue();
49054 return SDValue();
49056 SDValue N0 = N->getOperand(0);
49057 SDValue N1 = N->getOperand(1);
49064 ArrayRef<SDValue> Ops) {
49075 ArrayRef<SDValue> Ops) {
49082 return SDValue();
49085 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
49091 if (SDValue V = combineMulToPMADDWD(N, DL, DAG, Subtarget))
49094 if (SDValue V = combineMulToPMULDQ(N, DL, DAG, Subtarget))
49102 return SDValue();
49106 return SDValue();
49116 return SDValue();
49121 return SDValue();
49125 return SDValue();
49128 return SDValue();
49134 SDValue NewMul = SDValue();
49243 SDValue Shift1 =
49246 SDValue Shift2 =
49264 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
49271 return SDValue();
49274 SDValue ShiftOperand = N->getOperand(0);
49276 return SDValue();
49281 return SDValue();
49287 return SDValue();
49289 SDValue LHS = ShiftOperand.getOperand(0);
49290 SDValue RHS = ShiftOperand.getOperand(1);
49295 return SDValue();
49304 return SDValue();
49307 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
49313 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG,
49316 SDValue N0 = N->getOperand(0);
49317 SDValue N1 = N->getOperand(1);
49327 SDValue Cond = N0.getOperand(0);
49328 SDValue N00 = N0.getOperand(1);
49329 SDValue N01 = N0.getOperand(2);
49349 SDValue N00 = N0.getOperand(0);
49378 return SDValue();
49381 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
49384 SDValue N0 = N->getOperand(0);
49385 SDValue N1 = N->getOperand(1);
49390 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
49395 SDValue ShrAmtVal;
49418 return SDValue();
49420 SDValue N00 = N0.getOperand(0);
49421 SDValue N01 = N0.getOperand(1);
49427 return SDValue();
49429 return SDValue();
49436 SDValue NN =
49446 return SDValue();
49449 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
49453 SDValue N0 = N->getOperand(0);
49454 SDValue N1 = N->getOperand(1);
49459 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
49466 SDValue Cond = N0.getOperand(0);
49467 SDValue N00 = N0.getOperand(1);
49468 SDValue N01 = N0.getOperand(2);
49486 return SDValue();
49493 return SDValue();
49498 return SDValue();
49509 return SDValue();
49518 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
49519 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
49522 return SDValue();
49525 static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
49532 SDValue N0 = N->getOperand(0);
49533 SDValue N1 = N->getOperand(1);
49536 SDValue BC0 =
49538 SDValue BC1 =
49547 if (SDValue BCSrc = getSplitVectorSrc(BC0, BC1, false)) {
49548 SmallVector<SDValue> ShuffleOps;
49550 SDValue Vec = peekThroughBitcasts(BCSrc);
49558 SDValue Lo, Hi;
49563 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
49576 SmallVector<SDValue> Ops0, Ops1;
49581 all_of(Ops0, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
49585 all_of(Ops1, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
49596 SDValue LHS, RHS;
49598 auto FindShuffleOpAndIdx = [&](int M, int &Idx, ArrayRef<SDValue> Ops) {
49602 SDValue Src = Ops[M / 2];
49621 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
49632 SmallVector<SDValue> Ops0, Ops1;
49638 [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
49640 [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
49643 SDValue Op00 = peekThroughBitcasts(Ops0.front());
49644 SDValue Op10 = peekThroughBitcasts(Ops1.front());
49645 SDValue Op01 = peekThroughBitcasts(Ops0.back());
49646 SDValue Op11 = peekThroughBitcasts(Ops1.back());
49657 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00),
49666 return SDValue();
49669 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
49677 SDValue N0 = N->getOperand(0);
49678 SDValue N1 = N->getOperand(1);
49743 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
49751 SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG);
49752 SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG);
49756 SDValue Pack =
49776 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
49785 SDValue Src0, Src1;
49813 SDValue Op(N, 0);
49814 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49817 return SDValue();
49820 static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
49829 SDValue LHS = N->getOperand(0);
49830 SDValue RHS = N->getOperand(1);
49837 SDValue LHS0 = LHS.getOperand(0);
49838 SDValue LHS1 = LHS.getOperand(1);
49839 SDValue RHS0 = RHS.getOperand(0);
49840 SDValue RHS1 = RHS.getOperand(1);
49844 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
49849 SDValue NewLHS =
49852 SDValue NewRHS =
49862 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
49865 return SDValue();
49868 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
49875 SDValue N0 = N->getOperand(0);
49876 SDValue N1 = N->getOperand(1);
49895 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
49896 return SDValue(N, 0);
49898 return SDValue();
49901 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
49910 SDValue N0 = N->getOperand(0);
49911 SDValue N1 = N->getOperand(1);
49946 auto MergeShifts = [&](SDValue X, uint64_t Amt0, uint64_t Amt1) {
49970 SDValue Op(N, 0);
49971 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49983 SDValue BC = peekThroughOneUseBitcasts(N0.getOperand(0));
49988 SDValue Src = BC.getOperand(0);
49998 auto TryConstantFold = [&](SDValue V) {
50004 return SDValue();
50028 if (SDValue C = TryConstantFold(N0))
50033 SDValue BC = peekThroughOneUseBitcasts(N0);
50037 if (SDValue RHS = TryConstantFold(BC.getOperand(1))) {
50039 SDValue LHS = DAG.getNode(Opcode, DL, VT,
50047 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBitsPerElt),
50049 return SDValue(N, 0);
50051 return SDValue();
50054 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
50064 SDValue Vec = N->getOperand(0);
50065 SDValue Scl = N->getOperand(1);
50066 SDValue Idx = N->getOperand(2);
50075 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
50077 return SDValue(N, 0);
50082 SDValue Op(N, 0);
50083 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
50087 return SDValue();
50093 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
50100 if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
50101 SDValue N0 = N->getOperand(0);
50102 SDValue N1 = N->getOperand(1);
50103 SDValue CMP0 = N0.getOperand(1);
50104 SDValue CMP1 = N1.getOperand(1);
50109 return SDValue();
50111 SDValue CMP00 = CMP0->getOperand(0);
50112 SDValue CMP01 = CMP0->getOperand(1);
50154 SDValue FSetCC =
50159 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
50165 SDValue OnesOrZeroesF =
50178 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
50180 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
50187 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
50188 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
50190 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
50197 return SDValue();
50201 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
50206 return SDValue();
50208 SDValue X, Y;
50209 SDValue N0 = N->getOperand(0);
50210 SDValue N1 = N->getOperand(1);
50212 if (SDValue Not = IsNOT(N0, DAG)) {
50215 } else if (SDValue Not = IsNOT(N1, DAG)) {
50219 return SDValue();
50232 static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
50241 return SDValue();
50243 auto GetNot = [&DAG](SDValue V) {
50250 return SDValue();
50252 SDValue IVEN = SVN->getOperand(0);
50255 return SDValue();
50258 return SDValue();
50259 SDValue Src = IVEN.getOperand(1);
50260 if (SDValue Not = IsNOT(Src, DAG)) {
50261 SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
50262 SDValue NotIVEN =
50268 return SDValue();
50271 SDValue X, Y;
50272 SDValue N0 = N->getOperand(0);
50273 SDValue N1 = N->getOperand(1);
50276 if (SDValue Not = GetNot(N0)) {
50279 } else if (SDValue Not = GetNot(N1)) {
50283 return SDValue();
50293 SDValue LoX, HiX;
50295 SDValue LoY, HiY;
50298 SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
50299 SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
50306 return SDValue();
50318 static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
50322 return SDValue();
50325 return SDValue();
50327 SDValue N0 = N.getOperand(0);
50328 SDValue N1 = N.getOperand(1);
50332 return SDValue();
50334 if (SDValue NN0 = PromoteMaskArithmetic(N0, DL, VT, DAG, Depth + 1))
50339 return SDValue();
50343 return SDValue();
50348 if (SDValue NN1 = PromoteMaskArithmetic(N1, DL, VT, DAG, Depth + 1))
50356 else if (SDValue Cst =
50360 return SDValue();
50372 static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
50381 SDValue Narrow = N.getOperand(0);
50385 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, 0);
50387 return SDValue();
50416 static SDValue convertIntLogicToFPLogic(unsigned Opc, const SDLoc &DL, EVT VT,
50417 SDValue N0, SDValue N1,
50426 return SDValue();
50428 SDValue N00 = N0.getOperand(0);
50429 SDValue N10 = N1.getOperand(0);
50437 return SDValue();
50441 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
50447 return SDValue();
50456 return SDValue();
50465 SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL);
50466 SDValue N01 = N0.getOperand(1);
50467 SDValue N11 = N1.getOperand(1);
50468 SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00);
50469 SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01);
50470 SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10);
50471 SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11);
50472 SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0);
50473 SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1);
50474 SDValue Logic = DAG.getNode(Opc, DL, BoolVecVT, Setcc0, Setcc1);
50480 static SDValue combineBitOpWithMOVMSK(unsigned Opc, const SDLoc &DL, SDValue N0,
50481 SDValue N1, SelectionDAG &DAG) {
50488 return SDValue();
50490 SDValue Vec0 = N0.getOperand(0);
50491 SDValue Vec1 = N1.getOperand(0);
50499 return SDValue();
50503 SDValue Result =
50511 static SDValue combineBitOpWithShift(unsigned Opc, const SDLoc &DL, EVT VT,
50512 SDValue N0, SDValue N1,
50519 return SDValue();
50522 SDValue BC0 = peekThroughOneUseBitcasts(N0);
50523 SDValue BC1 = peekThroughOneUseBitcasts(N1);
50528 return SDValue();
50535 return SDValue();
50536 SDValue BitOp =
50538 SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1));
50543 return SDValue();
50549 static SDValue combineBitOpWithPACK(unsigned Opc, const SDLoc &DL, EVT VT,
50550 SDValue N0, SDValue N1, SelectionDAG &DAG) {
50556 return SDValue();
50563 return SDValue();
50567 return SDValue();
50577 return SDValue();
50579 SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0));
50580 SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1));
50587 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
50589 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
50590 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
50593 return SDValue();
50606 SDValue X, Y;
50618 SDValue Sra =
50627 return SDValue();
50631 return SDValue();
50634 return SDValue();
50638 return SDValue();
50642 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
50643 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
50649 static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
50651 return SDValue();
50653 SDValue Base = Ld->getBasePtr();
50655 return SDValue();
50657 SDValue ShiftedIndex = Base.getOperand(0);
50659 return SDValue();
50671 static SDValue combineAndNotOrIntoAndNotAnd(SDNode *N, SelectionDAG &DAG) {
50677 if (!TLI.hasAndNot(SDValue(N, 0)))
50678 return SDValue();
50680 SDValue X, Y, Z;
50692 return SDValue();
50705 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
50712 return SDValue();
50724 SDValue Index = getIndexFromUnindexedLoad(Ld);
50758 SDValue Inp = Node->getOperand(i == 0 ? 1 : 0);
50759 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
50762 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
50765 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
50766 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
50772 return SDValue();
50780 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
50790 return SDValue();
50795 SDValue Src = N->getOperand(0);
50797 return SDValue();
50802 return SDValue();
50807 return SDValue();
50815 return SDValue();
50818 return SDValue();
50822 SDValue SubVec = Src.getOperand(0);
50828 return SDValue();
50832 auto IsLegalSetCC = [&](SDValue V) {
50846 return SDValue();
50851 SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
50854 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
50860 static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG,
50861 SDValue OpMustEq, SDValue Op, unsigned Depth) {
50868 return SDValue();
50875 return SDValue();
50878 if (SDValue R =
50902 return SDValue();
50905 static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG,
50911 return SDValue();
50917 if (SDValue OpMatch =
50921 return SDValue();
50924 static SDValue combineX86SubCmpForFlags(SDNode *N, SDValue Flag,
50942 SDValue SetCC = N->getOperand(0);
50945 return SDValue();
50950 return SDValue();
50954 return SDValue();
50956 SDValue X = SetCC.getOperand(1);
50962 SDValue CCN = SetCC.getOperand(0);
50970 SmallVector<SDValue> Ops(BrCond->op_values());
50978 SDValue NewBrCond =
50986 static SDValue combineAndOrForCcmpCtest(SDNode *N, SelectionDAG &DAG,
51000 return SDValue();
51002 SDValue SetCC0 = N->getOperand(0);
51003 SDValue SetCC1 = N->getOperand(1);
51006 return SDValue();
51008 auto GetCombineToOpc = [&](SDValue V) -> unsigned {
51009 SDValue Op = V.getOperand(1);
51025 return SDValue();
51032 return SDValue();
51039 SDValue SrcCC =
51043 SDValue CC1N = SetCC1.getOperand(0);
51049 SDValue CFlags = DAG.getTargetConstant(
51051 SDValue Sub = SetCC1.getOperand(1);
51055 SDValue CCMP = (NewOpc == X86ISD::CCMP)
51066 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
51069 SDValue N0 = N->getOperand(0);
51070 SDValue N1 = N->getOperand(1);
51088 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
51089 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
51098 SmallVector<SDValue, 2> SrcOps;
51100 if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
51104 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
51110 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
51138 SDValue Neg = DAG.getNegative(N0.getOperand(0), dl, VT);
51142 SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg,
51149 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
51152 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
51155 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), dl, N0, N1, DAG))
51158 if (SDValue R = combineBitOpWithShift(N->getOpcode(), dl, VT, N0, N1, DAG))
51161 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), dl, VT, N0, N1, DAG))
51164 if (SDValue FPLogic = convertIntLogicToFPLogic(N->getOpcode(), dl, VT, N0, N1,
51168 if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
51172 return SDValue();
51174 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
51177 if (SDValue R = combineAndNotIntoANDNP(N, DAG))
51180 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
51183 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
51186 if (SDValue R = combineAndNotOrIntoAndNotAnd(N, DAG))
51198 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1);
51206 SDValue Src = N0;
51221 SDValue BitNo = Src.getOperand(1);
51231 if (SDValue BT = getBT(Src, BitNo, dl, DAG))
51238 SDValue Op(N, 0);
51239 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
51244 auto GetDemandedMasks = [&](SDValue Op) {
51280 return SDValue(N, 0);
51283 SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG);
51284 SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG);
51294 SDValue BitMask = N1;
51295 SDValue SrcVec = N0.getOperand(0);
51319 if (SDValue Shuffle = combineX86ShufflesRecursively(
51329 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
51332 return SDValue();
51336 static SDValue canonicalizeBitSelect(SDNode *N, const SDLoc &DL,
51344 return SDValue();
51346 SDValue N0 = peekThroughBitcasts(N->getOperand(0));
51347 SDValue N1 = peekThroughBitcasts(N->getOperand(1));
51349 return SDValue();
51355 return SDValue();
51363 return SDValue();
51367 return SDValue();
51372 return SDValue();
51374 return SDValue();
51383 SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1));
51384 SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0));
51385 SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0));
51386 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
51387 SDValue Res = getAVX512Node(X86ISD::VPTERNLOG, DL, OpVT, {A, B, C, Imm},
51392 SDValue X = N->getOperand(0);
51393 SDValue Y =
51400 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
51404 SDValue N0 = N->getOperand(0);
51405 SDValue N1 = N->getOperand(1);
51439 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, const SDLoc &DL,
51447 return SDValue();
51449 SDValue X, Y, Mask;
51451 return SDValue();
51463 return SDValue();
51466 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
51472 return SDValue();
51476 return SDValue();
51493 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
51494 SDValue Cmp = Op.getOperand(1);
51498 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
51501 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
51502 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
51514 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
51518 return SDValue();
51520 auto isORCandidate = [](SDValue N) {
51529 return SDValue();
51532 auto isSetCCCandidate = [](SDValue N) {
51541 SDValue LHS = OR->getOperand(0);
51542 SDValue RHS = OR->getOperand(1);
51556 !isORCandidate(SDValue(OR, 0)))
51557 return SDValue();
51564 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
51565 SDValue Ret, NewRHS;
51570 return SDValue();
51582 return SDValue();
51589 static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
51590 SDValue And1_L, SDValue And1_R,
51593 return SDValue();
51594 SDValue NotOp = And0_L->getOperand(0);
51598 return SDValue();
51603 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
51604 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
51605 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
51606 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
51615 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
51619 SDValue N0 = Node->getOperand(0);
51621 return SDValue();
51622 SDValue N1 = Node->getOperand(1);
51624 return SDValue();
51627 SDValue N00 = N0->getOperand(0);
51628 SDValue N01 = N0->getOperand(1);
51629 SDValue N10 = N1->getOperand(0);
51630 SDValue N11 = N1->getOperand(1);
51631 if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
51633 if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
51635 if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
51637 if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
51639 return SDValue();
51646 static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
51647 SDValue X, SDValue Y,
51651 return SDValue();
51658 SDValue EFLAGS;
51668 return SDValue();
51692 SDValue NewSub = DAG.getNode(
51695 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
51712 return SDValue();
51724 SDValue NewSub =
51727 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
51754 SDValue NewSub =
51757 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
51765 return SDValue();
51770 return SDValue();
51772 SDValue Z = EFLAGS.getOperand(0);
51784 SDValue Zero = DAG.getConstant(0, DL, ZVT);
51786 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
51789 SDValue(Neg.getNode(), 1));
51798 SDValue One = DAG.getConstant(1, DL, ZVT);
51800 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
51808 SDValue One = DAG.getConstant(1, DL, ZVT);
51810 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
51830 static SDValue combineAddOrSubToADCOrSBB(SDNode *N, const SDLoc &DL,
51833 SDValue X = N->getOperand(0);
51834 SDValue Y = N->getOperand(1);
51837 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG))
51841 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) {
51847 return SDValue();
51850 static SDValue combineOrXorWithSETCC(unsigned Opc, const SDLoc &DL, EVT VT,
51851 SDValue N0, SDValue N1,
51867 if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG))
51883 return SDValue();
51886 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
51889 SDValue N0 = N->getOperand(0);
51890 SDValue N1 = N->getOperand(1);
51906 SmallVector<SDValue, 2> SrcOps;
51908 if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
51912 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
51918 SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
51919 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
51926 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
51929 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), dl, N0, N1, DAG))
51932 if (SDValue R = combineBitOpWithShift(N->getOpcode(), dl, VT, N0, N1, DAG))
51935 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), dl, VT, N0, N1, DAG))
51938 if (SDValue FPLogic = convertIntLogicToFPLogic(N->getOpcode(), dl, VT, N0, N1,
51943 return SDValue();
51945 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
51948 if (SDValue R = canonicalizeBitSelect(N, dl, DAG, Subtarget))
51951 if (SDValue R = combineLogicBlendIntoPBLENDV(N, dl, DAG, Subtarget))
51958 SDValue Cond = N0.getOperand(1);
51968 SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG);
51970 SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
52007 SDValue Op(N, 0);
52008 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
52013 auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
52031 return SDValue(N, 0);
52037 if (SDValue R = foldMaskedMerge(N, DAG))
52040 if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), dl, VT, N0, N1, DAG))
52043 return SDValue();
52050 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
52054 return SDValue();
52056 SDValue N0 = N->getOperand(0);
52057 SDValue N1 = N->getOperand(1);
52061 return SDValue();
52065 return SDValue();
52068 SDValue Shift = N0.getOperand(0);
52070 return SDValue();
52075 return SDValue();
52080 return SDValue();
52086 SDValue ShiftOp = Shift.getOperand(0);
52091 SDValue Cond =
52106 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
52110 return SDValue();
52114 default: return SDValue();
52118 case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
52122 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
52128 SDValue Shift = N->getOperand(0);
52129 SDValue Ones = N->getOperand(1);
52132 return SDValue();
52139 return SDValue();
52149 /// Return the source value x to be truncated or SDValue() if the pattern was
52160 /// So return the smax(x, C1) value to be truncated or SDValue() if the
52162 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
52172 SDValue UMin, SMin, SMax;
52190 return SDValue();
52200 /// Return the source value to be truncated or SDValue() if the pattern was not
52202 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
52217 SDValue SMin, SMax;
52226 return SDValue();
52229 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
52233 return SDValue();
52245 if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
52247 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
52270 if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
52274 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
52277 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
52285 if (SDValue SSatVal = detectSSatPattern(In, VT))
52295 SDValue SatVal;
52296 if (SDValue SSatVal = detectSSatPattern(In, VT)) {
52299 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) {
52310 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
52320 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
52326 return SDValue();
52329 static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl,
52335 SDValue Ptr = Ld->getBasePtr();
52336 SDValue Chain = Ld->getChain();
52340 return SDValue();
52343 return SDValue();
52347 return SDValue();
52372 SDValue UserPtr = UserLd->getBasePtr();
52385 if (getTargetConstantBitsFromNode(SDValue(N, 0), NumBits, Undefs,
52387 getTargetConstantBitsFromNode(SDValue(User, 0), NumBits,
52390 SDValue Extract = extractSubVector(
52391 SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
52393 return DCI.CombineTo(N, Extract, SDValue(User, 1));
52401 return SDValue();
52404 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
52427 return SDValue();
52430 SDValue Ptr1 = Ld->getBasePtr();
52431 SDValue Ptr2 =
52435 SDValue Load1 =
52439 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
52443 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
52446 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
52457 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
52461 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
52470 SDValue Ptr = Ld->getBasePtr();
52471 SDValue Chain = Ld->getChain();
52481 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl,
52484 return DCI.CombineTo(N, Extract, SDValue(User, 1));
52489 if (SDValue V = combineConstantPoolLoads(Ld, dl, DAG, DCI, Subtarget))
52498 SDValue Cast =
52506 return SDValue();
52512 static int getOneTrueElt(SDValue V) {
52527 const SDValue &Op = BV->getOperand(i);
52548 SelectionDAG &DAG, SDValue &Addr,
52549 SDValue &Index, Align &Alignment,
52576 static SDValue
52585 SDValue Addr, VecIndex;
52589 return SDValue();
52603 SDValue Load =
52608 SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
52611 SDValue Insert =
52617 static SDValue
52622 return SDValue();
52635 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
52637 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
52649 return SDValue();
52652 return SDValue();
52656 SDValue NewML = DAG.getMaskedLoad(
52660 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
52666 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
52673 return SDValue();
52676 if (SDValue ScalarLoad =
52682 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
52688 SDValue Mask = Mld->getMask();
52696 return SDValue(N, 0);
52698 if (SDValue NewMask =
52706 return SDValue();
52713 static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
52720 SDValue Addr, VecIndex;
52724 return SDValue();
52728 SDValue Value = MS->getValue();
52736 SDValue Extract =
52745 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
52750 return SDValue();
52757 return SDValue();
52759 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget))
52764 SDValue Mask = Mst->getMask();
52770 return SDValue(N, 0);
52772 if (SDValue NewMask =
52780 SDValue Value = Mst->getValue();
52790 return SDValue();
52793 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
52799 SDValue StoredVal = St->getValue();
52820 SDValue Val = StoredVal.getOperand(0);
52834 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
52848 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
52851 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
52855 SDValue Ptr0 = St->getBasePtr();
52856 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl);
52858 SDValue Ch0 =
52862 SDValue Ch1 =
52890 SDValue LogicOp = DAG.getNode(
52908 return SDValue();
52921 return SDValue();
52942 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,
52962 auto IsExtractedElement = [](SDValue V) {
52970 return SDValue();
52972 if (SDValue Extract = IsExtractedElement(StoredVal)) {
52973 SDValue Trunc = peekThroughOneUseBitcasts(Extract);
52975 SDValue Src = Trunc.getOperand(0);
52995 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
52999 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
53006 return SDValue();
53015 SDValue Cast =
53028 SDValue Cmov;
53035 return SDValue();
53039 return SDValue();
53042 SDValue V = SDValue(Ld, 0);
53046 return SDValue();
53049 SDValue CC = Cmov.getOperand(2);
53050 SDValue Src = DAG.getAnyExtOrTrunc(Cmov.getOperand(!InvertCC), dl, VT);
53056 SDValue Ops[] = {St->getChain(), Src, St->getBasePtr(), CC,
53069 return SDValue();
53077 return SDValue();
53085 return SDValue();
53089 return SDValue();
53094 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
53111 SDValue OldExtract = St->getOperand(1);
53112 SDValue ExtOp0 = OldExtract.getOperand(0);
53115 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
53116 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
53123 return SDValue();
53126 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
53131 SDValue StoredVal = N->getOperand(1);
53143 return SDValue(N, 0);
53146 return SDValue();
53162 static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS,
53185 auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
53194 SmallVector<SDValue, 2> SrcOps;
53196 SDValue BC = peekThroughBitcasts(Op);
53198 !isAnyZero(SrcMask) && all_of(SrcOps, [BC](SDValue Op) {
53204 N0 = !SrcOps.empty() ? SrcOps[0] : SDValue();
53205 N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
53221 // NOTE: A default initialized SDValue represents an UNDEF of type VT.
53222 SDValue A, B;
53228 SDValue C, D;
53251 B = SDValue();
53253 A = SDValue();
53256 D = SDValue();
53258 C = SDValue();
53314 SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
53315 SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
53350 static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
53369 SDValue LHS = N->getOperand(0);
53370 SDValue RHS = N->getOperand(1);
53374 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
53386 SDValue LHS = N->getOperand(0);
53387 SDValue RHS = N->getOperand(1);
53392 ArrayRef<SDValue> Ops) {
53395 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
53406 return SDValue();
53422 static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
53425 SDValue LHS = N->getOperand(0);
53426 SDValue RHS = N->getOperand(1);
53429 auto combineConjugation = [&](SDValue &r) {
53431 SDValue XOR = LHS.getOperand(0);
53442 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0));
53443 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F);
53452 SDValue Res;
53463 static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
53474 auto IsVectorAllNegativeZero = [&DAG](SDValue Op) {
53483 return SDValue();
53487 return SDValue();
53489 SDValue LHS = N->getOperand(0);
53490 SDValue RHS = N->getOperand(1);
53492 SDValue FAddOp1, MulOp0, MulOp1;
53495 &HasNoSignedZero](SDValue N) -> bool {
53498 SDValue Op0 = N.getOperand(0);
53525 return SDValue();
53532 SDValue CFmul =
53538 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
53540 if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget))
53543 if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
53546 return SDValue();
53549 static SDValue combineLRINT_LLRINT(SDNode *N, SelectionDAG &DAG,
53552 SDValue Src = N->getOperand(0);
53558 return SDValue();
53570 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
53574 SDValue Src = N->getOperand(0);
53581 auto IsFreeTruncation = [VT](SDValue Op) {
53600 auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
53601 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
53602 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
53608 return SDValue();
53613 return SDValue();
53632 SDValue Op0 = Src.getOperand(0);
53633 SDValue Op1 = Src.getOperand(1);
53641 return SDValue();
53651 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
53656 return SDValue();
53661 return SDValue();
53666 return SDValue();
53669 SDValue LHS, RHS;
53672 return SDValue();
53678 auto IsSext = [&DAG](SDValue V) {
53681 auto IsZext = [&DAG](SDValue V) {
53688 return SDValue();
53691 auto isOpTruncateFree = [](SDValue Op) {
53709 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS),
53730 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
53734 return SDValue();
53739 return SDValue();
53741 SDValue SSatVal = detectSSatPattern(In, VT);
53743 return SDValue();
53747 SDValue N0 = SSatVal.getOperand(0);
53748 SDValue N1 = SSatVal.getOperand(1);
53751 return SDValue();
53753 SDValue N00 = N0.getOperand(0);
53754 SDValue N01 = N0.getOperand(1);
53755 SDValue N10 = N1.getOperand(0);
53756 SDValue N11 = N1.getOperand(1);
53770 return SDValue();
53783 return SDValue();
53790 return SDValue();
53800 SDValue ZExtIn, SExtIn;
53802 SDValue N00Elt = N00.getOperand(i);
53803 SDValue N01Elt = N01.getOperand(i);
53804 SDValue N10Elt = N10.getOperand(i);
53805 SDValue N11Elt = N11.getOperand(i);
53811 return SDValue();
53817 return SDValue();
53830 return SDValue();
53831 SDValue N00In = N00Elt.getOperand(0);
53832 SDValue N01In = N01Elt.getOperand(0);
53833 SDValue N10In = N10Elt.getOperand(0);
53834 SDValue N11In = N11Elt.getOperand(0);
53842 return SDValue();
53845 auto ExtractVec = [&DAG, &DL, NumElems](SDValue &Ext) {
53857 ArrayRef<SDValue> Ops) {
53872 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
53875 SDValue Src = N->getOperand(0);
53879 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
53883 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
53887 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
53891 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
53897 SDValue BCSrc = Src.getOperand(0);
53907 return SDValue();
53910 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
53913 SDValue In = N->getOperand(0);
53916 if (SDValue SSatVal = detectSSatPattern(In, VT))
53918 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL))
53923 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53924 return SDValue(N, 0);
53926 return SDValue();
53938 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
53944 return SDValue();
53948 SDValue Op = peekThroughBitcasts(SDValue(N, 0));
53953 return SDValue();
53961 return SDValue();
53962 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
53971 SDValue InsVector = Op.getOperand(0);
53972 SDValue InsVal = Op.getOperand(1);
53974 return SDValue();
53975 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
53984 SDValue Op1 = Op.getOperand(1);
53985 SDValue Op0 = Op.getOperand(0);
54003 return SDValue();
54014 return SDValue();
54084 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
54088 SDValue Arg = isFNEG(DAG, N);
54090 return SDValue();
54099 return SDValue();
54106 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
54107 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
54114 if (SDValue NegArg =
54118 return SDValue();
54121 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
54127 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
54157 SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
54177 if (SDValue NegOp0 =
54188 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
54193 return SDValue();
54201 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
54202 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
54213 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
54219 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
54221 return SDValue();
54223 SDValue LHS = N->getOperand(0);
54225 return SDValue();
54233 static SDValue combineXorSubCTLZ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
54238 return SDValue();
54243 return SDValue();
54245 SDValue N0 = N->getOperand(0);
54246 SDValue N1 = N->getOperand(1);
54250 return SDValue();
54252 SDValue OpCTLZ;
54253 SDValue OpSizeTM1;
54259 return SDValue();
54266 return SDValue();
54269 return SDValue();
54272 return SDValue();
54274 SDValue Op = OpCTLZ.getOperand(0);
54289 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
54292 SDValue N0 = N->getOperand(0);
54293 SDValue N1 = N->getOperand(1);
54305 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
54308 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), DL, N0, N1, DAG))
54311 if (SDValue R = combineBitOpWithShift(N->getOpcode(), DL, VT, N0, N1, DAG))
54314 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), DL, VT, N0, N1, DAG))
54317 if (SDValue FPLogic = convertIntLogicToFPLogic(N->getOpcode(), DL, VT, N0, N1,
54321 if (SDValue R = combineXorSubCTLZ(N, DL, DAG, Subtarget))
54325 return SDValue();
54327 if (SDValue SetCC = foldXor1SetCC(N, DAG))
54330 if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), DL, VT, N0, N1, DAG))
54333 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
54363 SDValue TruncExtSrc = N0.getOperand(0);
54367 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT);
54368 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT);
54374 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
54380 static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG,
54383 SDValue N0 = N->getOperand(0);
54388 SDValue Src = N0.getOperand(0);
54398 SDValue Rev =
54404 return SDValue();
54408 static SDValue combineAVG(SDNode *N, SelectionDAG &DAG,
54412 SDValue N0 = N->getOperand(0);
54413 SDValue N1 = N->getOperand(1);
54422 SDValue SignMask = DAG.getConstant(SignBit, DL, VT);
54429 return SDValue();
54432 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
54443 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
54444 return SDValue(N, 0);
54446 return SDValue();
54449 static bool isNullFPScalarOrVectorConst(SDValue V) {
54459 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
54462 return SDValue();
54470 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
54472 SDValue N0 = N->getOperand(0);
54473 SDValue N1 = N->getOperand(1);
54481 return SDValue();
54483 auto isAllOnesConstantFP = [](SDValue V) {
54498 return SDValue();
54502 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
54505 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
54509 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
54512 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
54519 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
54526 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
54533 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
54546 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
54553 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
54559 return SDValue();
54574 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
54578 return SDValue();
54586 return SDValue();
54588 SDValue Op0 = N->getOperand(0);
54589 SDValue Op1 = N->getOperand(1);
54608 return SDValue();
54632 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
54633 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
54640 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
54646 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
54647 return SDValue(N, 0);
54650 SDValue In = N->getOperand(0);
54659 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
54661 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
54664 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54666 return SDValue(N, 0);
54670 return SDValue();
54673 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
54680 SDValue In = N->getOperand(IsStrict ? 1 : 0);
54689 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
54692 SDValue Convert =
54697 SDValue Convert =
54701 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54703 return SDValue(N, 0);
54707 return SDValue();
54711 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
54714 SDValue N0 = N->getOperand(0);
54715 SDValue N1 = N->getOperand(1);
54740 if (SDValue Not = IsNOT(N0, DAG))
54747 SDValue Src = N0.getOperand(0);
54774 SDValue BC0 = peekThroughOneUseBitcasts(N0);
54778 SDValue Not = getConstVector(EltBits0, VT, DAG, DL);
54786 SDValue Op(N, 0);
54787 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
54792 auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
54827 return SDValue(N, 0);
54834 if (SDValue Not = IsNOT(N1, DAG))
54841 SDValue BC1 = peekThroughOneUseBitcasts(N1);
54844 SDValue NewMask = DAG.getNode(ISD::OR, DL, ShufVT, BC1.getOperand(1),
54846 SDValue NewShuf =
54853 return SDValue();
54856 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
54858 SDValue N1 = N->getOperand(1);
54866 return SDValue(N, 0);
54869 return SDValue();
54872 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
54875 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
54883 return SDValue(N, 0);
54889 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
54892 SDValue Convert = DAG.getNode(
54897 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
54902 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54904 return SDValue(N, 0);
54909 return SDValue();
54913 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
54918 SDValue N0 = N->getOperand(0);
54919 SDValue N1 = N->getOperand(1);
54923 return SDValue();
54926 SDValue IntermediateBitwidthOp;
54935 return SDValue();
54937 SDValue CMovOp0 = N0.getOperand(0);
54938 SDValue CMovOp1 = N0.getOperand(1);
54943 return SDValue();
54965 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
54974 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
54978 if (SDValue V = combineSextInRegCmov(N, DAG))
54982 SDValue N0 = N->getOperand(0);
54983 SDValue N1 = N->getOperand(1);
54994 SDValue N00 = N0.getOperand(0);
55000 return SDValue();
55004 if (SDValue Promote = PromoteMaskArithmetic(N0, dl, DAG, Subtarget))
55008 SDValue Tmp =
55013 return SDValue();
55021 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
55025 return SDValue();
55030 return SDValue();
55032 SDValue Add = Ext->getOperand(0);
55034 return SDValue();
55036 SDValue AddOp0 = Add.getOperand(0);
55037 SDValue AddOp1 = Add.getOperand(1);
55047 return SDValue();
55054 return SDValue();
55069 return SDValue();
55073 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
55074 SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT);
55096 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
55097 SDValue CMovN = Extend->getOperand(0);
55099 return SDValue();
55106 SDValue CMovOp0 = CMovN.getOperand(0);
55107 SDValue CMovOp1 = CMovN.getOperand(1);
55111 return SDValue();
55115 return SDValue();
55120 return SDValue();
55131 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
55143 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
55145 SDValue N0 = N->getOperand(0);
55151 return SDValue();
55157 return SDValue();
55161 return SDValue();
55165 return SDValue();
55171 return SDValue();
55177 return SDValue();
55179 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
55187 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
55190 SDValue N0 = N->getOperand(0);
55197 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
55203 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
55208 return SDValue(N, 0);
55211 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
55215 return SDValue();
55217 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
55220 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0,
55225 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), DL, DAG, Subtarget))
55232 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
55235 return SDValue();
55244 static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) {
55255 return SDValue();
55257 SmallVector<SDValue, 8> Ops;
55260 for (const SDValue &Op : V->op_values()) {
55271 return SDValue();
55276 return SDValue(NV, 0);
55282 for (const SDValue &Op : V->op_values()) {
55285 return SDValue();
55289 return SDValue(NV, 0);
55292 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
55305 return SDValue();
55307 SDValue A = N->getOperand(IsStrict ? 1 : 0);
55308 SDValue B = N->getOperand(IsStrict ? 2 : 1);
55309 SDValue C = N->getOperand(IsStrict ? 3 : 2);
55316 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags);
55325 return SDValue();
55327 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
55330 if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
55339 SDValue Vec = V.getOperand(0);
55340 if (SDValue NegV = TLI.getCheaperNegatedExpression(
55349 if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) {
55364 return SDValue();
55384 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
55392 SDValue N2 = N->getOperand(2);
55394 SDValue NegN2 =
55397 return SDValue();
55407 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
55411 SDValue N0 = N->getOperand(0);
55418 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
55424 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
55429 return SDValue(N, 0);
55432 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
55436 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
55439 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0,
55444 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), dl, DAG, Subtarget))
55447 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
55450 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
55456 SDValue N00 = N0.getOperand(0);
55457 SDValue N01 = N0.getOperand(1);
55466 return SDValue();
55472 static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS,
55473 SDValue RHS, ISD::CondCode CC,
55480 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
55483 return SDValue();
55486 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
55490 const SDValue LHS = N->getOperand(0);
55491 const SDValue RHS = N->getOperand(1);
55497 if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
55503 if (SDValue V =
55511 auto MatchOrCmpEq = [&](SDValue N0, SDValue N1) {
55520 return SDValue();
55522 if (SDValue AndN = MatchOrCmpEq(LHS, RHS))
55524 if (SDValue AndN = MatchOrCmpEq(RHS, LHS))
55529 auto MatchAndCmpEq = [&](SDValue N0, SDValue N1) {
55538 return SDValue();
55540 if (SDValue AndN = MatchAndCmpEq(LHS, RHS))
55542 if (SDValue AndN = MatchAndCmpEq(RHS, LHS))
55574 SDValue BaseOp = LHS.getOperand(0);
55575 SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC);
55576 SDValue SETCC1 = DAG.getSetCC(
55590 SDValue Op0 = LHS;
55591 SDValue Op1 = RHS;
55641 SDValue LHSOut = LHS;
55642 SDValue RHSOut = RHS;
55647 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
55650 else if (SDValue NewRHS = incDecVectorConstant(
55663 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
55666 else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
55681 if (SDValue R = truncateAVX512SetCCNoBWI(VT, OpVT, LHSOut, RHSOut,
55689 if (SDValue R =
55715 SDValue AddC = LHS.getOperand(1);
55719 SDValue C0 = SDValue();
55720 SDValue C1 = SDValue();
55747 SDValue NewLHS =
55749 SDValue NewRHS =
55760 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
55769 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC);
55772 return SDValue();
55775 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
55778 SDValue Src = N->getOperand(0);
55808 if (SDValue NotSrc = IsNOT(Src, DAG)) {
55842 SDValue ShiftLHS = Src.getOperand(0);
55843 SDValue ShiftRHS = Src.getOperand(1);
55856 SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS);
55863 SDValue SrcBC = peekThroughOneUseBitcasts(Src);
55875 SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0));
55876 SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc);
55886 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
55887 return SDValue(N, 0);
55889 return SDValue();
55892 static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG,
55901 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
55902 return SDValue(N, 0);
55904 return SDValue();
55907 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
55910 SDValue Mask = MemOp->getMask();
55919 return SDValue(N, 0);
55923 return SDValue();
55926 static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
55927 SDValue Index, SDValue Base, SDValue Scale,
55932 SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(),
55941 SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
55950 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
55954 SDValue Index = GorS->getIndex();
55955 SDValue Base = GorS->getBasePtr();
55956 SDValue Scale = GorS->getScale();
56020 SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base);
56046 SDValue Mask = GorS->getMask();
56052 return SDValue(N, 0);
56056 return SDValue();
56060 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
56064 SDValue EFLAGS = N->getOperand(1);
56067 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
56070 return SDValue();
56074 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
56077 SDValue EFLAGS = N->getOperand(3);
56083 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
56084 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
56089 return SDValue();
56093 static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
56109 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
56113 return SDValue();
56122 return SDValue();
56129 SDValue SourceConst;
56132 {N->getOperand(0), SDValue(BV, 0)});
56134 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
56136 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
56137 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
56139 SDValue Res = DAG.getBitcast(VT, NewAnd);
56145 return SDValue();
56151 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
56155 SDValue Trunc = N->getOperand(0);
56157 return SDValue();
56159 SDValue ExtElt = Trunc.getOperand(0);
56162 return SDValue();
56169 return SDValue();
56176 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
56178 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
56183 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
56186 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
56202 return SDValue();
56210 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
56224 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
56244 return SDValue();
56247 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
56253 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
56257 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
56273 return SDValue();
56281 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
56295 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
56314 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
56323 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
56324 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
56341 return SDValue();
56346 return SDValue();
56350 std::pair<SDValue, SDValue> Tmp =
56360 return SDValue();
56362 if (SDValue V = combineToFPTruncExtElt(N, DAG))
56365 return SDValue();
56369 static SDValue combineFP_TO_xINT_SAT(SDNode *N, SelectionDAG &DAG,
56372 return SDValue();
56380 SDValue V2F32Value = DAG.getUNDEF(SrcVT);
56383 SDValue NewSrc = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
56392 return SDValue();
56395 static bool needCarryOrOverflowFlag(SDValue Flags) {
56430 static bool onlyZeroFlagUsed(SDValue Flags) {
56457 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
56462 return SDValue();
56469 SDValue Op = N->getOperand(0);
56473 if (SDValue CMP =
56474 combineX86SubCmpForFlags(N, SDValue(N, 0), DAG, DCI, Subtarget))
56482 onlyZeroFlagUsed(SDValue(N, 0))) {
56503 Op.hasOneUse() && onlyZeroFlagUsed(SDValue(N, 0))) {
56504 SDValue Src = Op.getOperand(0);
56508 SDValue BoolVec = Src.getOperand(0);
56530 if (Op.getOpcode() == ISD::ZERO_EXTEND && onlyZeroFlagUsed(SDValue(N, 0))) {
56531 SDValue Src = Op.getOperand(0);
56540 return SDValue();
56542 SDValue Trunc = Op;
56552 onlyZeroFlagUsed(SDValue(N, 0))) {
56559 return SDValue();
56563 default: return SDValue();
56568 return SDValue();
56575 if (needCarryOrOverflowFlag(SDValue(N, 0)))
56576 return SDValue();
56581 if (needCarryOrOverflowFlag(SDValue(N, 0)))
56582 return SDValue();
56588 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
56589 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
56604 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
56611 SDValue LHS = N->getOperand(0);
56612 SDValue RHS = N->getOperand(1);
56618 if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST))
56623 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
56628 auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
56629 SDValue Ops[] = {N0, N1};
56632 SDValue Op(N, 0);
56652 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
56653 SDValue LHS = N->getOperand(0);
56654 SDValue RHS = N->getOperand(1);
56655 SDValue BorrowIn = N->getOperand(2);
56657 if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) {
56670 return SDValue();
56674 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
56676 SDValue LHS = N->getOperand(0);
56677 SDValue RHS = N->getOperand(1);
56678 SDValue CarryIn = N->getOperand(2);
56693 SDValue(N, 1).use_empty()) {
56696 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
56697 SDValue Res1 = DAG.getNode(
56716 if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) {
56729 return SDValue();
56732 static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
56749 return SDValue();
56754 return SDValue();
56756 SDValue Op0, Op1, Accum;
56762 return SDValue();
56775 SDValue Mul;
56777 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
56781 SDValue Vec0L, Vec0H, Vec1L, Vec1H;
56786 return SDValue();
56799 return SDValue();
56807 return SDValue();
56811 return SDValue();
56818 return SDValue();
56822 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
56823 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
56826 ArrayRef<SDValue> Ops) {
56833 SDValue R = SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDBuilder);
56842 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
56848 return SDValue();
56853 return SDValue();
56857 SDValue N00, N01, N10, N11;
56860 return SDValue();
56866 return SDValue();
56873 return SDValue();
56881 SDValue In0, In1;
56883 SDValue N00Elt = N00.getOperand(i);
56884 SDValue N01Elt = N01.getOperand(i);
56885 SDValue N10Elt = N10.getOperand(i);
56886 SDValue N11Elt = N11.getOperand(i);
56888 SDValue N00In, N01In, N10In, N11In;
56894 return SDValue();
56903 return SDValue();
56914 return SDValue();
56923 return SDValue();
56927 ArrayRef<SDValue> Ops) {
56957 static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
56960 return SDValue();
56964 return SDValue();
56981 return SDValue();
56990 SDValue LHS =
56992 SDValue RHS =
57001 static SDValue pushAddIntoCmovOfConsts(SDNode *N, const SDLoc &DL,
57009 auto isSuitableCmov = [](SDValue V) {
57021 SDValue Cmov = N->getOperand(0);
57022 SDValue OtherOp = N->getOperand(1);
57026 return SDValue();
57031 return SDValue();
57034 SDValue FalseOp = Cmov.getOperand(0);
57035 SDValue TrueOp = Cmov.getOperand(1);
57051 SDValue X = OtherOp.getOperand(0), Y = OtherOp.getOperand(1);
57066 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
57070 SDValue Op0 = N->getOperand(0);
57071 SDValue Op1 = N->getOperand(1);
57074 if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget))
57077 if (SDValue MAdd = matchPMADDWD(DAG, N, DL, VT, Subtarget))
57079 if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
57081 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
57085 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
57095 SDValue Sum =
57112 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
57119 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
57128 SDValue Accum, Lo0, Lo1, Hi0, Hi1;
57156 static SDValue combineSubABS(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1,
57159 return SDValue();
57161 SDValue Cond = N1.getOperand(3);
57163 return SDValue();
57166 SDValue FalseOp = N1.getOperand(0);
57167 SDValue TrueOp = N1.getOperand(1);
57174 SDValue NegX = Cond.getValue(0);
57175 SDValue X = Cond.getOperand(1);
57179 return SDValue();
57182 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp,
57202 return SDValue();
57205 static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
57206 SDValue Op0 = N->getOperand(0);
57207 SDValue Op1 = N->getOperand(1);
57218 SDValue SetCC = Op1.getOperand(0);
57223 SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG);
57229 return SDValue();
57232 static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) {
57238 return SDValue();
57240 SDValue Sub = N->getOperand(4);
57242 return SDValue();
57244 SDValue SetCC = Sub.getOperand(1);
57247 return SDValue();
57249 SmallVector<SDValue, 5> Ops(N->op_values());
57258 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
57262 SDValue Op0 = N->getOperand(0);
57263 SDValue Op1 = N->getOperand(1);
57266 auto IsNonOpaqueConstant = [&](SDValue Op) {
57279 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0),
57281 SDValue NewAdd =
57286 if (SDValue V = combineSubABS(VT, DL, Op0, Op1, DAG))
57290 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
57306 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0,
57311 if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget))
57314 if (SDValue V = combineAddOrSubToADCOrSBB(N, DL, DAG))
57320 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
57326 SDValue LHS = N->getOperand(0);
57327 SDValue RHS = N->getOperand(1);
57361 return SDValue();
57389 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
57390 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
57396 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
57399 if (llvm::all_of(Ops, [](SDValue Op) {
57404 SDValue Op0 = Ops[0];
57441 SDValue SrcVec = Op0.getOperand(0);
57461 SDValue Src0 = peekThroughBitcasts(Ops[0]);
57462 SDValue Src1 = peekThroughBitcasts(Ops[1]);
57494 if (llvm::all_of(Ops, [Op0](SDValue Op) {
57497 auto ConcatSubOperand = [&](EVT VT, ArrayRef<SDValue> SubOps, unsigned I) {
57498 SmallVector<SDValue> Subs;
57499 for (SDValue SubOp : SubOps)
57507 for (SDValue &Sub : Subs)
57514 auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
57518 SDValue BC0 = peekThroughBitcasts(SubOps[0].getOperand(Op));
57519 if (isa<LoadSDNode>(BC0) && all_of(SubOps, [&](SDValue SubOp) {
57524 SDValue BC = peekThroughBitcasts(SubOps[I].getOperand(Op));
57558 if (!IsSplat && llvm::all_of(Ops, [](SDValue Op) {
57585 llvm::all_of(Ops, [Op0](SDValue Op) {
57600 none_of(Ops, [](SDValue Op) {
57625 all_of(Ops, [&Op0](SDValue Op) {
57629 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0));
57665 SmallVector<SDValue, 2> SubOps;
57675 SDValue Src = concatSubVectors(Ops[0].getOperand(1),
57679 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
57691 SmallVector<SDValue, 2> SubOps;
57707 SDValue Src0 = concatSubVectors(Ops[0].getOperand(0),
57709 SDValue Src1 = concatSubVectors(Ops[0].getOperand(2),
57713 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
57728 SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
57730 SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
57732 SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
57747 SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
57749 SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
57792 llvm::all_of(Ops, [](SDValue Op) {
57795 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
57796 SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
57814 llvm::all_of(Ops, [Op0](SDValue Op) {
57825 llvm::all_of(Ops, [Op0](SDValue Op) {
57877 SDValue LHS = ConcatSubOperand(VT, Ops, 0);
57878 SDValue RHS = ConcatSubOperand(VT, Ops, 1);
57906 llvm::all_of(Ops, [Op0](SDValue Op) {
57974 llvm::all_of(Ops, [Op0](SDValue Op) {
57994 SDValue Sel =
58041 if (SDValue Ld =
58048 if (all_of(Ops, [](SDValue Op) { return getTargetConstantFromNode(Op); })) {
58064 SDValue CV = DAG.getConstantPool(C, PVT);
58067 SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI);
58068 SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits());
58086 if (SDValue BcastLd =
58088 SDValue BcastSrc =
58100 SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
58107 return SDValue();
58110 static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
58116 SmallVector<SDValue, 4> Ops(N->ops());
58134 return SDValue();
58138 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
58143 return SDValue();
58146 static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
58150 return SDValue();
58157 SDValue Vec = N->getOperand(0);
58158 SDValue SubVec = N->getOperand(1);
58190 SDValue Ins = SubVec.getOperand(0);
58203 return SDValue();
58238 SmallVector<SDValue, 2> SubVectorOps;
58240 if (SDValue Fold =
58256 if (all_of(SubVectorOps, [](SDValue SubOp) {
58259 SDValue Op(N, 0);
58260 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
58275 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
58276 SDValue BcastLd =
58280 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
58297 return SDValue();
58306 static SDValue narrowExtractedVectorSelect(SDNode *Ext, const SDLoc &DL,
58308 SDValue Sel = Ext->getOperand(0);
58311 return SDValue();
58318 return SDValue();
58322 return SDValue();
58341 return SDValue();
58350 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
58351 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
58352 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
58353 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
58357 static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
58371 return SDValue();
58374 SDValue InVec = N->getOperand(0);
58376 SDValue InVecBC = peekThroughBitcasts(InVec);
58387 auto isConcatenatedNot = [](SDValue V) {
58391 SDValue NotOp = V->getOperand(0);
58397 SDValue Concat = splitVectorIntBinary(InVecBC, DAG, SDLoc(InVecBC));
58404 return SDValue();
58406 if (SDValue V = narrowExtractedVectorSelect(N, DL, DAG))
58437 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
58462 SmallVector<SDValue, 2> ShuffleInputs;
58472 SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
58482 auto IsExtractFree = [](SDValue V) {
58523 SDValue Src = InVec.getOperand(0);
58532 SDValue Ext = InVec.getOperand(0);
58542 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
58543 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
58544 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
58549 SDValue InVecSrc = InVec.getOperand(0);
58551 SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
58609 SDValue Src0 = InVec.getOperand(0);
58610 SDValue Mask = InVec.getOperand(1);
58611 SDValue Src1 = InVec.getOperand(2);
58615 SDValue Shuffle =
58629 SDValue Ext =
58634 return SDValue();
58637 static SDValue combineSCALAR_TO_VECTOR(SDNode *N, SelectionDAG &DAG,
58641 SDValue Src = N->getOperand(0);
58663 auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) {
58665 return SDValue();
58680 return SDValue();
58683 if (SDValue AnyExt = IsExt64(peekThroughOneUseBitcasts(Src), false))
58688 if (SDValue ZeroExt = IsExt64(peekThroughOneUseBitcasts(Src), true))
58697 SDValue SrcOp = Src.getOperand(0);
58708 SDValue HalfSrc;
58718 // Ensure the same SDValue from the SDNode use is being used.
58727 return SDValue(User, 0);
58729 return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
58743 SDValue SrcVec =
58760 SDValue SrcVec0 =
58762 SDValue SrcVec1 =
58771 return SDValue();
58775 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
58778 SDValue LHS = N->getOperand(0);
58779 SDValue RHS = N->getOperand(1);
58793 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(64), DCI))
58794 return SDValue(N, 0);
58824 return SDValue();
58828 static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG,
58831 SDValue LHS = N->getOperand(0);
58832 SDValue RHS = N->getOperand(1);
58867 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
58868 return SDValue(N, 0);
58870 return SDValue();
58873 static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
58877 SDValue In = N->getOperand(0);
58894 SDValue Load = DAG.getExtLoad(
58897 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
58924 SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT));
58932 SDValue Op(N, 0);
58934 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
58938 return SDValue();
58941 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
58955 SDValue Src = N->getOperand(0).getOperand(0);
58960 SDValue Shift = DAG.getNode(X86ISD::KSHIFTR, DL, SrcVT, Src,
58969 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
58970 return SDValue(N, 0);
58972 return SDValue();
58978 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
58981 return SDValue();
58984 return SDValue();
58988 return SDValue();
58991 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
59000 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
59005 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
59015 return SDValue();
59032 return SDValue();
59035 return SDValue();
59038 return SDValue();
59042 return SDValue();
59046 return SDValue();
59055 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
59057 SmallVector<SDValue, 4> Ops(NumConcats, Fill);
59065 SDValue Cvt, Chain;
59097 static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
59105 return SDValue();
59109 SDValue Ptr = MemIntrin->getBasePtr();
59110 SDValue Chain = MemIntrin->getChain();
59124 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
59127 return DCI.CombineTo(N, Extract, SDValue(User, 1));
59130 return SDValue();
59133 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
59136 return SDValue();
59140 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
59145 return SDValue();
59149 SDValue Cvt, Chain;
59158 SDValue Cvt0, Cvt1;
59159 SDValue Op0 = Src.getOperand(0);
59160 SDValue Op1 = Src.getOperand(1);
59165 return SDValue();
59186 return SDValue();
59190 return SDValue();
59200 SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
59224 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
59225 SDValue Src = N->getOperand(0);
59232 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
59237 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
59242 return SDValue();
59245 static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG,
59249 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBits), DCI))
59250 return SDValue(N, 0);
59252 return SDValue();
59258 static SDValue FixupMMXIntrinsicTypes(SDNode *N, SelectionDAG &DAG) {
59262 SmallVector<SDValue, 8> Args;
59263 for (const SDValue &Arg : N->op_values()) {
59281 SDValue Result = DAG.getNode(N->getOpcode(), dl, NewVTs, Args);
59283 SmallVector<SDValue, 2> Returns;
59291 return SDValue();
59293 static SDValue combineINTRINSIC_WO_CHAIN(SDNode *N, SelectionDAG &DAG,
59296 return SDValue();
59304 return SDValue();
59307 static SDValue combineINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
59310 return SDValue();
59318 return SDValue();
59321 static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
59324 return SDValue();
59332 return SDValue();
59335 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
59533 return SDValue();
59596 SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc &dl,
59597 SDValue Value, SDValue Addr,
59607 SDValue Chain = Value;
59640 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
59652 auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
59663 auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
59676 auto IsFoldableZext = [](SDValue Op) {
59695 SDValue N0 = Op.getOperand(0);
59716 SDValue N0 = Op.getOperand(0);
59717 SDValue N1 = Op.getOperand(1);
60129 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
60130 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
60134 return SDValue();
60147 SDValue CC = getSETCC(Cond, Glue, DL, DAG);
60149 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
60156 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
60158 std::vector<SDValue> &Ops,
60160 SDValue Result;