Lines Matching defs:DAG
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
10 // selection DAG.
1428 // so that DAG combine doesn't try to turn it into uint_to_fp.
2745 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2747 EVT PtrTy = getPointerTy(DAG.getDataLayout());
2749 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2899 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2900 MachineFunction &MF = DAG.getMachineFunction();
2914 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2989 SelectionDAG &DAG) {
2994 RHS = DAG.getConstant(0, DL, RHS.getValueType());
3007 RHS = DAG.getConstant(0, DL, RHS.getValueType());
3416 const SelectionDAG &DAG,
3430 return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
3501 SelectionDAG &DAG) const {
3504 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
3511 if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
3550 // Maybe we could check if the DAG has the flipped node already in the
3644 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
3645 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
3648 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
3939 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
3947 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3956 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
3957 DAG.getConstant(Values[i], dl, EltVT);
3960 Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
3961 DAG.getConstant(0, dl, EltVT));
3963 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3965 ConstsNode = DAG.getBitcast(VT, ConstsNode);
3970 MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
3978 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3987 Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
3993 Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT));
3994 Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT));
3997 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
4000 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
4002 Ops.push_back(DAG.getConstant(V, dl, EltVT));
4006 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
4007 return DAG.getBitcast(VT, ConstsNode);
4011 SelectionDAG &DAG, const SDLoc &dl) {
4013 return getConstVector(Bits, Undefs, VT, DAG, dl);
4018 SelectionDAG &DAG, const SDLoc &dl) {
4027 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4029 Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
4032 Vec = DAG.getConstantFP(+0.0, dl, VT);
4036 Vec = DAG.getConstant(0, dl, VT);
4039 Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
4041 return DAG.getBitcast(VT, Vec);
4067 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
4072 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
4085 return DAG.getBuildVector(ResultVT, dl,
4092 return DAG.getUNDEF(ResultVT);
4094 SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
4095 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
4098 /// Generate a DAG to grab 128-bits from a vector > 128 bits. This
4105 SelectionDAG &DAG, const SDLoc &dl) {
4108 return extractSubVector(Vec, IdxVal, DAG, dl, 128);
4111 /// Generate a DAG to grab 256-bits from a 512-bit vector.
4113 SelectionDAG &DAG, const SDLoc &dl) {
4115 return extractSubVector(Vec, IdxVal, DAG, dl, 256);
4119 SelectionDAG &DAG, const SDLoc &dl,
4138 SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
4139 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
4142 /// Generate a DAG to put 128-bits into a vector > 128 bits. This
4149 SelectionDAG &DAG, const SDLoc &dl) {
4151 return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
4157 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4171 Vec = extract128BitVector(Vec, 0, DAG, dl);
4173 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
4174 : DAG.getUNDEF(VT);
4175 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
4176 DAG.getVectorIdxConstant(0, dl));
4182 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4190 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
4206 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4209 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
4216 SelectionDAG &DAG) {
4235 Ops.push_back(DAG.getUNDEF(SubVT));
4247 if (collectConcatOps(Lo.getNode(), LoOps, DAG) &&
4248 collectConcatOps(Hi.getNode(), HiOps, DAG) &&
4266 Ops.push_back(DAG.getUNDEF(SubVT));
4280 SelectionDAG &DAG) {
4282 if (!collectConcatOps(V.getNode(), SubOps, DAG))
4293 EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
4295 return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps);
4300 static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG) {
4302 return collectConcatOps(N, Ops, DAG);
4305 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
4315 SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
4316 if (DAG.isSplatValue(Op, /*AllowUndefs*/ false))
4319 SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
4324 static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) {
4337 std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl);
4341 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
4342 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
4343 DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps),
4344 DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps));
4349 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG,
4360 return splitVectorOp(Op, DAG, dl);
4365 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG,
4372 return splitVectorOp(Op, DAG, dl);
4383 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4407 return Builder(DAG, DL, Ops);
4416 SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
4418 Subs.push_back(Builder(DAG, DL, SubOps));
4420 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
4426 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
4438 !DAG.getTargetLoweringInfo().isTypeLegal(SVT))
4450 return DAG.getConstant(SplatValue, DL, DstVT);
4477 Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512);
4480 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
4484 Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
4489 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4507 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
4516 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4517 DAG.getConstant(0, dl, WideOpVT),
4519 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4528 SDValue Undef = DAG.getUNDEF(WideOpVT);
4532 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
4533 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
4535 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4536 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4538 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4539 DAG.getConstant(0, dl, WideOpVT),
4541 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4542 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4545 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4550 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4551 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4552 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4560 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4561 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4566 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4567 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4569 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4570 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4572 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4577 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4578 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4582 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
4583 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4584 DAG.getConstant(0, dl, WideOpVT),
4588 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4591 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
4592 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4593 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4595 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4596 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4604 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
4613 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
4614 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
4615 Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
4616 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4617 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4618 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4619 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4620 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4623 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4627 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4628 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4629 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4630 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4634 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
4635 DAG.getTargetConstant(LowShift, dl, MVT::i8));
4636 Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
4637 DAG.getTargetConstant(LowShift, dl, MVT::i8));
4641 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
4642 DAG.getTargetConstant(HighShift, dl, MVT::i8));
4643 High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
4644 DAG.getTargetConstant(HighShift, dl, MVT::i8));
4647 Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
4648 SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
4651 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4654 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
4661 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
4662 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
4663 return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
4669 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
4673 SDValue Vec = DAG.getAllOnesConstant(dl, MVT::getVectorVT(MVT::i32, NumElts));
4674 return DAG.getBitcast(VT, Vec);
4678 SDValue In, SelectionDAG &DAG) {
4691 In = extractSubVector(In, 0, DAG, DL,
4697 Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode);
4699 return DAG.getNode(Opcode, DL, VT, In);
4704 SDValue Mask, SelectionDAG &DAG) {
4705 LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask);
4706 RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS);
4707 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
4742 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
4746 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
4756 return DAG.getBuildVector(VT, dl, Ops);
4759 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4763 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4767 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4771 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4775 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4781 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4805 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
4806 DAG.getBitcast(VT, RHS), PackMask);
4812 DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits &&
4813 DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits)
4814 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4816 if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits &&
4817 DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits)
4818 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4822 SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
4825 LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt);
4826 RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt);
4828 SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
4829 LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask);
4830 RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask);
4832 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4836 LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt);
4837 RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt);
4839 LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt);
4840 RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt);
4841 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4851 SelectionDAG &DAG) {
4854 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
4860 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
5288 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
5302 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
5303 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
5304 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), VT, Not,
5329 return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1),
5330 getConstVector(EltBits, UndefElts, VT, DAG, DL));
5337 if (collectConcatOps(V.getNode(), CatOps, DAG)) {
5339 SDValue NotCat = IsNOT(CatOp, DAG);
5342 CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
5344 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), VT, CatOps);
5348 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
5351 if (SDValue Op1 = IsNOT(V.getOperand(1), DAG))
5352 if (SDValue Op0 = IsNOT(V.getOperand(0), DAG))
5353 return DAG.getNode(ISD::AND, SDLoc(V), VT, DAG.getBitcast(VT, Op0),
5354 DAG.getBitcast(VT, Op1));
5997 const SelectionDAG &DAG, unsigned Depth,
6007 const SelectionDAG &DAG, unsigned Depth,
6072 if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG,
6074 !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG,
6157 if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG,
6298 DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
6300 DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
6318 !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
6320 !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
6370 std::optional<uint64_t> Amt = DAG.getValidShiftAmount(N, DemandedElts);
6465 if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt)
6548 const SelectionDAG &DAG, unsigned Depth,
6562 if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
6573 const SelectionDAG &DAG, unsigned Depth,
6577 KnownZero, DAG, Depth, ResolveKnownElts);
6582 const SelectionDAG &DAG, unsigned Depth = 0,
6590 return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth,
6597 SelectionDAG &DAG) {
6606 SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
6608 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
6610 SDValue BcstLd = DAG.getMemIntrinsicNode(
6612 DAG.getMachineFunction().getMachineMemOperand(
6614 DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
6621 SelectionDAG &DAG, unsigned Depth) {
6634 return DAG.getUNDEF(VT.getVectorElementType());
6637 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6652 return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT)
6653 : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT);
6655 return DAG.getUNDEF(ShufSVT);
6659 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6670 return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1);
6671 return getShuffleScalarElt(Vec, Index, DAG, Depth + 1);
6680 return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1);
6687 return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1);
6695 return getShuffleScalarElt(Src, Index, DAG, Depth + 1);
6707 return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1);
6712 : DAG.getUNDEF(VT.getVectorElementType());
6724 SelectionDAG &DAG,
6746 V = getZeroVector(VT, Subtarget, DAG, DL);
6749 V = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6750 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V);
6751 V = DAG.getBitcast(VT, V);
6755 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, V, Op.getOperand(i),
6756 DAG.getVectorIdxConstant(i, DL));
6766 SelectionDAG &DAG,
6774 DAG, Subtarget);
6785 SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), DL, MVT::i32);
6787 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt,
6788 DAG.getConstant(I * 8, DL, MVT::i8));
6789 V = V ? DAG.getNode(ISD::OR, DL, MVT::i32, V, Elt) : Elt;
6792 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V);
6793 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32, V);
6794 V = DAG.getBitcast(MVT::v8i16, V);
6805 Elt = DAG.getZExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6807 Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6813 NextElt = DAG.getZExtOrTrunc(NextElt, DL, MVT::i32);
6815 NextElt = DAG.getAnyExtOrTrunc(NextElt, DL, MVT::i32);
6816 NextElt = DAG.getNode(ISD::SHL, DL, MVT::i32, NextElt,
6817 DAG.getConstant(8, DL, MVT::i8));
6819 Elt = DAG.getNode(ISD::OR, DL, MVT::i32, NextElt, Elt);
6829 V = getZeroVector(MVT::v8i16, Subtarget, DAG, DL);
6831 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, Elt);
6832 V = DAG.getBitcast(MVT::v8i16, V);
6836 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
6837 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16, V, Elt,
6838 DAG.getVectorIdxConstant(i / 2, DL));
6841 return DAG.getBitcast(MVT::v16i8, V);
6848 SelectionDAG &DAG,
6854 return LowerBuildVectorAsInsert(Op, DL, NonZeroMask, NumNonZero, NumZero, DAG,
6860 SelectionDAG &DAG,
6875 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
6876 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
6877 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
6878 return DAG.getBitcast(VT, Dup);
6938 ? DAG.getUNDEF(VT)
6939 : getZeroVector(VT, Subtarget, DAG, DL);
6941 V1 = DAG.getBitcast(VT, V1);
6942 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask);
6970 V1 = DAG.getBitcast(MVT::v4f32, V1);
6972 V2 = DAG.getBitcast(MVT::v4f32, V2);
6980 DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6981 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
6982 return DAG.getBitcast(VT, Result);
6987 SelectionDAG &DAG, const TargetLowering &TLI,
6992 SrcOp = DAG.getBitcast(ShVT, SrcOp);
6994 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
6995 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
6999 SelectionDAG &DAG) {
7017 } else if (DAG.isBaseWithConstantOffset(Ptr) &&
7031 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7032 MaybeAlign InferredAlign = DAG.InferPtrAlign(Ptr);
7053 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
7054 DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
7060 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
7061 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
7066 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask);
7121 const SDLoc &DL, SelectionDAG &DAG,
7173 return DAG.getUNDEF(VT);
7175 return VT.isInteger() ? DAG.getConstant(0, DL, VT)
7176 : DAG.getConstantFP(0.0, DL, VT);
7178 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7206 return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes,
7227 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
7232 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
7237 DAG.makeEquivalentMemoryOrdering(LD, NewLd);
7243 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout());
7263 return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
7283 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
7284 : DAG.getConstantFP(0.0, DL, VT);
7285 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
7295 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
7298 DAG, Subtarget, IsAfterLegalize);
7300 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
7301 HalfLD, DAG.getVectorIdxConstant(0, DL));
7318 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
7320 SDValue ResNode = DAG.getMemIntrinsicNode(
7325 DAG.makeEquivalentMemoryOrdering(LD, ResNode);
7326 return DAG.getBitcast(VT, ResNode);
7346 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
7365 ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize)
7368 RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
7371 EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
7375 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) {
7379 Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL);
7388 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad);
7390 return DAG.getBitcast(VT, Broadcast);
7403 SelectionDAG &DAG,
7408 if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
7415 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
7505 SelectionDAG &DAG) {
7514 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7557 SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand);
7559 Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits());
7560 return DAG.getBitcast(VT, Bcst);
7579 LLVMContext *Ctx = DAG.getContext();
7580 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
7586 SDValue CP = DAG.getConstantPool(C, PVT);
7590 SDVTList Tys = DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);
7591 SDValue Ops[] = {DAG.getEntryNode(), CP};
7593 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7595 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7597 return DAG.getBitcast(VT, Brdcst);
7602 SDValue VCP = DAG.getConstantPool(VecC, PVT);
7606 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7607 SDValue Ops[] = {DAG.getEntryNode(), VCP};
7609 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7610 return DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, dl, Tys,
7648 bool OptForSize = DAG.shouldOptForSize();
7677 DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
7680 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7681 SDValue Ops[] = {DAG.getEntryNode(), CP};
7683 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7684 return DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7692 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7705 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7708 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7710 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7719 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7722 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7724 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7729 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7771 SelectionDAG &DAG) {
7775 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7831 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7832 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
7835 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
7836 DAG.getVectorIdxConstant(Idx, DL));
7842 static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG,
7849 NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16,
7851 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
7852 return DAG.getBitcast(VT, Res);
7857 SelectionDAG &DAG,
7897 Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
7898 DAG.getConstant(1, dl, MVT::i8));
7902 SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
7903 DAG.getAllOnesConstant(dl, MVT::i32),
7904 DAG.getConstant(0, dl, MVT::i32));
7905 Select = DAG.getBitcast(MVT::v32i1, Select);
7906 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);
7909 SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
7910 DAG.getAllOnesConstant(dl, ImmVT),
7911 DAG.getConstant(0, dl, ImmVT));
7913 Select = DAG.getBitcast(VecVT, Select);
7914 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select,
7915 DAG.getVectorIdxConstant(0, dl));
7923 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
7924 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
7925 ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
7926 ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
7927 DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
7930 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
7932 DstVec = DAG.getBitcast(VecVT, Imm);
7933 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
7934 DAG.getVectorIdxConstant(0, dl));
7937 DstVec = DAG.getUNDEF(VT);
7940 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
7942 DAG.getVectorIdxConstant(InsertIdx, dl));
7981 const SDLoc &DL, SelectionDAG &DAG,
7994 V0 = DAG.getUNDEF(VT);
7995 V1 = DAG.getUNDEF(VT);
8095 const SDLoc &DL, SelectionDAG &DAG,
8103 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
8104 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
8105 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
8106 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
8109 SDValue LO = DAG.getUNDEF(NewVT);
8110 SDValue HI = DAG.getUNDEF(NewVT);
8115 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
8117 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
8121 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
8124 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
8127 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
8136 const X86Subtarget &Subtarget, SelectionDAG &DAG,
8146 SDValue InVec0 = DAG.getUNDEF(VT);
8147 SDValue InVec1 = DAG.getUNDEF(VT);
8263 SelectionDAG &DAG,
8274 const TargetOptions &Options = DAG.getTarget().Options;
8293 SelectionDAG &DAG) {
8297 if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
8305 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
8307 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
8322 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1);
8323 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1);
8324 return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask);
8327 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
8330 static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
8335 V0 = DAG.getUNDEF(VT);
8336 V1 = DAG.getUNDEF(VT);
8420 const SDLoc &DL, SelectionDAG &DAG,
8428 V0 = extractSubVector(V0, 0, DAG, DL, Width);
8430 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, DL, Width);
8433 V1 = extractSubVector(V1, 0, DAG, DL, Width);
8435 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, DL, Width);
8447 V0 = extractSubVector(V0, 0, DAG, DL, 128);
8448 V1 = extractSubVector(V1, 0, DAG, DL, 128);
8449 SDValue Half = DAG.getNode(HOpcode, DL, HalfVT, V0, V1);
8450 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, DL, 256);
8453 return DAG.getNode(HOpcode, DL, VT, V0, V1);
8459 SelectionDAG &DAG) {
8476 if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
8477 return getHopForBuildVector(BV, DL, DAG, HOpcode, V0, V1);
8503 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, Half, InVec0, InVec1) &&
8504 isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, Half, NumElts, InVec2,
8509 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, Half, InVec0,
8511 isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, Half, NumElts, InVec2,
8533 return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
8541 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, NumElts, InVec0,
8544 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, NumElts, InVec0,
8547 else if (isHorizontalBinOpPart(BV, ISD::FADD, DL, DAG, 0, NumElts, InVec0,
8550 else if (isHorizontalBinOpPart(BV, ISD::FSUB, DL, DAG, 0, NumElts, InVec0,
8565 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
8573 SelectionDAG &DAG);
8583 SelectionDAG &DAG) {
8586 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8630 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
8643 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
8644 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
8645 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
8652 return LowerShift(Res, Subtarget, DAG);
8659 SelectionDAG &DAG,
8674 return getOnesVector(VT, DAG, DL);
8684 const SDLoc &DL, SelectionDAG &DAG,
8697 IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec),
8700 IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG,
8704 IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec),
8707 IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT);
8717 Subtarget, DAG, SDLoc(IndicesVec));
8719 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8721 return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits);
8725 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));
8730 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
8747 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx,
8748 DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT));
8749 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx,
8750 DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT));
8784 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8789 return DAG.getSelectCC(
8791 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
8792 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
8793 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
8801 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
8802 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
8803 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
8804 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
8805 return DAG.getNode(
8807 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
8808 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
8810 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
8811 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
8812 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
8813 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
8814 auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
8821 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
8822 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
8823 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
8827 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
8837 return DAG.getBitcast(
8839 MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),
8840 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
8848 SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);
8849 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8851 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8854 return DAG.getBitcast(
8855 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
8856 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8859 SDValue Res = DAG.getSelectCC(
8860 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
8861 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
8862 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
8864 return DAG.getBitcast(VT, Res);
8872 SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG,
8875 DAG, SDLoc(IndicesVec));
8877 DAG, Subtarget);
8878 return extract256BitVector(Res, 0, DAG, DL);
8882 SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);
8884 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
8886 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
8888 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8890 return DAG.getBitcast(
8891 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
8892 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8895 SDValue Res = DAG.getSelectCC(
8896 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
8897 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
8898 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
8900 return DAG.getBitcast(VT, Res);
8931 IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec);
8933 SrcVec = DAG.getBitcast(ShuffleVT, SrcVec);
8935 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
8936 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
8937 return DAG.getBitcast(VT, Res);
8953 SelectionDAG &DAG,
8993 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8997 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
9007 return LowerBUILD_VECTORvXi1(Op, dl, DAG, Subtarget);
9011 return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget);
9013 if (SDValue VectorCst = materializeVectorConstant(Op, dl, DAG, Subtarget))
9050 return DAG.getUNDEF(VT);
9054 return DAG.getFreeze(DAG.getUNDEF(VT));
9058 return getZeroVector(VT, Subtarget, DAG, dl);
9068 SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT));
9080 SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts);
9081 SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT));
9082 SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt);
9083 return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask);
9104 DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems));
9105 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl);
9109 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, dl, Subtarget, DAG))
9111 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, dl, Subtarget, DAG))
9113 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, dl, Subtarget, DAG))
9115 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, dl, Subtarget, DAG))
9134 LLVMContext &Context = *DAG.getContext();
9149 InsIndex = DAG.getVectorIdxConstant(i, dl);
9153 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
9161 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
9162 MachineFunction &MF = DAG.getMachineFunction();
9164 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
9168 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);
9178 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
9179 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);
9193 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9201 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9204 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
9210 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
9212 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item);
9213 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
9214 return DAG.getBitcast(VT, Item);
9224 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
9226 NumBits/2, DAG, *this, dl);
9238 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9239 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
9253 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
9263 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, dl, DAG, Subtarget))
9270 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
9279 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
9291 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
9292 DAG.getBuildVector(NarrowVT, dl, Ops));
9295 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
9307 DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
9308 SDValue Upper = DAG.getBuildVector(
9312 return concatSubVectors(Lower, Upper, DAG, dl);
9320 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
9322 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
9330 NumZero, DAG, Subtarget))
9335 NumZero, DAG, Subtarget))
9340 if (SDValue V = LowerBuildVectorv4x32(Op, dl, DAG, Subtarget))
9349 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);
9351 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9361 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
9364 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9367 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9380 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
9386 if (SDValue Sh = buildFromShuffleMostly(Op, dl, DAG))
9393 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
9395 Result = DAG.getUNDEF(VT);
9399 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
9400 Op.getOperand(i), DAG.getVectorIdxConstant(i, dl));
9411 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9413 Ops[i] = DAG.getUNDEF(VT);
9430 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
9437 // TODO: Detect subvector broadcast here instead of DAG combine?
9438 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9475 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9477 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9479 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9483 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
9484 : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
9485 : DAG.getUNDEF(ResVT));
9493 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(i),
9494 DAG.getVectorIdxConstant(i * NumSubElems, dl));
9506 SelectionDAG & DAG) {
9538 Op = widenSubVector(ShiftVT, SubVec, false, Subtarget, DAG, dl);
9539 Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, Op,
9540 DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
9541 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
9542 DAG.getVectorIdxConstant(0, dl));
9547 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
9553 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec,
9554 DAG.getVectorIdxConstant(Idx * SubVecNumElts, dl));
9560 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9562 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9564 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9573 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, DAG.getUNDEF(ResVT),
9574 Op.getOperand(0), DAG.getVectorIdxConstant(0, dl));
9575 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1),
9576 DAG.getVectorIdxConstant(NumElems / 2, dl));
9581 SelectionDAG &DAG) {
9584 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
9594 return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
9879 const SelectionDAG &DAG,
9932 return (ZeroV1.isZero() || DAG.MaskedVectorIsZero(V1, ZeroV1)) &&
9933 (ZeroV2.isZero() || DAG.MaskedVectorIsZero(V2, ZeroV2));
9939 const SelectionDAG &DAG) {
9949 bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) ||
9950 isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG));
9955 const SelectionDAG &DAG) {
9968 if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) ||
9969 isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG))
10022 SelectionDAG &DAG) {
10023 return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
10059 SelectionDAG &DAG) {
10060 return DAG.getTargetConstant(getSHUFPDImm(Mask), DL, MVT::i8);
10100 SelectionDAG &DAG) {
10112 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
10118 PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);
10139 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
10144 return DAG.getBitcast(
10145 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
10146 DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
10150 const X86Subtarget &Subtarget, SelectionDAG &DAG,
10158 SelectionDAG &DAG) {
10166 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
10171 Subtarget, DAG, DL);
10172 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
10174 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
10180 SelectionDAG &DAG,
10199 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1,
10202 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
10203 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
10208 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1,
10211 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
10212 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
10238 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
10239 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
10247 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) {
10254 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) {
10268 SelectionDAG &DAG) {
10272 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
10277 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
10282 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
10286 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
10295 SelectionDAG &DAG) {
10311 V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),
10312 DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3});
10313 V1 = DAG.getBitcast(VT, V1);
10314 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
10359 SelectionDAG &DAG, bool ZeroUppers) {
10366 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT))
10371 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src);
10375 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10376 return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits());
10381 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10382 return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
10389 SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512);
10390 return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers);
10395 SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src);
10397 Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
10422 SelectionDAG &DAG) {
10447 Src = DAG.getBitcast(SrcVT, Src);
10450 ((DAG.ComputeNumSignBits(Src) > EltSizeInBits) ||
10451 (DAG.computeKnownBits(Src).countMinLeadingZeros() >= EltSizeInBits)))
10461 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10472 SelectionDAG &DAG) {
10514 return DAG.areNonVolatileConsecutiveLoads(
10526 SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
10530 Src = DAG.getBitcast(SrcVT, Src);
10535 Src = DAG.getNode(
10537 DAG.getTargetConstant(Offset * EltSizeInBits, DL, MVT::i8));
10539 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10624 const SelectionDAG &DAG,
10646 if ((N1.isUndef() || IsZero1 || DAG.MaskedValueIsZero(N1, ZeroMask)) &&
10647 (N2.isUndef() || IsZero2 || DAG.MaskedValueIsZero(N2, ZeroMask))) {
10658 DAG.ComputeNumSignBits(N1) > NumPackedBits) &&
10660 DAG.ComputeNumSignBits(N2) > NumPackedBits)) {
10678 if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2))
10685 if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1))
10696 SelectionDAG &DAG) {
10702 if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
10729 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
10730 DAG.getBitcast(SrcVT, V2));
10747 SelectionDAG &DAG) {
10759 Zero = DAG.getConstantFP(0.0, DL, EltVT);
10761 AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT);
10765 Zero = DAG.getConstant(0, DL, EltVT);
10766 AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10786 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10787 VMask = DAG.getBitcast(LogicVT, VMask);
10788 V = DAG.getBitcast(LogicVT, V);
10789 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10790 return DAG.getBitcast(VT, And);
10800 SelectionDAG &DAG) {
10803 SDValue Zero = DAG.getConstant(0, DL, EltVT);
10804 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10812 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10813 return getBitSelect(DL, VT, V1, V2, V1Mask, DAG);
10819 SelectionDAG &DAG);
10909 SelectionDAG &DAG) {
10919 V1 = getZeroVector(VT, Subtarget, DAG, DL);
10921 V2 = getZeroVector(VT, Subtarget, DAG, DL);
10940 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
10941 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10952 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10953 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10961 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10962 DAG.getTargetConstant(LoMask, DL, MVT::i8));
10963 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10964 DAG.getTargetConstant(HiMask, DL, MVT::i8));
10965 return DAG.getVectorShuffle(
10979 Subtarget, DAG))
10984 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10985 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10991 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
11026 ? DAG.getUNDEF(MVT::i8)
11027 : DAG.getSignedConstant(Mask[i] < Size ? -1 : 0, DL, MVT::i8));
11029 V1 = DAG.getBitcast(BlendVT, V1);
11030 V2 = DAG.getBitcast(BlendVT, V2);
11031 return DAG.getBitcast(
11033 DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
11043 bool OptForSize = DAG.shouldOptForSize();
11046 Subtarget, DAG))
11053 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11054 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
11069 SelectionDAG &DAG,
11096 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11097 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
11108 SelectionDAG &DAG) {
11115 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
11175 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11176 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
11192 SelectionDAG &DAG) {
11237 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11238 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11243 V1 = DAG.getBitcast(UnpackVT, V1);
11244 V2 = DAG.getBitcast(UnpackVT, V2);
11247 return DAG.getBitcast(
11248 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
11287 return DAG.getVectorShuffle(
11289 DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT,
11291 DAG.getUNDEF(VT), PermMask);
11301 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11356 SDValue Rotate = DAG.getBitcast(
11357 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
11358 DAG.getBitcast(ByteVT, Lo),
11359 DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
11372 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
11418 const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11449 &DAG](SDValue &Input,
11459 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input);
11485 DAG, true))
11496 lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
11499 DL, VT, V1, V2, Mask, Subtarget, DAG))
11503 DAG))
11507 DL, VT, V1, V2, Mask, Subtarget, DAG))
11532 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11533 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11534 return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask);
11560 SelectionDAG &DAG) {
11583 V1 = DAG.getBitcast(RotateVT, V1);
11584 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
11585 DAG.getTargetConstant(ShlAmt, DL, MVT::i8));
11586 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
11587 DAG.getTargetConstant(SrlAmt, DL, MVT::i8));
11588 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
11589 return DAG.getBitcast(VT, Rot);
11593 DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1),
11594 DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
11595 return DAG.getBitcast(VT, Rot);
11711 SelectionDAG &DAG) {
11722 Lo = DAG.getBitcast(ByteVT, Lo);
11723 Hi = DAG.getBitcast(ByteVT, Hi);
11729 return DAG.getBitcast(
11730 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
11731 DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
11746 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
11747 DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
11749 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
11750 DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
11751 return DAG.getBitcast(VT,
11752 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
11769 SelectionDAG &DAG) {
11780 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
11781 DAG.getTargetConstant(Rotation, DL, MVT::i8));
11798 return DAG.getNode(X86ISD::VALIGN, DL, VT, Src,
11799 getZeroVector(VT, Subtarget, DAG, DL),
11800 DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8));
11807 return DAG.getNode(X86ISD::VALIGN, DL, VT,
11808 getZeroVector(VT, Subtarget, DAG, DL), Src,
11809 DAG.getTargetConstant(ZeroHi, DL, MVT::i8));
11820 SelectionDAG &DAG) {
11843 Res = DAG.getBitcast(MVT::v16i8, Res);
11852 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11853 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11854 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11855 DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
11858 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11859 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11860 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11861 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11867 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11868 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11870 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11871 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11872 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11873 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11877 return DAG.getBitcast(VT, Res);
11969 SelectionDAG &DAG, bool BitwiseOnly) {
11994 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
11996 V = DAG.getBitcast(ShiftVT, V);
11997 V = DAG.getNode(Opcode, DL, ShiftVT, V,
11998 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
11999 return DAG.getBitcast(VT, V);
12128 const APInt &Zeroable, SelectionDAG &DAG) {
12131 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
12132 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12133 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12136 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
12137 V2 ? V2 : DAG.getUNDEF(VT),
12138 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12139 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12155 ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
12183 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
12195 InputV = DAG.getBitcast(VT, InputV);
12198 DL, ExtVT, InputV, DAG);
12199 return DAG.getBitcast(VT, InputV);
12203 InputV = DAG.getBitcast(VT, InputV);
12210 return DAG.getBitcast(
12211 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12212 DAG.getBitcast(MVT::v4i32, InputV),
12213 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
12218 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12219 DAG.getBitcast(MVT::v4i32, InputV),
12220 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
12223 return DAG.getBitcast(
12224 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
12225 DAG.getBitcast(MVT::v8i16, InputV),
12226 getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
12236 SDValue Lo = DAG.getBitcast(
12237 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12238 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12239 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
12242 return DAG.getBitcast(VT, Lo);
12245 SDValue Hi = DAG.getBitcast(
12246 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12247 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12248 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
12249 return DAG.getBitcast(VT,
12250 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
12262 PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
12266 AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
12268 InputV = DAG.getBitcast(MVT::v16i8, InputV);
12269 return DAG.getBitcast(
12270 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
12271 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
12281 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
12294 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12295 : getZeroVector(InputVT, Subtarget, DAG, DL);
12296 InputV = DAG.getBitcast(InputVT, InputV);
12297 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
12302 return DAG.getBitcast(VT, InputV);
12320 SelectionDAG &DAG) {
12389 InputV, Mask, Subtarget, DAG);
12424 V = DAG.getBitcast(MVT::v2i64, V);
12425 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12426 return DAG.getBitcast(VT, V);
12437 SelectionDAG &DAG) {
12454 return DAG.getBitcast(EltVT, S);
12483 SelectionDAG &DAG) {
12517 DAG);
12518 if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
12520 V2S = DAG.getBitcast(EltVT, V2S);
12530 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
12537 SDValue BitMask = getConstVector(Bits, VT, DAG, DL);
12538 V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask);
12539 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12540 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2));
12541 return DAG.getNode(ISD::OR, DL, VT, V1, V2);
12544 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12571 return DAG.getNode(MovOpc, DL, ExtVT, V1, V2);
12578 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
12580 V2 = DAG.getBitcast(VT, V2);
12590 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
12592 V2 = DAG.getBitcast(MVT::v16i8, V2);
12593 V2 = DAG.getNode(
12595 DAG.getTargetConstant(V2Index * EltBits / 8, DL, MVT::i8));
12596 V2 = DAG.getBitcast(VT, V2);
12609 SelectionDAG &DAG) {
12648 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
12649 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
12651 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
12652 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
12697 SelectionDAG &DAG) {
12730 (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask, DAG)))
12737 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12740 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
12741 DAG.getVectorIdxConstant(0, DL));
12752 SelectionDAG &DAG) {
12835 DL, VT, V, BroadcastIdx, Subtarget, DAG))
12860 DAG.getMemBasePlusOffset(BaseAddr, TypeSize::getFixed(Offset), DL);
12866 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
12868 V = DAG.getMemIntrinsicNode(
12870 DAG.getMachineFunction().getMachineMemOperand(
12872 DAG.makeEquivalentMemoryOrdering(Ld, V);
12873 return DAG.getBitcast(VT, V);
12876 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
12877 DAG.getMachineFunction().getMachineMemOperand(
12879 DAG.makeEquivalentMemoryOrdering(Ld, V);
12902 V = extractSubVector(V, 0, DAG, DL, 128);
12903 V = DAG.getVectorShuffle(V.getValueType(), DL, V, V, ExtractMask);
12914 V = extract128BitVector(V, ExtractIdx, DAG, DL);
12920 V = DAG.getBitcast(MVT::f64, V);
12922 V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);
12923 return DAG.getBitcast(VT, V);
12925 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);
12934 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
12941 V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
12947 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V));
12959 ArrayRef<int> Mask, SelectionDAG &DAG) {
13020 VA = DAG.getUNDEF(MVT::v4f32);
13046 SelectionDAG &DAG) {
13052 if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
13056 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
13057 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
13070 SelectionDAG &DAG) {
13078 Mask, Subtarget, DAG))
13088 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
13089 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13092 return DAG.getNode(
13094 Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
13095 Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
13096 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13104 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13110 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13117 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13124 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
13127 return DAG.getNode(
13129 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
13133 Zeroable, Subtarget, DAG))
13137 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask, DAG))
13141 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
13142 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13154 SelectionDAG &DAG) {
13162 Mask, Subtarget, DAG))
13168 V1 = DAG.getBitcast(MVT::v4i32, V1);
13173 return DAG.getBitcast(
13175 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13176 getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
13184 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13190 DAG, /*BitwiseOnly*/ false))
13196 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13202 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13210 Zeroable, Subtarget, DAG))
13214 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask, DAG))
13222 Zeroable, Subtarget, DAG))
13226 Subtarget, DAG))
13234 Zeroable, Subtarget, DAG);
13240 V1 = DAG.getBitcast(MVT::v2f64, V1);
13241 V2 = DAG.getBitcast(MVT::v2f64, V2);
13242 return DAG.getBitcast(MVT::v2i64,
13243 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
13253 SDValue V2, SelectionDAG &DAG) {
13277 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
13278 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13316 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
13317 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13332 return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG);
13334 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
13335 getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
13346 SelectionDAG &DAG) {
13353 Zeroable, Subtarget, DAG))
13361 Mask, Subtarget, DAG))
13367 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
13369 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
13375 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
13376 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13383 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
13385 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
13390 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
13391 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13396 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) {
13397 ZExt = DAG.getBitcast(MVT::v4f32, ZExt);
13402 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13412 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13417 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13422 V2, Mask, DAG))
13430 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13432 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13436 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask, DAG))
13440 return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13450 SelectionDAG &DAG) {
13459 Zeroable, Subtarget, DAG))
13468 Subtarget, DAG, /*BitwiseOnly*/ true))
13472 lowerShuffleAsBitRotate(DL, MVT::v4i32, V1, Mask, Subtarget, DAG))
13480 Mask, Subtarget, DAG))
13496 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13497 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13501 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13507 DAG, /*BitwiseOnly*/ false))
13513 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13521 Zeroable, Subtarget, DAG))
13525 Zeroable, Subtarget, DAG))
13529 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask, DAG))
13537 Zeroable, Subtarget, DAG))
13541 Subtarget, DAG))
13553 Zeroable, Subtarget, DAG);
13557 Mask, Subtarget, DAG))
13566 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13567 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13568 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13569 return DAG.getBitcast(MVT::v4i32, ShufPS);
13590 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
13601 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13602 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13608 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13609 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13634 V = DAG.getNode(ShufWOp, DL, VT, V,
13635 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13636 V = DAG.getBitcast(PSHUFDVT, V);
13637 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
13638 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
13639 return DAG.getBitcast(VT, V);
13768 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
13784 V = DAG.getNode(
13787 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13810 V = DAG.getBitcast(
13812 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13813 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13824 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
14046 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14047 getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
14049 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14050 getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
14052 V = DAG.getBitcast(
14054 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
14055 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
14066 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14067 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
14074 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14075 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
14084 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
14092 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14093 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14108 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
14109 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
14116 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
14117 DAG.getBuildVector(ShufVT, DL, V1Mask));
14119 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
14120 DAG.getBuildVector(ShufVT, DL, V2Mask));
14125 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
14130 return DAG.getBitcast(VT, V);
14148 SelectionDAG &DAG) {
14156 Zeroable, Subtarget, DAG))
14161 Subtarget, DAG))
14170 Subtarget, DAG, /*BitwiseOnly*/ false))
14175 Mask, Subtarget, DAG))
14180 Subtarget, DAG))
14184 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
14189 lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
14194 Subtarget, DAG))
14200 Subtarget, DAG);
14210 DAG, /*BitwiseOnly*/ false))
14216 Zeroable, DAG))
14222 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14230 Zeroable, Subtarget, DAG))
14234 Zeroable, Subtarget, DAG))
14238 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
14243 lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
14248 Subtarget, DAG))
14253 Subtarget, DAG))
14257 lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
14262 Zeroable, Subtarget, DAG))
14274 SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL);
14275 V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2,
14276 getZeroVector(MVT::v16i16, Subtarget, DAG, DL),
14277 DAG.getTargetConstant(0xEE, DL, MVT::i8));
14278 V1V2 = DAG.getBitcast(MVT::v8i32, V1V2);
14279 V1 = extract128BitVector(V1V2, 0, DAG, DL);
14280 V2 = extract128BitVector(V1V2, 4, DAG, DL);
14284 DAG.getConstant(0, DL, MVT::i32));
14286 DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
14288 DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
14289 V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),
14291 V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),
14295 SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8);
14296 V1 = DAG.getBitcast(MVT::v4i32, V1);
14297 V2 = DAG.getBitcast(MVT::v4i32, V2);
14298 V1 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V1, ShAmt);
14299 V2 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V2, ShAmt);
14300 V1 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V1, ShAmt);
14301 V2 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V2, ShAmt);
14306 SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2);
14308 Result = DAG.getBitcast(MVT::v4i32, Result);
14309 Result = DAG.getNode(PackOpc, DL, MVT::v8i16, Result, Result);
14319 V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
14320 DAG.getBitcast(MVT::v4i32, V1),
14321 DAG.getTargetConstant(16, DL, MVT::i8));
14322 V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
14323 DAG.getBitcast(MVT::v4i32, V2),
14324 DAG.getTargetConstant(16, DL, MVT::i8));
14325 return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL,
14331 Mask, Subtarget, DAG))
14339 Zeroable, DAG, V1InUse, V2InUse);
14345 Zeroable, Subtarget, DAG);
14352 SelectionDAG &DAG) {
14362 Mask, Subtarget, DAG))
14367 DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14371 V1 = DAG.getBitcast(MVT::v8i16, V1);
14372 V2 = DAG.getBitcast(MVT::v8i16, V2);
14373 return DAG.getBitcast(MVT::v8f16,
14374 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14383 SelectionDAG &DAG) {
14396 V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512);
14397 V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512);
14407 MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true);
14408 MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512);
14410 MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true);
14415 Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1);
14417 Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2);
14420 Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits());
14435 SelectionDAG &DAG) {
14443 DAG, /*BitwiseOnly*/ false))
14448 Subtarget, DAG))
14453 lowerShuffleWithPACK(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14458 Zeroable, Subtarget, DAG))
14463 Subtarget, DAG))
14467 Subtarget, DAG))
14473 Zeroable, DAG))
14482 Mask, Subtarget, DAG))
14487 Subtarget, DAG))
14490 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14552 V1 = DAG.getBitcast(
14554 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14555 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
14565 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14566 MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),
14567 OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));
14580 return DAG.getBitcast(
14582 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14583 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
14590 Zeroable, Subtarget, DAG))
14594 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14599 Zeroable, Subtarget, DAG))
14610 // want to preserve that and we can DAG combine any longer sequences into
14612 // the complexity of DAG combining bad patterns back into PSHUFB is too high,
14627 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14635 Zeroable, Subtarget, DAG))
14647 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14653 DAG);
14657 SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
14658 return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);
14664 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14674 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14677 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14695 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
14697 WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
14698 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
14699 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
14702 V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
14706 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14709 Result = DAG.getBitcast(MVT::v8i16, Result);
14710 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
14717 V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14718 DAG.getBitcast(MVT::v8i16, V1),
14719 DAG.getTargetConstant(8, DL, MVT::i8));
14721 V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14722 DAG.getBitcast(MVT::v8i16, V2),
14723 DAG.getTargetConstant(8, DL, MVT::i8));
14724 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14731 Zeroable, Subtarget, DAG);
14751 VLoHalf = DAG.getBitcast(MVT::v8i16, V);
14752 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
14753 DAG.getConstant(0x00FF, DL, MVT::v8i16));
14756 VHiHalf = DAG.getUNDEF(MVT::v8i16);
14768 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14770 VLoHalf = DAG.getBitcast(
14771 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
14772 VHiHalf = DAG.getBitcast(
14773 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
14776 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14777 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14779 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
14790 SelectionDAG &DAG) {
14792 V1 = DAG.getBitcast(MVT::v8i16, V1);
14793 V2 = DAG.getBitcast(MVT::v8i16, V2);
14794 return DAG.getBitcast(VT,
14795 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14800 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14802 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14804 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14806 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14808 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14810 return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14812 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14826 SelectionDAG &DAG, bool SimpleOnly) {
14844 std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL);
14845 return std::make_pair(DAG.getBitcast(SplitVT, LoV),
14846 DAG.getBitcast(SplitVT, HiV));
14909 return DAG.getUNDEF(SplitVT);
14911 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14913 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14917 V1Blend = DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14926 V2Blend = DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14934 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
14942 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
14957 SelectionDAG &DAG) {
14983 Subtarget, DAG);
14997 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
15003 Subtarget, DAG);
15011 SelectionDAG &DAG) {
15030 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
15031 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
15032 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS,
15033 getSHUFPDImmForMask(SHUFPDMask, DL, DAG));
15046 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
15129 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
15130 return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT),
15177 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
15188 return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG);
15221 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
15226 SDValue Flipped = DAG.getBitcast(PVT, V1);
15228 DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
15229 Flipped = DAG.getBitcast(VT, Flipped);
15230 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
15238 SelectionDAG &DAG) {
15249 VT, MemVT, Ld, Ofs, DAG))
15270 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
15271 DAG.getVectorIdxConstant(0, DL));
15272 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
15273 getZeroVector(VT, Subtarget, DAG, DL), LoV,
15274 DAG.getVectorIdxConstant(0, DL));
15283 Subtarget, DAG))
15299 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
15300 DAG.getVectorIdxConstant(0, DL));
15301 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
15302 DAG.getVectorIdxConstant(2, DL));
15311 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
15312 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15339 V1 = DAG.getUNDEF(VT);
15341 V2 = DAG.getUNDEF(VT);
15343 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
15344 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15356 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15481 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15498 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15517 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15581 SelectionDAG &DAG, bool UseConcat = false) {
15591 return DAG.getUNDEF(HalfVT);
15594 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15595 DAG.getVectorIdxConstant(HalfIdx, DL));
15601 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15604 SDValue Op1 = DAG.getUNDEF(HalfVT);
15607 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15611 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15612 DAG.getVectorIdxConstant(Offset, DL));
15621 SelectionDAG &DAG) {
15638 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15639 DAG.getVectorIdxConstant(HalfNumElts, DL));
15640 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15641 DAG.getVectorIdxConstant(0, DL));
15648 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15649 DAG.getVectorIdxConstant(0, DL));
15650 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15651 DAG.getVectorIdxConstant(HalfNumElts, DL));
15676 UndefLower, DAG);
15683 !is128BitUnpackShuffleMask(HalfMask, DAG) &&
15703 UndefLower, DAG);
15722 UndefLower, DAG);
15735 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15772 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15785 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
15905 DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15907 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
15982 SelectionDAG &DAG) {
15994 V1 = getZeroVector(VT, Subtarget, DAG, DL);
15996 V2 = getZeroVector(VT, Subtarget, DAG, DL);
15998 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
15999 DAG.getTargetConstant(Immediate, DL, MVT::i8));
16009 SelectionDAG &DAG) {
16020 V1 = DAG.getBitcast(MVT::v4i64, V1);
16021 V2 = DAG.getBitcast(MVT::v4i64, V2);
16023 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
16024 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
16028 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
16032 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
16033 DAG.getConstant(0, DL, MVT::v32i8), Unpack,
16034 DAG.getVectorIdxConstant(0, DL));
16056 SelectionDAG &DAG) {
16108 SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
16109 SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
16110 SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
16111 DAG.getTargetConstant(0x20, DL, MVT::i8));
16112 SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
16113 DAG.getTargetConstant(0x31, DL, MVT::i8));
16115 DAG.ReplaceAllUsesWith(SecondHalf, &Perm2);
16118 DAG.ReplaceAllUsesWith(FirstHalf, &Perm1);
16129 SelectionDAG &DAG) {
16135 Subtarget, DAG))
16141 Mask, Subtarget, DAG))
16146 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
16153 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
16154 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16159 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
16160 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
16165 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16170 Mask, DAG, Subtarget))
16175 DAG, Subtarget);
16179 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask, DAG))
16183 Zeroable, Subtarget, DAG))
16188 Zeroable, Subtarget, DAG))
16202 return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);
16208 Zeroable, Subtarget, DAG);
16213 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16222 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16228 Zeroable, Subtarget, DAG))
16235 Zeroable, Subtarget, DAG);
16239 Subtarget, DAG);
16249 SelectionDAG &DAG) {
16256 Subtarget, DAG))
16260 Zeroable, Subtarget, DAG))
16265 Subtarget, DAG))
16272 Subtarget, DAG, /*BitwiseOnly*/ true))
16282 return DAG.getBitcast(
16284 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
16285 DAG.getBitcast(MVT::v8i32, V1),
16286 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16291 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
16292 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
16298 DAG, /*BitwiseOnly*/ false))
16304 Zeroable, Subtarget, DAG))
16308 Zeroable, Subtarget, DAG))
16314 Subtarget, DAG))
16318 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask, DAG))
16328 Zeroable, Subtarget, DAG);
16333 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16338 lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG))
16347 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16352 Zeroable, Subtarget, DAG);
16362 SelectionDAG &DAG) {
16368 Zeroable, Subtarget, DAG))
16373 Subtarget, DAG))
16381 if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG,
16386 Zeroable, Subtarget, DAG))
16387 return DAG.getBitcast(MVT::v8f32, ZExt);
16398 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
16400 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
16403 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
16404 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16407 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask, DAG))
16412 return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
16418 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16425 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16426 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
16429 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16430 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16434 DAG, Subtarget);
16440 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16446 Zeroable, Subtarget, DAG))
16456 Mask, DAG))
16462 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
16464 Subtarget, DAG);
16470 Zeroable, Subtarget, DAG);
16474 Subtarget, DAG);
16484 SelectionDAG &DAG) {
16496 Zeroable, Subtarget, DAG))
16503 Mask, DAG))
16509 if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
16512 Subtarget, DAG);
16515 Zeroable, Subtarget, DAG))
16520 Subtarget, DAG))
16527 Subtarget, DAG, /*BitwiseOnly*/ true))
16531 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16544 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16545 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16548 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask, DAG))
16555 DAG, /*BitwiseOnly*/ false))
16560 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16566 Zeroable, Subtarget, DAG))
16570 Zeroable, Subtarget, DAG))
16576 Subtarget, DAG))
16582 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16588 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask, DAG))
16593 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16594 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16601 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16602 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16604 CastV1, CastV2, DAG);
16605 return DAG.getBitcast(MVT::v8i32, ShufPS);
16611 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16616 Zeroable, Subtarget, DAG);
16626 SelectionDAG &DAG) {
16636 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16641 Subtarget, DAG))
16645 Zeroable, Subtarget, DAG))
16649 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask, DAG))
16654 lowerShuffleWithPACK(DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16659 Subtarget, DAG))
16665 Subtarget, DAG, /*BitwiseOnly*/ false))
16670 Subtarget, DAG))
16676 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16682 lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))
16687 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
16694 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16698 DAG, Subtarget);
16707 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16712 Zeroable, Subtarget, DAG))
16717 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG);
16722 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16727 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16734 Mask, DAG))
16739 Subtarget, DAG);
16749 SelectionDAG &DAG) {
16759 Zeroable, Subtarget, DAG))
16764 Subtarget, DAG))
16768 Zeroable, Subtarget, DAG))
16772 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask, DAG))
16777 lowerShuffleWithPACK(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16782 Subtarget, DAG))
16788 DAG, /*BitwiseOnly*/ false))
16793 Subtarget, DAG))
16799 lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))
16805 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16813 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
16817 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16821 DAG, Subtarget);
16825 Zeroable, Subtarget, DAG))
16830 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG);
16835 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16840 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16848 Mask, Zeroable, DAG))
16855 Mask, DAG))
16860 Subtarget, DAG);
16871 SelectionDAG &DAG) {
16879 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16884 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16899 Subtarget, DAG))
16901 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16903 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
16908 V1 = DAG.getBitcast(FpVT, V1);
16909 V2 = DAG.getBitcast(FpVT, V2);
16910 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16914 V1 = DAG.getBitcast(MVT::v16i16, V1);
16915 V2 = DAG.getBitcast(MVT::v16i16, V2);
16916 return DAG.getBitcast(VT,
16917 DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask));
16922 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16924 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16926 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16928 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16930 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16932 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16943 SelectionDAG &DAG) {
16962 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16963 DAG.getVectorIdxConstant(0, DL));
16964 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
16965 getZeroVector(VT, Subtarget, DAG, DL), LoV,
16966 DAG.getVectorIdxConstant(0, DL));
16976 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
16977 DAG.getVectorIdxConstant(0, DL));
16978 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
16979 DAG.getVectorIdxConstant(4, DL));
17007 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
17008 DAG.getVectorIdxConstant(0, DL));
17009 return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
17023 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
17041 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
17042 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
17049 SelectionDAG &DAG) {
17057 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
17066 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
17067 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
17072 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
17073 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17077 V2, Subtarget, DAG))
17080 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask, DAG))
17085 Zeroable, Subtarget, DAG))
17089 Subtarget, DAG))
17093 Zeroable, Subtarget, DAG))
17096 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);
17103 SelectionDAG &DAG) {
17116 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
17118 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
17121 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
17122 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17125 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask, DAG))
17129 Zeroable, Subtarget, DAG))
17133 return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
17137 Zeroable, Subtarget, DAG))
17141 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
17142 return DAG.getBitcast(MVT::v16f32, ZExt);
17147 DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
17154 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
17155 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
17160 Zeroable, Subtarget, DAG))
17163 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);
17170 SelectionDAG &DAG) {
17179 Subtarget, DAG, /*BitwiseOnly*/ true))
17190 return DAG.getBitcast(
17192 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
17193 DAG.getBitcast(MVT::v16i32, V1),
17194 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
17199 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
17200 getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
17204 V2, Subtarget, DAG))
17210 DAG, /*BitwiseOnly*/ false))
17215 Zeroable, Subtarget, DAG))
17221 Subtarget, DAG))
17224 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask, DAG))
17229 Subtarget, DAG))
17233 Zeroable, Subtarget, DAG))
17236 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG);
17243 SelectionDAG &DAG) {
17254 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
17261 Subtarget, DAG, /*BitwiseOnly*/ true))
17265 Subtarget, DAG))
17278 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
17279 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17282 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask, DAG))
17289 Subtarget, DAG, /*BitwiseOnly*/ false))
17294 lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask, Subtarget, DAG))
17299 Zeroable, Subtarget, DAG))
17305 Subtarget, DAG))
17311 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
17312 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
17314 CastV1, CastV2, DAG);
17315 return DAG.getBitcast(MVT::v16i32, ShufPS);
17321 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
17326 Zeroable, Subtarget, DAG))
17330 Zeroable, Subtarget, DAG))
17333 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG);
17340 SelectionDAG &DAG) {
17350 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
17354 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask, DAG))
17359 lowerShuffleWithPACK(DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
17365 Subtarget, DAG, /*BitwiseOnly*/ false))
17370 Subtarget, DAG))
17376 lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))
17385 RepeatedMask, Subtarget, DAG);
17390 Zeroable, Subtarget, DAG))
17394 Zeroable, Subtarget, DAG))
17401 DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
17404 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);
17411 SelectionDAG &DAG) {
17421 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
17425 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask, DAG))
17430 lowerShuffleWithPACK(DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17436 DAG, /*BitwiseOnly*/ false))
17441 Subtarget, DAG))
17447 lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
17452 Zeroable, Subtarget, DAG))
17456 Zeroable, Subtarget, DAG))
17462 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17466 DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget))
17470 Zeroable, Subtarget, DAG))
17477 Mask, Subtarget, DAG))
17484 DAG, V1InUse, V2InUse);
17491 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17496 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG);
17498 return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17510 SelectionDAG &DAG) {
17521 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17526 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
17531 Subtarget, DAG))
17538 Subtarget, DAG))
17540 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17543 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17548 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
17551 V1 = DAG.getBitcast(MVT::v32i16, V1);
17552 V2 = DAG.getBitcast(MVT::v32i16, V2);
17553 return DAG.getBitcast(VT,
17554 DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask));
17563 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17565 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17567 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17569 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17571 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17573 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17583 SelectionDAG &DAG) {
17611 SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL);
17612 Res = DAG.getNode(X86ISD::KSHIFTR, DL, Res.getValueType(), Res,
17613 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17614 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17615 DAG.getVectorIdxConstant(0, DL));
17659 SelectionDAG &DAG) {
17692 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, Src == 0 ? V1 : V2,
17693 DAG.getVectorIdxConstant(0, DL));
17694 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17695 DAG.getConstant(0, DL, VT), Extract,
17696 DAG.getVectorIdxConstant(0, DL));
17701 lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget, DAG))
17710 SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL);
17717 DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17718 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17723 Res = DAG.getNode(Opcode, DL, WideVT, Res,
17724 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17725 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17726 DAG.getVectorIdxConstant(0, DL));
17740 return DAG.getSetCC(
17741 DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask),
17742 DAG.getVectorShuffle(OpVT, DL, Op1, DAG.getUNDEF(OpVT), Mask), CC);
17780 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17781 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17783 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17788 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17791 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17913 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
17924 SelectionDAG &DAG) {
17940 return DAG.getUNDEF(VT);
17946 return DAG.getCommutedVectorShuffle(*SVOp);
17957 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17975 return getZeroVector(VT, Subtarget, DAG, DL);
17993 Subtarget, DAG))
18003 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
18018 V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
18020 V1 = DAG.getBitcast(NewVT, V1);
18021 V2 = DAG.getBitcast(NewVT, V2);
18022 return DAG.getBitcast(
18023 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
18033 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget))
18034 return DAG.getBitcast(VT, HOp);
18036 V1 = DAG.getBitcast(VT, Ops[0]);
18037 V2 = DAG.getBitcast(VT, Ops[1]);
18050 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18053 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18056 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18059 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18067 SelectionDAG &DAG) {
18094 DAG, DL);
18096 Subtarget, DAG, DL);
18097 Passthru = Passthru.isUndef() ? DAG.getUNDEF(LargeVecVT)
18100 Subtarget, DAG, DL);
18103 DAG.getNode(ISD::VECTOR_COMPRESS, DL, LargeVecVT, Vec, Mask, Passthru);
18104 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Compressed,
18105 DAG.getConstant(0, DL, MVT::i64));
18113 Vec = DAG.getNode(ISD::ANY_EXTEND, DL, LargeVecVT, Vec);
18115 ? DAG.getUNDEF(LargeVecVT)
18116 : DAG.getNode(ISD::ANY_EXTEND, DL, LargeVecVT, Passthru);
18119 DAG.getNode(ISD::VECTOR_COMPRESS, DL, LargeVecVT, Vec, Mask, Passthru);
18120 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Compressed);
18129 SelectionDAG &DAG) {
18140 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);
18146 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
18155 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond,
18156 DAG.getBitcast(NVT, LHS),
18157 DAG.getBitcast(NVT, RHS)));
18169 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
18196 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
18197 DAG.getConstant(0, dl, CondVT),
18200 return DAG.getSelect(dl, VT, Mask, LHS, RHS);
18206 if (CondEltSize != DAG.ComputeNumSignBits(Cond))
18211 Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT);
18212 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);
18222 bool FreeCond = isFreeToSplitVector(Cond.getNode(), DAG);
18223 bool FreeLHS = isFreeToSplitVector(LHS.getNode(), DAG) ||
18225 bool FreeRHS = isFreeToSplitVector(RHS.getNode(), DAG) ||
18228 return splitVectorOp(Op, DAG, dl);
18250 Cond = DAG.getBitcast(CastVT, Cond);
18251 LHS = DAG.getBitcast(CastVT, LHS);
18252 RHS = DAG.getBitcast(CastVT, RHS);
18253 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
18254 return DAG.getBitcast(VT, Select);
18259 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
18274 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
18275 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18276 DAG.getBitcast(MVT::v4i32, Vec), Idx));
18279 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,
18280 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18281 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
18297 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18298 DAG.getBitcast(MVT::v4i32, Vec), Idx);
18299 return DAG.getBitcast(MVT::f32, Extract);
18310 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
18329 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18331 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
18335 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
18336 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
18337 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
18345 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18348 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
18349 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18351 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
18352 DAG.getVectorIdxConstant(0, dl));
18391 SelectionDAG &DAG) const {
18399 return ExtractBitFromMaskVector(Op, DAG, Subtarget);
18441 Vec = extract128BitVector(Vec, IdxVal, DAG, dl);
18450 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
18451 DAG.getVectorIdxConstant(IdxVal, dl));
18466 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
18467 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18468 DAG.getBitcast(MVT::v4i32, Vec), Idx));
18471 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
18472 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18473 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
18477 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
18490 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18491 DAG.getBitcast(MVT::v4i32, Vec),
18492 DAG.getVectorIdxConstant(DWordIdx, dl));
18495 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
18496 DAG.getConstant(ShiftVal, dl, MVT::i8));
18497 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
18502 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
18503 DAG.getBitcast(MVT::v8i16, Vec),
18504 DAG.getVectorIdxConstant(WordIdx, dl));
18507 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
18508 DAG.getConstant(ShiftVal, dl, MVT::i8));
18509 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
18520 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18521 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18522 DAG.getVectorIdxConstant(0, dl));
18536 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18537 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18538 DAG.getVectorIdxConstant(0, dl));
18546 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18560 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18561 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
18562 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
18563 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
18567 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18568 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx);
18572 SelectionDAG &DAG) const {
18579 return InsertBitToMaskVector(Op, DAG, Subtarget);
18589 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
18590 DAG.getBitcast(IVT, N0),
18591 DAG.getBitcast(MVT::i16, N1), N2);
18592 return DAG.getBitcast(VT, Res);
18609 SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT);
18610 SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt);
18611 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1);
18615 RawIndices.push_back(DAG.getConstant(I, dl, IdxSVT));
18616 SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices);
18619 return DAG.getSelectCC(dl, IdxSplat, Indices, EltSplat, N0,
18636 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType());
18637 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType());
18640 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts);
18641 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector);
18650 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18651 : getOnesVector(VT, DAG, dl);
18652 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);
18667 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18668 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
18669 DAG.getTargetConstant(1, dl, MVT::i8));
18684 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1);
18688 return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask);
18692 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18698 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
18699 DAG.getVectorIdxConstant(IdxIn128, dl));
18702 return insert128BitVector(N0, V, IdxVal, DAG, dl);
18710 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18711 return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18717 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1);
18719 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1);
18720 N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18721 return DAG.getBitcast(VT, N1);
18739 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
18740 N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8);
18741 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
18747 // zero here. The DAG Combiner may combine an extract_elt index into
18752 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
18755 bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize();
18764 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18765 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
18766 DAG.getTargetConstant(1, dl, MVT::i8));
18769 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18770 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
18771 DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
18783 SelectionDAG &DAG) {
18790 return getZeroVector(OpVT, Subtarget, DAG, dl);
18800 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
18803 return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
18813 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18814 return DAG.getBitcast(
18815 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
18822 SelectionDAG &DAG) {
18825 return insert1BitVector(Op, DAG, Subtarget);
18829 SelectionDAG &DAG) {
18841 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18844 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
18845 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18847 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
18848 DAG.getVectorIdxConstant(0, dl));
18878 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18885 auto PtrVT = getPointerTy(DAG.getDataLayout());
18886 SDValue Result = DAG.getTargetConstantPool(
18890 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18894 DAG.getNode(ISD::ADD, DL, PtrVT,
18895 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18901 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18908 auto PtrVT = getPointerTy(DAG.getDataLayout());
18909 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18912 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18917 DAG.getNode(ISD::ADD, DL, PtrVT,
18918 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18924 SelectionDAG &DAG) const {
18925 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18929 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18936 auto PtrVT = getPointerTy(DAG.getDataLayout());
18937 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18939 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
18943 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18944 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18952 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18968 const Module &Mod = *DAG.getMachineFunction().getFunction().getParent();
18977 CodeModel::Model M = DAG.getTarget().getCodeModel();
18978 auto PtrVT = getPointerTy(DAG.getDataLayout());
18992 Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags);
18995 Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags);
19003 Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
19007 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
19008 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
19014 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
19015 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
19020 Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
19021 DAG.getSignedConstant(Offset, dl, PtrVT));
19027 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
19028 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
19031 static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA,
19036 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
19037 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
19040 bool UseTLSDESC = DAG.getTarget().useTLSDESC();
19041 SDValue Chain = DAG.getEntryNode();
19044 TGA = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, OperandFlags);
19050 "Unexpected TLSDESC DAG");
19054 "Unexpected TLSDESC DAG");
19058 "Unexpected TLSDESC DAG");
19062 TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
19071 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
19074 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX,
19075 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT),
19078 Chain = DAG.getNode(CallType, dl, NodeTys, {Chain, TGA, InGlue});
19080 Chain = DAG.getNode(CallType, dl, NodeTys, {Chain, TGA});
19082 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), dl);
19088 Ret = DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
19094 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
19097 Value *Ptr = Constant::getNullValue(PointerType::get(*DAG.getContext(), Seg));
19099 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
19101 return DAG.getNode(ISD::ADD, dl, PtrVT, Ret, Offset);
19106 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
19108 return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD,
19114 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
19116 return GetTLSADDR(DAG, GA, PtrVT, X86::RAX, X86II::MO_TLSGD);
19121 LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
19123 return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD);
19127 SelectionDAG &DAG, const EVT PtrVT,
19133 DAG.getMachineFunction().getInfo<X86MachineFunctionInfo>();
19139 Base = GetTLSADDR(DAG, GA, PtrVT, ReturnReg, X86II::MO_TLSLD,
19143 Base = GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSLDM,
19154 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
19157 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
19160 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
19164 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
19171 PointerType::get(*DAG.getContext(), is64Bit ? X86AS::FS : X86AS::GS));
19174 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
19198 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
19200 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
19204 Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
19205 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
19209 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
19210 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
19215 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
19219 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
19223 if (DAG.getTarget().useEmulatedTLS())
19224 return LowerToTLSEmulatedModel(GA, DAG);
19227 auto PtrVT = getPointerTy(DAG.getDataLayout());
19231 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
19236 return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
19237 return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
19239 return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
19241 return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
19245 return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
19267 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
19270 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
19274 Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
19275 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
19280 SDValue Chain = DAG.getEntryNode();
19281 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
19282 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
19284 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
19285 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
19288 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
19294 return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
19310 SDValue Chain = DAG.getEntryNode();
19316 Subtarget.is64Bit() ? PointerType::get(*DAG.getContext(), X86AS::GS)
19317 : PointerType::get(*DAG.getContext(), X86AS::FS));
19320 ? DAG.getIntPtrConstant(0x58, dl)
19322 ? DAG.getIntPtrConstant(0x2C, dl)
19323 : DAG.getExternalSymbol("_tls_array", PtrVT));
19326 DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr));
19333 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
19335 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
19338 IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo());
19340 const DataLayout &DL = DAG.getDataLayout();
19342 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
19343 IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
19345 res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
19348 res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
19351 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
19354 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
19358 return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
19391 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
19393 DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG);
19394 return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
19400 SelectionDAG &DAG,
19424 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
19426 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
19429 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19430 DAG.getVectorIdxConstant(0, dl));
19431 return DAG.getMergeValues({Value, Chain}, dl);
19434 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
19436 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19437 DAG.getVectorIdxConstant(0, dl));
19441 static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
19460 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
19462 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},
19465 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19466 DAG.getVectorIdxConstant(0, dl));
19467 return DAG.getMergeValues({Value, Chain}, dl);
19470 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);
19472 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19473 DAG.getVectorIdxConstant(0, dl));
19502 SelectionDAG &DAG,
19526 VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
19531 VecOp = extract128BitVector(VecOp, 0, DAG, DL);
19535 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
19536 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
19537 DAG.getVectorIdxConstant(0, DL));
19544 SelectionDAG &DAG,
19583 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
19584 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19585 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19586 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19587 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx);
19591 SelectionDAG &DAG,
19611 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19612 : DAG.getUNDEF(MVT::v8i64);
19613 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
19614 DAG.getVectorIdxConstant(0, DL));
19617 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
19621 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
19624 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19625 DAG.getVectorIdxConstant(0, DL));
19628 return DAG.getMergeValues({Res, Chain}, DL);
19637 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19638 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
19639 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19640 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
19641 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
19642 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19643 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19647 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19648 DAG.getVectorIdxConstant(i, DL));
19651 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
19655 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);
19658 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19662 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19663 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
19667 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
19670 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
19671 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19674 return DAG.getMergeValues({Cvt, Chain}, DL);
19680 SelectionDAG &DAG) {
19683 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19687 SDValue Rnd = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
19689 return DAG.getNode(
19692 DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}),
19694 return DAG.getNode(ISD::FP_ROUND, dl, VT,
19695 DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd);
19719 SelectionDAG &DAG) const {
19723 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19729 return promoteXINT_TO_FP(Op, dl, DAG);
19734 return LowerWin64_INT128_TO_FP(Op, DAG);
19736 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
19739 if (SDValue R = lowerFPToIntToFP(Op, dl, DAG, Subtarget))
19747 return DAG.getNode(
19749 {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19750 DAG.getUNDEF(SrcVT))});
19751 return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
19752 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19753 DAG.getUNDEF(SrcVT)));
19756 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget);
19773 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
19775 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
19780 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19782 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
19785 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext);
19796 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19800 MachineFunction &MF = DAG.getMachineFunction();
19804 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19805 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19806 Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment);
19808 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG);
19811 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19818 MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const {
19823 Tys = DAG.getVTList(MVT::f80, MVT::Other);
19825 Tys = DAG.getVTList(DstVT, MVT::Other);
19829 DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
19834 MachineFunction &MF = DAG.getMachineFunction();
19839 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19840 Tys = DAG.getVTList(MVT::Other);
19842 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
19843 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
19847 DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
19848 Result = DAG.getLoad(
19850 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
19860 static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
19862 bool IsOptimizingSize = DAG.shouldOptForSize();
19869 SelectionDAG &DAG,
19888 LLVMContext *Context = DAG.getContext();
19893 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19894 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19904 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19908 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0));
19909 SDValue CLod0 = DAG.getLoad(
19910 MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
19911 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19913 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
19915 SDValue CLod1 = DAG.getLoad(
19917 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19918 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19920 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
19924 shouldUseHorizontalOp(true, DAG, Subtarget)) {
19925 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
19927 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19928 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
19930 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
19931 DAG.getVectorIdxConstant(0, dl));
19937 SelectionDAG &DAG,
19941 SDValue Bias = DAG.getConstantFP(
19946 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));
19949 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
19952 SDValue Or = DAG.getNode(
19954 DAG.getBitcast(MVT::v2i64, Load),
19955 DAG.getBitcast(MVT::v2i64,
19956 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
19957 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
19958 DAG.getBitcast(MVT::v2f64, Or),
19959 DAG.getVectorIdxConstant(0, dl));
19965 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19972 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19975 return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl);
19980 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19983 return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType());
19987 SelectionDAG &DAG,
20003 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
20004 DAG.getConstant(0, DL, MVT::v2i32));
20005 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
20008 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
20009 DAG.getVectorIdxConstant(0, DL));
20010 return DAG.getMergeValues({Res, Chain}, DL);
20014 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
20015 DAG.getUNDEF(MVT::v2i32));
20017 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
20019 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
20026 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
20027 SDValue VBias = DAG.getConstantFP(
20029 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
20030 DAG.getBitcast(MVT::v2i64, VBias));
20031 Or = DAG.getBitcast(MVT::v2f64, Or);
20034 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
20036 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
20040 SelectionDAG &DAG,
20064 IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
20065 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
20066 DAG.getVectorIdxConstant(0, DL));
20069 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
20073 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
20076 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
20077 DAG.getVectorIdxConstant(0, DL));
20080 return DAG.getMergeValues({Res, Chain}, DL);
20086 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
20088 *DAG.getContext(),
20090 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
20091 SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
20092 SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other);
20093 SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
20094 SDValue VBias = DAG.getMemIntrinsicNode(
20096 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8),
20099 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
20100 DAG.getBitcast(MVT::v4i64, VBias));
20101 Or = DAG.getBitcast(MVT::v4f64, Or);
20104 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
20106 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
20136 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
20138 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
20141 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
20142 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
20148 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
20149 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
20152 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
20153 VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
20156 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
20157 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
20160 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
20161 VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
20163 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
20165 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
20166 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
20169 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
20173 SDValue VecCstFSub = DAG.getConstantFP(
20180 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
20183 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
20186 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
20188 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
20193 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
20194 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
20197 static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
20207 return lowerUINT_TO_FP_v2i32(Op, dl, DAG, Subtarget);
20210 return lowerUINT_TO_FP_vXi32(Op, dl, DAG, Subtarget);
20213 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget);
20218 SelectionDAG &DAG) const {
20223 auto PtrVT = getPointerTy(DAG.getDataLayout());
20226 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
20233 return promoteXINT_TO_FP(Op, dl, DAG);
20238 return lowerUINT_TO_FP_vec(Op, dl, DAG, Subtarget);
20241 return LowerWin64_INT128_TO_FP(Op, DAG);
20243 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
20255 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);
20257 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},
20259 return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
20262 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
20264 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
20271 return LowerUINT_TO_FP_i64(Op, dl, DAG, Subtarget);
20276 return LowerUINT_TO_FP_i32(Op, dl, DAG, Subtarget);
20282 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
20286 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
20289 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
20290 SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign);
20291 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
20294 BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG);
20296 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
20307 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
20310 DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign);
20314 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
20317 DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,
20322 SDValue SignSet = DAG.getSetCC(
20323 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
20324 Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
20329 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF), PtrVT);
20333 SDValue Zero = DAG.getIntPtrConstant(0, dl);
20334 SDValue Four = DAG.getIntPtrConstant(4, dl);
20335 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
20336 FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
20339 SDValue Fudge = DAG.getExtLoad(
20341 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
20353 DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge});
20357 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},
20359 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});
20366 SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge);
20367 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
20368 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
20377 SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
20386 auto PtrVT = getPointerTy(DAG.getDataLayout());
20414 MachineFunction &MF = DAG.getMachineFunction();
20418 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
20420 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
20440 // for DAG type consistency we have to match the FP operand type.
20456 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
20458 EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
20459 *DAG.getContext(), TheVT);
20462 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
20466 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE);
20480 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);
20481 SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);
20482 Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63);
20484 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal,
20485 DAG.getConstantFP(0.0, DL, TheVT));
20488 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
20492 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
20501 Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
20502 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
20509 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
20517 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
20518 DAG.getVTList(MVT::Other),
20521 SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
20526 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
20531 static SDValue LowerAVXExtend(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
20552 unsigned ExtendInVecOpc = DAG.getOpcode_EXTEND_VECTOR_INREG(Opc);
20556 return splitVectorIntUnary(Op, DAG, dl);
20575 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20581 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo);
20583 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20584 SDValue Undef = DAG.getUNDEF(InVT);
20586 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20587 OpHi = DAG.getBitcast(HalfVT, OpHi);
20589 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
20594 const SDLoc &dl, SelectionDAG &DAG) {
20596 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20597 DAG.getVectorIdxConstant(0, dl));
20598 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20599 DAG.getVectorIdxConstant(8, dl));
20600 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);
20601 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);
20602 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20603 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20608 SelectionDAG &DAG) {
20618 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20619 return DAG.getNode(ISD::SRL, DL, VT, Extend,
20620 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
20628 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
20638 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT), In,
20639 DAG.getVectorIdxConstant(0, DL));
20643 SDValue One = DAG.getConstant(1, DL, WideVT);
20644 SDValue Zero = DAG.getConstant(0, DL, WideVT);
20646 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20651 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
20656 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
20657 DAG.getVectorIdxConstant(0, DL));
20663 SelectionDAG &DAG) {
20669 return LowerZERO_EXTEND_Mask(Op, DL, Subtarget, DAG);
20672 return LowerAVXExtend(Op, DL, DAG, Subtarget);
20681 const SDLoc &DL, SelectionDAG &DAG,
20706 LLVMContext &Ctx = *DAG.getContext();
20724 In = widenSubVector(In, false, Subtarget, DAG, DL, 128);
20725 SDValue LHS = DAG.getBitcast(InVT, In);
20726 SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS;
20727 SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS);
20728 Res = extractSubVector(Res, 0, DAG, DL, SrcSizeInBits / 2);
20729 Res = DAG.getBitcast(PackedVT, Res);
20730 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20735 std::tie(Lo, Hi) = splitVector(In, DAG, DL);
20741 truncateVectorWithPACK(Opcode, DstHalfVT, Lo, DL, DAG, Subtarget))
20742 return widenSubVector(Res, false, Subtarget, DAG, DL, DstSizeInBits);
20751 Lo = DAG.getBitcast(InVT, Lo);
20752 Hi = DAG.getBitcast(InVT, Hi);
20753 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20754 return DAG.getBitcast(DstVT, Res);
20760 Lo = DAG.getBitcast(InVT, Lo);
20761 Hi = DAG.getBitcast(InVT, Hi);
20762 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20770 Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
20773 return DAG.getBitcast(DstVT, Res);
20776 Res = DAG.getBitcast(PackedVT, Res);
20777 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20787 truncateVectorWithPACK(Opcode, PackedVT, In, DL, DAG, Subtarget);
20788 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20792 Lo = truncateVectorWithPACK(Opcode, HalfPackedVT, Lo, DL, DAG, Subtarget);
20793 Hi = truncateVectorWithPACK(Opcode, HalfPackedVT, Hi, DL, DAG, Subtarget);
20794 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20795 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20804 SelectionDAG &DAG) {
20805 In = DAG.getZeroExtendInReg(In, DL, DstVT);
20806 return truncateVectorWithPACK(X86ISD::PACKUS, DstVT, In, DL, DAG, Subtarget);
20812 SelectionDAG &DAG) {
20814 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SrcVT, In,
20815 DAG.getValueType(DstVT));
20816 return truncateVectorWithPACK(X86ISD::PACKSS, DstVT, In, DL, DAG, Subtarget);
20824 SelectionDAG &DAG,
20856 !isFreeToSplitVector(In.getNode(), DAG) &&
20857 (!Subtarget.hasAVX() || DAG.ComputeNumSignBits(In) != 64))
20871 KnownBits Known = DAG.computeKnownBits(In);
20881 unsigned NumSignBits = DAG.ComputeNumSignBits(In);
20901 if (std::optional<uint64_t> ShAmt = DAG.getValidShiftAmount(In)) {
20904 return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops());
20916 SelectionDAG &DAG, const SDNodeFlags Flags = SDNodeFlags()) {
20928 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20931 Subtarget, DAG))
20932 return widenSubVector(Res, false, Subtarget, DAG, DL,
20938 if (SDValue Src = matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG,
20940 return truncateVectorWithPACK(PackOpcode, DstVT, Src, DL, DAG, Subtarget);
20949 SelectionDAG &DAG) {
20971 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20973 if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG))
20974 return widenSubVector(Res, false, Subtarget, DAG, DL,
20983 return truncateVectorWithPACKUS(DstVT, In, DL, Subtarget, DAG);
20986 return truncateVectorWithPACKSS(DstVT, In, DL, Subtarget, DAG);
20991 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In);
20992 return truncateVectorWithPACKSS(DstVT, Trunc, DL, Subtarget, DAG);
20999 SelectionDAG &DAG,
21011 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
21015 In = DAG.getNode(ISD::SHL, DL, ExtVT,
21016 DAG.getBitcast(ExtVT, In),
21017 DAG.getConstant(ShiftInx, DL, ExtVT));
21018 In = DAG.getBitcast(InVT, In);
21020 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
21040 Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);
21041 Hi = DAG.getVectorShuffle(
21044 Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);
21047 Lo = extract128BitVector(In, 0, DAG, DL);
21048 Hi = extract128BitVector(In, 8, DAG, DL);
21052 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
21053 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
21054 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
21061 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
21066 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
21068 In = DAG.getNode(ISD::SHL, DL, InVT, In,
21069 DAG.getConstant(ShiftInx, DL, InVT));
21073 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
21074 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
21077 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
21086 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
21096 std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
21099 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
21101 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
21102 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
21103 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
21110 VT, In, DL, Subtarget, DAG, Op->getFlags()))
21115 return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG);
21122 return LowerTruncateVecI1(Op, DL, DAG, Subtarget);
21126 if (!Subtarget.hasAVX512() || isFreeToSplitVector(In.getNode(), DAG))
21128 VT, In, DL, Subtarget, DAG, Op->getFlags()))
21135 return splitVectorIntUnary(Op, DAG, DL);
21154 In = DAG.getBitcast(MVT::v8i32, In);
21155 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
21156 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
21157 DAG.getVectorIdxConstant(0, DL));
21160 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
21161 DAG.getVectorIdxConstant(0, DL));
21162 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
21163 DAG.getVectorIdxConstant(2, DL));
21165 return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo),
21166 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask);
21177 In = DAG.getBitcast(MVT::v32i8, In);
21178 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
21179 In = DAG.getBitcast(MVT::v4i64, In);
21182 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
21183 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
21184 DAG.getVectorIdxConstant(0, DL));
21185 return DAG.getBitcast(MVT::v8i16, In);
21189 ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG)
21190 : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG);
21194 return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG);
21202 SelectionDAG &DAG,
21210 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src);
21212 DAG.getNode(X86ISD::CVTTP2SI, dl, VT,
21213 DAG.getNode(ISD::FSUB, dl, SrcVT, Src,
21214 DAG.getConstantFP(2147483648.0f, dl, SrcVT)));
21226 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big);
21227 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small);
21231 DAG.getNode(X86ISD::VSRAI, dl, VT, Small,
21232 DAG.getTargetConstant(DstBits - 1, dl, MVT::i8));
21233 return DAG.getNode(ISD::OR, dl, VT, Small,
21234 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
21237 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
21251 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
21252 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
21254 return DAG.getNode(Op.getOpcode(), dl, VT,
21255 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
21279 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
21280 : DAG.getUNDEF(MVT::v8f64);
21281 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,
21282 DAG.getVectorIdxConstant(0, dl));
21285 Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src});
21288 Res = DAG.getNode(Opc, dl, ResVT, Src);
21291 Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res);
21292 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,
21293 DAG.getVectorIdxConstant(0, dl));
21295 return DAG.getMergeValues({Res, Chain}, dl);
21310 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
21313 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
21317 Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI
21322 Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl,
21329 Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res);
21333 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
21334 DAG.getVectorIdxConstant(0, dl));
21337 return DAG.getMergeValues({Res, Chain}, dl);
21348 Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT
21353 Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
21358 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
21361 return DAG.getMergeValues({Res, Chain}, dl);
21384 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
21385 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
21386 DAG.getVectorIdxConstant(0, dl));
21389 Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
21393 Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
21396 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
21397 DAG.getVectorIdxConstant(0, dl));
21400 return DAG.getMergeValues({Res, Chain}, dl);
21414 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
21415 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
21416 DAG.getVectorIdxConstant(0, dl));
21419 Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
21423 Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);
21426 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
21427 DAG.getVectorIdxConstant(0, dl));
21430 return DAG.getMergeValues({Res, Chain}, dl);
21441 SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
21442 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
21444 Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
21447 Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp,
21448 DAG.getVectorIdxConstant(0, dl));
21449 return DAG.getMergeValues({Tmp, Chain}, dl);
21453 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
21454 DAG.getUNDEF(MVT::v2f32));
21458 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
21461 return DAG.getNode(Opc, dl, VT, Tmp);
21470 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget);
21491 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT,
21492 DAG.getConstant(UIntLimit, dl, VT));
21499 DAG.getNode(X86ISD::CVTTS2SI, dl, VT,
21500 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src));
21501 SDValue Big = DAG.getNode(
21503 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT,
21504 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset)));
21512 SDValue IsOverflown = DAG.getNode(
21513 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));
21514 return DAG.getNode(ISD::OR, dl, VT, Small,
21515 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
21529 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},
21533 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
21535 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
21537 return DAG.getMergeValues({Res, Chain}, dl);
21553 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other},
21557 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
21559 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
21561 return DAG.getMergeValues({Res, Chain}, dl);
21579 makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
21582 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
21588 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
21590 return DAG.getMergeValues({V, Chain}, dl);
21598 SelectionDAG &DAG) const {
21613 return LRINT_LLRINTHelper(Op.getNode(), DAG);
21617 SelectionDAG &DAG) const {
21629 SDValue Chain = DAG.getEntryNode();
21636 SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
21639 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
21643 Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
21644 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
21647 Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
21654 Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
21658 return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
21662 X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
21731 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
21732 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
21739 SDValue MinClamped = DAG.getNode(
21742 SDValue BothClamped = DAG.getNode(
21745 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped);
21749 return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21753 SDValue MinClamped = DAG.getNode(
21756 SDValue BothClamped = DAG.getNode(
21759 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
21768 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21769 return DAG.getSelectCC(
21773 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
21774 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
21777 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src);
21782 FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21792 Select = DAG.getSelectCC(
21797 Select = DAG.getSelectCC(
21807 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21808 return DAG.getSelectCC(
21812 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21837 return DAG.getNode(
21839 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL,
21842 return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21843 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In));
21853 TargetLowering::CallLoweringInfo CLI(DAG);
21854 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21856 In = DAG.getBitcast(MVT::i16, In);
21860 Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext());
21865 SDValue Callee = DAG.getExternalSymbol(
21867 getPointerTy(DAG.getDataLayout()));
21869 CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee,
21875 Res = DAG.getMergeValues({Res, Chain}, DL);
21880 In = DAG.getBitcast(MVT::i16, In);
21881 In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,
21882 getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In,
21883 DAG.getVectorIdxConstant(0, DL));
21886 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other},
21890 Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,
21891 DAG.getTargetConstant(4, DL, MVT::i32));
21893 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res,
21894 DAG.getVectorIdxConstant(0, DL));
21896 return DAG.getMergeValues({Res, Chain}, DL);
21908 In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In,
21909 DAG.getUNDEF(MVT::v2f16));
21910 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,
21911 DAG.getUNDEF(MVT::v4f16));
21913 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21915 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21923 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
21925 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21927 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21930 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21948 TargetLowering::CallLoweringInfo CLI(DAG);
21949 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21954 Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext());
21959 SDValue Callee = DAG.getExternalSymbol(
21962 getPointerTy(DAG.getDataLayout()));
21964 CallingConv::C, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()), Callee,
21970 Res = DAG.getBitcast(MVT::f16, Res);
21973 Res = DAG.getMergeValues({Res, Chain}, DL);
21994 SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
21997 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32,
21998 DAG.getConstantFP(0, DL, MVT::v4f32), In,
21999 DAG.getVectorIdxConstant(0, DL));
22000 Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other},
22005 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In);
22006 Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd);
22009 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
22010 DAG.getVectorIdxConstant(0, DL));
22011 Res = DAG.getBitcast(MVT::f16, Res);
22014 return DAG.getMergeValues({Res, Chain}, DL);
22022 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
22029 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
22030 DAG.getConstant(0, dl, MVT::v8i16), Src,
22031 DAG.getVectorIdxConstant(0, dl));
22035 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},
22039 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
22042 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
22043 DAG.getVectorIdxConstant(0, dl));
22046 return DAG.getMergeValues({Res, Chain}, dl);
22051 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
22060 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,
22061 DAG.getConstantFP(0, dl, MVT::v4f32), Src,
22062 DAG.getVectorIdxConstant(0, dl));
22063 Res = DAG.getNode(
22065 {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});
22069 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);
22070 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
22071 DAG.getTargetConstant(4, dl, MVT::i32));
22074 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
22075 DAG.getVectorIdxConstant(0, dl));
22078 return DAG.getMergeValues({Res, Chain}, dl);
22084 SelectionDAG &DAG) const {
22091 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0));
22092 Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res);
22093 Res = DAG.getBitcast(MVT::v8i16, Res);
22094 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
22095 DAG.getVectorIdxConstant(0, DL));
22101 makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first;
22102 return DAG.getBitcast(MVT::i16, Res);
22108 SelectionDAG &DAG,
22129 !shouldUseHorizontalOp(true, DAG, Subtarget))
22166 X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
22174 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
22175 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
22176 DAG.getVectorIdxConstant(LExtIndex / 2, DL));
22181 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
22184 return lowerAddSubToHorizontalOp(Op, SDLoc(Op), DAG, Subtarget);
22191 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
22203 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
22204 DAG.getConstantFP(Point5Pred, dl, VT), N0);
22205 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder);
22208 return DAG.getNode(ISD::FTRUNC, dl, VT, N0);
22213 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
22231 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
22254 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
22264 return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
22268 Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
22269 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
22270 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
22271 DAG.getVectorIdxConstant(0, dl));
22274 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
22282 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);
22286 Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign,
22287 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
22293 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
22312 SDValue SignMask = DAG.getConstantFP(
22314 SDValue MagMask = DAG.getConstantFP(
22319 Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign);
22320 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
22329 MagBits = DAG.getConstantFP(APF, dl, LogicVT);
22333 Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag);
22334 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
22338 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
22340 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,
22341 DAG.getVectorIdxConstant(0, dl));
22344 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
22355 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
22356 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
22357 Res = DAG.getZExtOrTrunc(Res, dl, VT);
22358 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));
22363 static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
22370 Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
22373 if (!DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
22381 DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
22382 Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src);
22391 BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(),
22392 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
22394 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
22397 BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo);
22400 return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo);
22405 SelectionDAG &DAG) {
22406 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
22407 DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
22424 static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG,
22429 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22430 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22432 return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
22434 return DAG.getNode(ISD::OR, DL, VecVT, A, B);
22435 return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
22441 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
22443 return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
22444 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
22454 SelectionDAG &DAG,
22487 DAG.getMachineFunction().getFunction().hasFnAttribute(
22543 X = DAG.getBitcast(TmpCastVT, X);
22546 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
22547 DAG.getConstant(0, DL, VecVT), X,
22548 DAG.getVectorIdxConstant(0, DL));
22557 Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
22562 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
22564 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
22566 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
22574 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
22575 DAG.getConstant(0, DL, KRegVT), CC);
22579 DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64, Cmp);
22580 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
22582 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
22583 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0));
22590 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
22591 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
22592 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
22674 SelectionDAG &DAG, X86::CondCode &X86CC) {
22699 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
22700 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
22705 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
22706 if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) {
22709 auto SplitLHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(LHS)), DL,
22711 auto SplitRHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(RHS)), DL,
22714 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.first, SplitRHS.first);
22716 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.second, SplitRHS.second);
22717 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22718 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi),
22719 DAG.getConstant(0, DL, MVT::i32));
22721 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22722 DAG.getBitcast(IntVT, MaskBits(LHS)),
22723 DAG.getBitcast(IntVT, MaskBits(RHS)));
22741 VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64);
22742 LHS = DAG.getBitcast(VT, LHS);
22743 RHS = DAG.getBitcast(VT, RHS);
22748 KnownBits KnownRHS = DAG.computeKnownBits(RHS);
22752 auto Split = DAG.SplitVector(LHS, DL);
22754 LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22756 RHS = DAG.getAllOnesConstant(DL, VT);
22762 LHS = DAG.getBitcast(VT, MaskBits(LHS));
22763 RHS = DAG.getBitcast(VT, MaskBits(RHS));
22765 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ);
22766 V = DAG.getSExtOrTrunc(V, DL, VT);
22768 auto Split = DAG.SplitVector(V, DL);
22770 V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22772 V = DAG.getNOT(DL, V, VT);
22773 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22774 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22775 DAG.getConstant(0, DL, MVT::i32));
22778 SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
22780 auto Split = DAG.SplitVector(V, DL);
22782 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second);
22785 RHS = DAG.getConstant(0, DL, VT);
22792 LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22793 RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22794 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE);
22795 return DAG.getNode(X86ISD::KORTEST, DL, MVT::i32, V, V);
22800 LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22801 RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22802 SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS);
22803 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
22808 LHS = DAG.getBitcast(MaskVT, MaskBits(LHS));
22809 RHS = DAG.getBitcast(MaskVT, MaskBits(RHS));
22810 SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS);
22811 V = DAG.getNOT(DL, V, MaskVT);
22812 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22813 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22814 DAG.getConstant(0, DL, MVT::i32));
22822 SelectionDAG &DAG,
22881 VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS));
22885 CmpNull ? DAG.getConstant(0, DL, VT)
22886 : DAG.getAllOnesConstant(DL, VT),
22887 CC, Mask, Subtarget, DAG, X86CC);
22895 DAG.matchBinOpReduction(Op.getNode(), BinOp, {LogicOp})) {
22898 CmpNull ? DAG.getConstant(0, DL, MatchVT)
22899 : DAG.getAllOnesConstant(DL, MatchVT),
22900 CC, Mask, Subtarget, DAG, X86CC);
22920 return LowerVectorAllEqual(DL, LHS, RHS, CC, SrcMask, Subtarget, DAG,
22935 DAG.getConstant(Cmp, DL, InnerVT), CC,
22936 SrcMask, Subtarget, DAG, X86CC);
22979 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
23016 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
23017 DAG.getConstant(0, dl, Op.getValueType()));
23065 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
23066 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
23075 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
23076 DAG.getConstant(0, dl, Op.getValueType()));
23078 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
23081 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
23082 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
23089 const SDLoc &dl, SelectionDAG &DAG,
23092 return EmitTest(Op0, X86CC, dl, DAG, Subtarget);
23105 !DAG.getMachineFunction().getFunction().hasMinSize()) {
23117 if (DAG.ComputeMaxSignificantBits(Op0.getOperand(0)) <= 16)
23120 if (DAG.ComputeMaxSignificantBits(Op1.getOperand(0)) <= 16)
23126 Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0);
23127 Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1);
23135 DAG.MaskedValueIsZero(Op1, APInt::getHighBitsSet(64, 32)) &&
23136 DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) {
23138 Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0);
23139 Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1);
23146 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
23147 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
23155 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
23156 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
23161 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
23162 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
23190 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
23198 if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
23209 SelectionDAG &DAG, int Enabled,
23235 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
23237 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate);
23248 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
23249 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
23250 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
23251 Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);
23252 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
23255 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op);
23262 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
23290 return DAG.getNode(Opcode, DL, VT, Op);
23299 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
23300 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
23301 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
23302 Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);
23303 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
23306 return DAG.getNode(X86ISD::RCP14, DL, VT, Op);
23323 SelectionDAG &DAG,
23325 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
23349 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
23355 SelectionDAG &DAG, X86::CondCode &X86CC) {
23374 KnownBits Known = DAG.computeKnownBits(Op0);
23392 bool OptForSize = DAG.shouldOptForSize();
23396 BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
23413 if (SDValue BT = getBT(Src, BitNo, dl, DAG)) {
23492 SelectionDAG &DAG, const SDLoc &dl) {
23495 SDValue CC = DAG.getCondCode(Cond);
23499 std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl);
23503 std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl);
23507 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
23508 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
23509 DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC),
23510 DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC));
23514 SelectionDAG &DAG) {
23530 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);
23537 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc,
23561 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
23564 return DAG.getBuildVector(VT, DL, NewVecC);
23574 SelectionDAG &DAG) {
23594 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false);
23606 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false);
23621 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
23622 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
23623 DAG.getConstant(0, dl, VT));
23627 SelectionDAG &DAG) {
23650 return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23654 return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23658 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
23660 Op1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
23662 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
23668 SDValue Cmp = DAG.getNode(Op.getOpcode(), dl, DVT,
23669 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op0),
23670 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op1), CC);
23671 return DVT == VT ? Cmp : DAG.getNode(ISD::TRUNCATE, dl, VT, Cmp);
23714 SDValue SignalCmp = DAG.getNode(
23716 {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS
23744 Cmp0 = DAG.getNode(
23746 {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});
23747 Cmp1 = DAG.getNode(
23749 {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
23750 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
23753 Cmp0 = DAG.getNode(
23754 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
23755 Cmp1 = DAG.getNode(
23756 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
23758 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
23761 Cmp = DAG.getNode(
23763 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23766 Cmp = DAG.getNode(
23767 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23774 Cmp = DAG.getNode(
23776 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23779 Cmp = DAG.getNode(
23780 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23788 Cmp = DAG.getBitcast(CastVT, Cmp);
23789 Cmp = DAG.getSetCC(dl, Op.getSimpleValueType(), Cmp,
23790 DAG.getConstant(0, dl, CastVT), ISD::SETNE);
23795 Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
23799 return DAG.getMergeValues({Cmp, Chain}, dl);
23823 return LowerIntVSETCC_AVX512(Op, dl, DAG);
23850 return DAG.getNode(Opc, dl, VT, Op0, Op1,
23851 DAG.getTargetConstant(CmpMode, dl, MVT::i8));
23862 Op1 = DAG.getBitcast(VT, BC0.getOperand(1));
23875 Result = DAG.getNode(ISD::SHL, dl, VT, Result,
23876 DAG.getConstant(ShiftAmt, dl, VT));
23877 Result = DAG.getNode(ISD::SRA, dl, VT, Result,
23878 DAG.getConstant(BitWidth - 1, dl, VT));
23885 return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23890 return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23904 else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0))
23913 !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1));
23916 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23925 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false)) {
23933 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false)) {
23950 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23951 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
23955 Result = DAG.getNOT(dl, Result, VT);
23963 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
23988 Op0 = DAG.getConstant(0, dl, MVT::v4i32);
23989 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23991 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23993 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23995 return DAG.getBitcast(VT, Result);
23999 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
24000 Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
24002 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24004 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
24006 return DAG.getBitcast(VT, Result);
24011 if (!FlipSigns && !Invert && DAG.ComputeNumSignBits(Op0) > 32 &&
24012 DAG.ComputeNumSignBits(Op1) > 32) {
24013 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
24014 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
24016 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24018 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
24020 return DAG.getBitcast(VT, Result);
24026 SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL
24030 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);
24031 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);
24034 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
24035 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
24038 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
24039 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
24044 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
24045 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
24046 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
24048 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
24049 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
24052 Result = DAG.getNOT(dl, Result, MVT::v4i32);
24054 return DAG.getBitcast(VT, Result);
24063 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
24064 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
24067 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
24071 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
24072 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
24075 Result = DAG.getNOT(dl, Result, MVT::v4i32);
24077 return DAG.getBitcast(VT, Result);
24085 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
24087 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
24088 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
24091 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
24095 Result = DAG.getNOT(dl, Result, VT);
24102 const SDLoc &dl, SelectionDAG &DAG,
24138 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24139 return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);
24150 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24151 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
24158 SelectionDAG &DAG,
24169 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) {
24170 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
24176 if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG,
24178 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
24183 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
24198 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
24210 SDVTList CmpVTs = DAG.getVTList(VT, MVT::i32);
24212 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
24213 SDValue Neg = DAG.getNode(X86ISD::SUB, dl, CmpVTs,
24214 DAG.getConstant(0, dl, VT), Op0);
24224 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
24226 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
24228 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
24230 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
24237 TranslateX86CC(CC, dl, /*IsFP*/ false, Op0, Op1, DAG);
24240 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
24241 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
24245 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
24251 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
24267 softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,
24275 return DAG.getMergeValues({Op0, Chain}, dl);
24300 Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType());
24309 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
24310 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
24311 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
24320 NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1), dl, DAG);
24324 X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
24332 DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP,
24336 EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1);
24339 SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
24340 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
24341 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
24344 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
24356 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
24357 Carry, DAG.getAllOnesConstant(DL, CarryVT));
24359 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
24360 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
24361 return getSETCC(CC, Cmp.getValue(1), DL, DAG);
24369 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
24406 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
24407 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
24414 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
24422 std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
24424 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
24426 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
24444 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
24451 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
24457 SelectionDAG &DAG,
24471 Neg = DAG.getNode(ISD::TRUNCATE, DL, SplatVT, CmpVal);
24473 Neg = DAG.getNode(
24475 DAG.getNode(ISD::ANY_EXTEND, DL, SplatVT, CmpVal.getOperand(0)),
24476 DAG.getConstant(1, DL, SplatVT));
24477 return DAG.getNegative(Neg, DL, SplatVT); // -(and (x, 0x1))
24488 SDValue Diff = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
24489 SDValue Flip = DAG.getNode(ISD::AND, DL, VT, Mask, Diff);
24490 return DAG.getNode(ISD::XOR, DL, VT, LHS, Flip);
24546 SDValue And = DAG.getNode(ISD::AND, DL, Src1.getValueType(), Mask,
24548 return DAG.getNode(RHS.getOpcode(), DL, VT, Src2, And); // y Op And
24553 SDValue Or = DAG.getNode(ISD::OR, DL, VT, Mask, Src1); // Mask | z
24554 return DAG.getNode(LHS.getOpcode(), DL, VT, Src2, Or); // y Op Or
24561 SDVTList CmpVTs = DAG.getVTList(CmpVT, MVT::i32);
24572 SDValue Zero = DAG.getConstant(0, DL, CmpVT);
24573 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpVal);
24575 SDValue One = DAG.getConstant(1, DL, CmpVT);
24576 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpVal, One);
24578 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
24579 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
24581 return DAG.getNode(ISD::OR, DL, VT, SBB, Y);
24587 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
24598 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond,
24599 DAG.getBitcast(NVT, Op1),
24600 DAG.getBitcast(NVT, Op2)));
24616 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
24617 DAG.getTargetConstant(SSECC, DL, MVT::i8));
24619 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
24623 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
24624 DAG.getTargetConstant(SSECC, DL, MVT::i8));
24643 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
24644 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
24645 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
24648 VCmp = DAG.getBitcast(VCmpVT, VCmp);
24650 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
24652 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VSel,
24653 DAG.getVectorIdxConstant(0, DL));
24655 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
24656 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
24657 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
24663 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
24664 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
24669 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
24708 DL, DAG, Subtarget)) {
24721 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT);
24722 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt);
24724 Shift = DAG.getNOT(DL, Shift, VT);
24725 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1);
24738 if (SDValue R = LowerSELECTWithCmpZero(Cond, Op2, Op1, X86::COND_E, DL, DAG,
24765 std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24767 CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
24773 if (isTruncWithZeroHighBitsInput(Cond, DAG))
24780 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
24781 CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
24789 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
24790 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
24804 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
24805 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
24807 return DAG.getNOT(DL, Res, Res.getValueType());
24821 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
24823 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24836 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
24837 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
24839 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
24840 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24846 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
24851 SelectionDAG &DAG) {
24864 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);
24874 In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT), In,
24875 DAG.getVectorIdxConstant(0, dl));
24883 V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
24885 SDValue NegOne = DAG.getAllOnesConstant(dl, WideVT);
24886 SDValue Zero = DAG.getConstant(0, dl, WideVT);
24887 V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
24893 V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V);
24898 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,
24899 DAG.getVectorIdxConstant(0, dl));
24905 SelectionDAG &DAG) {
24911 return LowerSIGN_EXTEND_Mask(Op, DL, Subtarget, DAG);
24914 return LowerAVXExtend(Op, DL, DAG, Subtarget);
24923 SelectionDAG &DAG) {
24951 In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128));
24962 return DAG.getNode(Op.getOpcode(), dl, VT, In);
24969 return DAG.getNode(ExtOpc, dl, VT, In);
24983 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
24984 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
24985 Hi = DAG.getNode(Opc, dl, HalfVT, Hi);
24986 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
24997 if (DAG.ComputeNumSignBits(In, DemandedElts) == InVT.getScalarSizeInBits()) {
25002 return DAG.getBitcast(VT,
25003 DAG.getVectorShuffle(InVT, dl, In, In, ShuffleMask));
25025 Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask);
25026 Curr = DAG.getBitcast(DestVT, Curr);
25029 SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
25030 DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
25035 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
25036 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
25037 SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});
25038 SignExt = DAG.getBitcast(VT, SignExt);
25045 SelectionDAG &DAG) {
25052 return LowerSIGN_EXTEND_Mask(Op, dl, Subtarget, DAG);
25068 return splitVectorIntUnary(Op, DAG, dl);
25083 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
25090 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
25091 OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
25093 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
25097 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
25113 std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL);
25117 DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(HalfOffset), DL);
25119 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
25122 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
25126 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
25132 SelectionDAG &DAG) {
25136 StoredVal = DAG.getBitcast(StoreVT, StoredVal);
25152 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(),
25154 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
25155 DAG.getVectorIdxConstant(i, DL));
25156 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
25162 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
25166 SelectionDAG &DAG) {
25181 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
25182 DAG.getUNDEF(MVT::v16i1), StoredVal,
25183 DAG.getVectorIdxConstant(0, dl));
25184 StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
25185 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
25188 StoredVal = DAG.getZeroExtendInReg(
25189 StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts));
25191 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
25207 if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal.getNode(), DAG))
25208 return splitVectorStore(St, DAG);
25215 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25217 assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==
25221 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
25222 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
25223 DAG.getUNDEF(StoreVT));
25230 StoredVal = DAG.getBitcast(CastVT, StoredVal);
25231 StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal,
25232 DAG.getVectorIdxConstant(0, dl));
25234 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
25239 SDVTList Tys = DAG.getVTList(MVT::Other);
25241 return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,
25253 SelectionDAG &DAG) {
25269 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
25276 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
25277 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
25278 DAG.getBitcast(MVT::v16i1, Val),
25279 DAG.getVectorIdxConstant(0, dl));
25280 return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
25298 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
25320 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG);
25325 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25326 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25332 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
25333 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25351 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
25357 DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
25358 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25359 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
25361 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
25362 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25370 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
25371 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25372 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25374 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
25375 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25379 TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
25380 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
25381 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25382 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25390 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
25392 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
25393 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
25398 if (isTruncWithZeroHighBitsInput(Cond, DAG))
25406 DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT));
25409 SDValue RHS = DAG.getConstant(0, dl, CondVT);
25412 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
25413 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, EFLAGS,
25424 SelectionDAG &DAG) const {
25425 MachineFunction &MF = DAG.getMachineFunction();
25441 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
25444 MVT SPTy = getPointerTy(DAG.getDataLayout());
25448 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25456 Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, {SPTy, MVT::Other},
25460 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
25462 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
25465 Result = DAG.getNode(
25467 DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
25468 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
25482 DAG.getNode(X86ISD::SEG_ALLOCA, dl, {SPTy, MVT::Other}, {Chain, Size});
25485 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
25486 Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size);
25491 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
25495 SP = DAG.getNode(
25497 DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
25498 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
25504 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
25507 return DAG.getMergeValues(Ops, dl);
25510 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
25511 MachineFunction &MF = DAG.getMachineFunction();
25522 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25523 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
25535 SDValue Store = DAG.getStore(
25537 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
25542 FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(4), DL);
25543 Store = DAG.getStore(
25545 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
25550 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
25551 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25553 DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
25557 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
25559 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
25560 Store = DAG.getStore(
25564 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
25567 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
25572 MachineFunction &MF = DAG.getMachineFunction();
25575 return DAG.expandVAArg(Op.getNode());
25584 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
25585 uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
25607 // Insert VAARG node into the DAG
25610 DAG.getTargetConstant(ArgSize, dl, MVT::i32),
25611 DAG.getTargetConstant(ArgMode, dl, MVT::i8),
25612 DAG.getTargetConstant(Align, dl, MVT::i32)};
25613 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
25614 SDValue VAARG = DAG.getMemIntrinsicNode(
25622 return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo());
25626 SelectionDAG &DAG) {
25631 DAG.getMachineFunction().getFunction().getCallingConv()))
25633 return DAG.expandVACopy(Op.getNode());
25642 return DAG.getMemcpy(
25644 DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
25673 SelectionDAG &DAG) {
25679 SrcOp = DAG.getBitcast(VT, SrcOp);
25690 return DAG.getConstant(0, dl, VT);
25713 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
25714 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
25718 return DAG.getNode(Opc, dl, VT, SrcOp,
25719 DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
25726 SelectionDAG &DAG) {
25736 ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
25758 ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
25759 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25760 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
25769 DAG.getConstant(0, dl, AmtVT.getScalarType()));
25770 MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
25771 SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
25772 if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
25774 ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
25782 ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
25791 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
25793 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
25796 SDValue ByteShift = DAG.getTargetConstant(
25798 ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
25799 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25801 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25814 ShAmt = DAG.getBitcast(ShVT, ShAmt);
25815 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
25821 const X86Subtarget &Subtarget, SelectionDAG &DAG,
25825 return DAG.getConstant(1, dl, MaskVT);
25827 return DAG.getConstant(0, dl, MaskVT);
25836 std::tie(Lo, Hi) = DAG.SplitScalar(Mask, dl, MVT::i32, MVT::i32);
25837 Lo = DAG.getBitcast(MVT::v32i1, Lo);
25838 Hi = DAG.getBitcast(MVT::v32i1, Hi);
25839 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
25845 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
25846 DAG.getBitcast(BitcastVT, Mask),
25847 DAG.getVectorIdxConstant(0, dl));
25857 SelectionDAG &DAG) {
25866 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25869 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25870 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
25883 SelectionDAG &DAG) {
25893 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
25894 DAG.getBitcast(MVT::v8i1, Mask),
25895 DAG.getVectorIdxConstant(0, dl));
25899 return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
25902 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25903 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
25929 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
25931 MachineFunction &MF = DAG.getMachineFunction();
25934 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25935 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
25947 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
25949 DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
25953 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
25955 return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset);
25960 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
25961 DAG.getConstant(RegNodeSize, dl, PtrVT));
25962 return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset);
25966 SelectionDAG &DAG) const {
26010 SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
26023 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
26025 DAG.getTargetConstant(RC, dl, MVT::i32));
26029 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26043 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1));
26056 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
26058 DAG.getTargetConstant(RC, dl, MVT::i32));
26063 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26077 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
26088 Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8);
26099 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
26101 DAG.getTargetConstant(RC, dl, MVT::i32));
26106 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26113 Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8);
26116 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26133 DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
26134 Src, DAG.getTargetConstant(RC, dl, MVT::i32)),
26135 Mask, PassThru, Subtarget, DAG);
26140 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
26141 Subtarget, DAG);
26157 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru,
26158 Subtarget, DAG);
26176 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2,
26177 DAG.getTargetConstant(RC, dl, MVT::i32)),
26178 Mask, passThru, Subtarget, DAG);
26182 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
26184 Mask, passThru, Subtarget, DAG);
26198 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
26200 Mask, passThru, Subtarget, DAG);
26212 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
26214 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
26215 DAG.getTargetConstant(RC, dl, MVT::i32));
26219 return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG);
26235 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
26236 Mask, passThru, Subtarget, DAG);
26248 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
26249 DAG.getTargetConstant(RC, dl, MVT::i32));
26254 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
26255 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
26272 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
26273 Mask, PassThru, Subtarget, DAG);
26290 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
26291 Mask, PassThru, Subtarget, DAG);
26308 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
26309 Mask, PassThru, Subtarget, DAG);
26317 Src3 = DAG.getBitcast(MaskVT, Src3);
26320 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
26327 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
26339 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
26349 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3,
26350 DAG.getTargetConstant(RC, dl, MVT::i32));
26355 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3);
26358 return getScalarMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
26359 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
26364 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26370 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
26372 Subtarget, DAG);
26375 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
26376 DAG.getConstant(0, dl, MVT::v8i1), FPclassMask,
26377 DAG.getVectorIdxConstant(0, dl));
26378 return DAG.getBitcast(MVT::i8, Ins);
26391 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
26397 return DAG.getNode(IntrData->Opc0, dl, MaskVT,
26410 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);
26416 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
26419 Subtarget, DAG);
26422 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
26423 DAG.getConstant(0, dl, MVT::v8i1), CmpMask,
26424 DAG.getVectorIdxConstant(0, dl));
26425 return DAG.getBitcast(MVT::i8, Ins);
26449 SDValue Comi = DAG.getNode(ComiOpCode, dl, MVT::i32, LHS, RHS);
26454 SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
26458 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
26459 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);
26463 SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
26467 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
26468 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);
26473 SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
26478 SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
26483 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26493 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
26494 DAG.getTargetConstant(CondVal, dl, MVT::i8));
26496 FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
26497 DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
26502 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
26503 DAG.getConstant(0, dl, MVT::v16i1), FCmp,
26504 DAG.getVectorIdxConstant(0, dl));
26505 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
26506 DAG.getBitcast(MVT::i16, Ins));
26518 CShAmt->getZExtValue(), DAG);
26520 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
26522 SrcOp, ShAmt, 0, Subtarget, DAG);
26533 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
26535 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
26547 : getZeroVector(VT, Subtarget, DAG, dl);
26558 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
26561 return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
26563 return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
26570 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26571 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26579 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26580 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26587 SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
26589 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26594 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
26595 SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);
26601 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2),
26604 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
26605 DAG.getAllOnesConstant(dl, MVT::i8));
26606 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
26609 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
26611 return DAG.getMergeValues(Results, dl);
26622 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
26626 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26627 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
26637 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), {Src, Src2});
26641 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26642 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
26657 Rnd = DAG.getTargetConstant(RC, dl, MVT::i32);
26661 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd);
26669 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26670 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd, PassThru, Mask);
26678 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
26682 PassThru = DAG.getConstant(0, dl, PassThru.getValueType());
26684 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
26780 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
26781 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
26782 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26841 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26842 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
26843 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
26844 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26856 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26857 return DAG.getNode(Opcode, dl, VTs, NewOps);
26869 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26870 return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
26874 MachineFunction &MF = DAG.getMachineFunction();
26875 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26876 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
26880 return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT,
26881 DAG.getMCSymbol(S, PtrVT));
26886 MachineFunction &MF = DAG.getMachineFunction();
26894 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
26895 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
26906 return recoverFramePointer(DAG, Fn, IncomingFPOp);
26912 MachineFunction &MF = DAG.getMachineFunction();
26924 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
26934 SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
26938 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
26941 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26943 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
26945 return DAG.getMergeValues({Result0, Result1}, DL);
26965 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26967 DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
27002 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
27003 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
27004 DAG.getTargetConstant(NewIntrinsic, DL,
27005 getPointerTy(DAG.getDataLayout())),
27011 EVT PtrVT = getPointerTy(DAG.getDataLayout());
27014 *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
27015 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
27016 DAG.getIntPtrConstant(0, dl), MachinePointerInfo(Ptr));
27024 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27033 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27034 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27035 TLI.getPointerTy(DAG.getDataLayout()));
27037 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
27042 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
27045 Mask = DAG.getBitcast(MaskVT, Mask);
27051 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
27053 return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
27056 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
27066 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27067 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27068 TLI.getPointerTy(DAG.getDataLayout()));
27076 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27078 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
27083 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
27089 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
27091 return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
27094 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27103 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27104 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27105 TLI.getPointerTy(DAG.getDataLayout()));
27113 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27117 SDVTList VTs = DAG.getVTList(MVT::Other);
27120 DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
27125 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
27134 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27135 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
27136 TLI.getPointerTy(DAG.getDataLayout()));
27137 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
27138 SDValue Segment = DAG.getRegister(0, MVT::i32);
27141 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27143 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
27156 SelectionDAG &DAG,
27166 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
27170 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
27172 SDNode *N1 = DAG.getMachineNode(
27179 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
27180 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
27183 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
27184 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
27192 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
27193 DAG.getConstant(32, DL, MVT::i8));
27194 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
27201 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
27211 SelectionDAG &DAG,
27217 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
27226 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
27232 SelectionDAG &DAG) {
27235 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
27237 return DAG.getMergeValues(Results, DL);
27240 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
27241 MachineFunction &MF = DAG.getMachineFunction();
27254 // Return the chain operand without making any DAG nodes.
27258 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
27259 MachineFunction &MF = DAG.getMachineFunction();
27272 // Return the chain operand without making any DAG nodes.
27280 SelectionDAG &DAG) {
27281 SDVTList VTs = DAG.getVTList(MVT::Other);
27282 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
27285 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
27292 MachineMemOperand *MMO, SelectionDAG &DAG) {
27293 SDVTList VTs = DAG.getVTList(MVT::Other);
27296 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
27309 SelectionDAG &DAG) {
27317 auto &MF = DAG.getMachineFunction();
27323 SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
27325 SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
27326 DAG.getTargetConstant(8, dl, MVT::i32)),
27329 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
27339 DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(),
27342 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
27348 return MarkEHRegistrationNode(Op, DAG);
27350 return MarkEHGuard(Op, DAG);
27353 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
27355 return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0),
27356 DAG.getConstant(0, dl, MVT::i32));
27362 return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
27364 DAG.getConstant(0, dl, MVT::i32),
27365 DAG.getConstant(0, dl, MVT::i32));
27369 DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true);
27379 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
27391 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
27409 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
27411 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27412 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
27419 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
27430 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
27432 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
27433 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
27441 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);
27464 SDValue Operation = DAG.getMemIntrinsicNode(
27467 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
27469 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
27477 SDVTList VTs = DAG.getVTList(
27502 SDValue Operation = DAG.getMemIntrinsicNode(
27508 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
27510 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
27520 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
27521 SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain);
27522 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27523 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
27534 auto *X86MFI = DAG.getMachineFunction().getInfo<X86MachineFunctionInfo>();
27568 SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
27574 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
27576 DAG.getTargetConstant(0, DL, MVT::i32), // Disp
27577 DAG.getRegister(0, MVT::i16), // Segment
27580 MachineSDNode *Res = DAG.getMachineNode(Opc, DL, VTs, Ops);
27581 SDValue Res0 = DAG.getTargetExtractSubreg(X86::sub_t0, DL, MVT::x86amx,
27583 SDValue Res1 = DAG.getTargetExtractSubreg(X86::sub_t1, DL, MVT::x86amx,
27585 return DAG.getMergeValues({Res0, Res1, SDValue(Res, 1)}, DL);
27600 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27603 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27604 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
27617 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);
27620 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27623 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27626 Res = DAG.getNode(ISD::SHL, DL, VT, Res,
27627 DAG.getShiftAmountConstant(Imm, VT, DL));
27628 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
27639 SDValue Operation = DAG.getMemIntrinsicNode(
27679 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(),
27715 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27718 return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
27730 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);
27731 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27735 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
27736 DAG.getConstant(1, dl, Op->getValueType(1)),
27737 DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
27739 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
27742 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
27752 return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
27763 return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
27774 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
27787 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
27793 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
27795 return DAG.getMergeValues(Results, dl);
27809 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
27811 return DAG.getMergeValues(Results, dl);
27815 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
27816 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27818 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
27819 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
27820 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
27840 return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT,
27844 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27845 SDValue Offset = DAG.getUNDEF(VMask.getValueType());
27847 return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
27856 MemIntr->getMemOperand(), DAG);
27859 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27862 VMask, MemVT, MemIntr->getMemOperand(), DAG);
27874 SelectionDAG &DAG) const {
27875 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
27878 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
27883 EVT PtrVT = getPointerTy(DAG.getDataLayout());
27886 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
27888 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
27889 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
27890 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
27895 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
27896 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
27901 SelectionDAG &DAG) const {
27902 DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true);
27903 return getReturnAddressFrameIndex(DAG);
27906 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
27907 MachineFunction &MF = DAG.getMachineFunction();
27927 return DAG.getFrameIndex(FrameAddrIndex, VT);
27931 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
27937 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
27939 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
27980 SelectionDAG &DAG) const {
27982 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
28005 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
28011 EVT PtrVT = getPointerTy(DAG.getDataLayout());
28013 Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
28017 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
28020 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
28021 DAG.getIntPtrConstant(RegInfo->getSlotSize(),
28023 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
28024 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
28025 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
28027 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
28028 DAG.getRegister(StoreAddrReg, PtrVT));
28032 SelectionDAG &DAG) const {
28042 (void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
28044 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
28045 DAG.getVTList(MVT::i32, MVT::Other),
28050 SelectionDAG &DAG) const {
28052 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
28057 SelectionDAG &DAG) const {
28059 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
28063 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
28068 SelectionDAG &DAG) const {
28093 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
28096 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
28097 DAG.getConstant(2, dl, MVT::i64));
28098 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
28104 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
28105 DAG.getConstant(10, dl, MVT::i64));
28106 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
28109 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
28110 DAG.getConstant(12, dl, MVT::i64));
28111 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
28116 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
28117 DAG.getConstant(20, dl, MVT::i64));
28118 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
28122 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
28123 DAG.getConstant(22, dl, MVT::i64));
28124 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
28127 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
28154 const DataLayout &DL = DAG.getDataLayout();
28180 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
28181 DAG.getConstant(10, dl, MVT::i32));
28182 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
28188 DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),
28191 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
28192 DAG.getConstant(1, dl, MVT::i32));
28193 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
28197 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
28198 DAG.getConstant(5, dl, MVT::i32));
28200 DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr,
28203 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
28204 DAG.getConstant(6, dl, MVT::i32));
28205 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
28208 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
28213 SelectionDAG &DAG) const {
28236 MachineFunction &MF = DAG.getMachineFunction();
28243 DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
28249 Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
28250 DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,
28254 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
28259 DAG.getNode(ISD::SRL, DL, MVT::i16,
28260 DAG.getNode(ISD::AND, DL, MVT::i16,
28261 CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
28262 DAG.getConstant(9, DL, MVT::i8));
28263 Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
28265 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
28267 DAG.getNode(ISD::AND, DL, MVT::i32,
28268 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
28269 DAG.getConstant(3, DL, MVT::i32));
28271 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
28273 return DAG.getMergeValues({RetVal, Chain}, DL);
28277 SelectionDAG &DAG) const {
28278 MachineFunction &MF = DAG.getMachineFunction();
28286 DAG.getFrameIndex(OldCWFrameIdx, getPointerTy(DAG.getDataLayout()));
28293 Chain = DAG.getMemIntrinsicNode(
28294 X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO);
28297 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);
28299 CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0),
28300 DAG.getConstant(0xf3ff, DL, MVT::i16));
28318 RMBits = DAG.getConstant(FieldVal, DL, MVT::i16);
28333 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
28334 DAG.getNode(ISD::ADD, DL, MVT::i32,
28335 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM,
28336 DAG.getConstant(1, DL, MVT::i8)),
28337 DAG.getConstant(4, DL, MVT::i32)));
28339 DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16),
28341 RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted,
28342 DAG.getConstant(0xc00, DL, MVT::i16));
28346 CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits);
28347 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(2));
28353 Chain = DAG.getMemIntrinsicNode(
28354 X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL);
28360 Chain = DAG.getNode(
28361 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
28362 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
28366 SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);
28368 CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0),
28369 DAG.getConstant(0xffff9fff, DL, MVT::i32));
28372 RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits);
28373 RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits,
28374 DAG.getConstant(3, DL, MVT::i8));
28377 CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits);
28378 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(4));
28381 Chain = DAG.getNode(
28382 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
28383 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
28395 SelectionDAG &DAG) const {
28396 MachineFunction &MF = DAG.getMachineFunction();
28408 DAG.getMemIntrinsicNode(X86ISD::FNSTENVm, DL, DAG.getVTList(MVT::Other),
28417 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
28424 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
28425 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
28426 DAG.getConstant(X87StateSize, DL, PtrVT));
28428 Chain = DAG.getNode(
28429 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
28430 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
28439 SelectionDAG &DAG,
28444 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
28449 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
28450 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
28451 DAG.getConstant(X87StateSize, DL, PtrVT));
28453 Chain = DAG.getNode(
28454 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
28455 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
28462 SelectionDAG &DAG) const {
28470 return createSetFPEnvNodes(Ptr, Chain, DL, MemVT, MMO, DAG, Subtarget);
28474 SelectionDAG &DAG) const {
28475 MachineFunction &MF = DAG.getMachineFunction();
28479 IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext());
28496 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
28497 SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT);
28499 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
28503 return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget);
28530 SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL,
28538 MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8));
28540 return DAG.getBuildVector(VT, DL, MaskBits);
28549 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
28563 return splitVectorIntUnary(Op, DAG, dl);
28570 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
28571 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
28572 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
28573 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
28575 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);
28581 SelectionDAG &DAG) {
28595 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
28596 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
28603 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
28604 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
28606 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
28608 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
28612 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
28613 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
28615 HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
28618 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
28619 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
28620 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
28621 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
28633 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
28638 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
28639 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
28640 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
28642 HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
28643 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
28645 HiZ = DAG.getBitcast(NextVT, HiZ);
28650 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
28651 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
28652 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
28653 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
28654 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
28663 SelectionDAG &DAG) {
28669 return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget);
28673 return splitVectorIntUnary(Op, DAG, DL);
28677 return splitVectorIntUnary(Op, DAG, DL);
28680 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
28684 SelectionDAG &DAG) {
28692 return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
28698 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
28702 SDValue PassThru = DAG.getUNDEF(OpVT);
28704 !DAG.isKnownNeverZero(Op))
28705 PassThru = DAG.getConstant(NumBits + NumBits - 1, dl, OpVT);
28708 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
28709 Op = DAG.getNode(X86ISD::BSR, dl, VTs, PassThru, Op);
28714 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
28715 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28717 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
28721 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
28722 DAG.getConstant(NumBits - 1, dl, OpVT));
28725 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
28730 SelectionDAG &DAG) {
28735 bool NonZeroSrc = DAG.isKnownNeverZero(N0);
28741 SDValue PassThru = DAG.getUNDEF(VT);
28743 PassThru = DAG.getConstant(NumBits, dl, VT);
28746 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
28747 Op = DAG.getNode(X86ISD::BSF, dl, VTs, PassThru, N0);
28754 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
28755 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28757 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
28760 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
28766 return lowerAddSubToHorizontalOp(Op, DL, DAG, Subtarget);
28769 return splitVectorIntBinary(Op, DAG, DL);
28774 return splitVectorIntBinary(Op, DAG, DL);
28777 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
28788 return splitVectorIntBinary(Op, DAG, DL);
28792 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28794 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28806 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT);
28807 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT);
28808 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask);
28809 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt);
28810 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra);
28815 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
28816 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
28819 DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits())
28820 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub);
28821 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
28829 SDValue Zero = DAG.getConstant(0, DL, VT);
28831 DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL,
28832 DAG.getVTList(VT, SetCCResultType), X, Y);
28835 SDValue SatMin = DAG.getConstant(MinVal, DL, VT);
28836 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT);
28838 DAG.getSetCC(DL, SetCCResultType, SumDiff, Zero, ISD::SETLT);
28839 Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin);
28840 return DAG.getSelect(DL, VT, Overflow, Result, SumDiff);
28848 SelectionDAG &DAG) {
28856 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
28857 DAG.getConstant(0, DL, VT), N0);
28858 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
28860 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
28866 SDValue Neg = DAG.getNegative(Src, DL, VT);
28867 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Neg, Src);
28873 return splitVectorIntUnary(Op, DAG, DL);
28877 return splitVectorIntUnary(Op, DAG, DL);
28884 SelectionDAG &DAG) {
28890 return splitVectorIntBinary(Op, DAG, DL);
28893 return splitVectorIntBinary(Op, DAG, DL);
28900 SelectionDAG &DAG) {
28906 return splitVectorIntBinary(Op, DAG, DL);
28909 return splitVectorIntBinary(Op, DAG, DL);
28916 SelectionDAG &DAG) {
28917 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28935 DAG.getTargetConstant(IsMaxOp + (IsNum ? 16 : 0), DL, MVT::i32);
28936 return DAG.getNode(Opc, DL, VT, X, Y, Imm, Op->getFlags());
28953 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28996 bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
28997 bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
28998 bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
29000 DAG.isKnownNeverZeroFloat(X) ||
29001 DAG.isKnownNeverZeroFloat(Y);
29018 SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
29022 SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
29024 SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
29025 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
29026 DAG.getConstant(0, DL, MVT::v8i1), IsNanZero,
29027 DAG.getVectorIdxConstant(0, DL));
29028 SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
29029 NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X);
29030 NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
29031 return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
29035 SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
29036 SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
29037 IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
29040 SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
29041 DAG.getConstantFP(0, DL, MVT::v2f64), X,
29042 DAG.getVectorIdxConstant(0, DL));
29043 SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
29044 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
29045 DAG.getVectorIdxConstant(1, DL));
29046 Hi = DAG.getBitcast(MVT::i32, Hi);
29047 SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
29048 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
29049 *DAG.getContext(), MVT::i32);
29050 IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
29053 NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
29054 NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
29056 NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
29057 NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
29061 bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
29067 if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
29070 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
29072 if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
29076 DAG.getSetCC(DL, SetCCType, NewX, NewX, IsNum ? ISD::SETO : ISD::SETUO);
29078 return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax);
29082 SelectionDAG &DAG) {
29088 return splitVectorIntBinary(Op, DAG, dl);
29091 return splitVectorIntBinary(Op, DAG, dl);
29094 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
29103 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
29104 SDValue LHS = DAG.getFreeze(Op.getOperand(0));
29105 SDValue RHS = DAG.getFreeze(Op.getOperand(1));
29106 SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, VTs, LHS, RHS);
29107 SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, VTs, RHS, LHS);
29108 return DAG.getNode(X86ISD::CMOV, dl, VT, Diff1, Diff0,
29109 DAG.getTargetConstant(CC, dl, MVT::i8),
29118 SDVTList WideVTs = DAG.getVTList(WideVT, MVT::i32);
29119 SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
29120 SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
29121 SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, WideVTs, LHS, RHS);
29122 SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, WideVTs, RHS, LHS);
29123 SDValue AbsDiff = DAG.getNode(X86ISD::CMOV, dl, WideVT, Diff1, Diff0,
29124 DAG.getTargetConstant(CC, dl, MVT::i8),
29126 return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff);
29135 SelectionDAG &DAG) {
29141 return splitVectorIntBinary(Op, DAG, dl);
29144 return splitVectorIntBinary(Op, DAG, dl);
29159 return DAG.getNode(
29161 DAG.getNode(ISD::MUL, dl, ExVT,
29162 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A),
29163 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B)));
29183 SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT));
29184 SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B);
29185 SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B);
29186 SDValue RLo = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BLo);
29187 SDValue RHi = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BHi);
29188 RLo = DAG.getNode(ISD::AND, dl, VT, DAG.getBitcast(VT, RLo), Mask);
29189 RHi = DAG.getNode(X86ISD::VSHLI, dl, ExVT, RHi,
29190 DAG.getTargetConstant(8, dl, MVT::i8));
29191 return DAG.getNode(ISD::OR, dl, VT, RLo, DAG.getBitcast(VT, RHi));
29199 SDValue Undef = DAG.getUNDEF(VT);
29200 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
29201 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
29209 LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl,
29211 HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl,
29216 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
29217 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
29219 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));
29220 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));
29224 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
29225 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
29226 return getPack(DAG, Subtarget, dl, VT, RLo, RHi);
29236 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
29237 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
29240 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
29241 DAG.getBitcast(MVT::v2i64, A),
29242 DAG.getBitcast(MVT::v2i64, B));
29244 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
29245 DAG.getBitcast(MVT::v2i64, Aodds),
29246 DAG.getBitcast(MVT::v2i64, Bodds));
29248 Evens = DAG.getBitcast(VT, Evens);
29249 Odds = DAG.getBitcast(VT, Odds);
29254 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
29270 KnownBits AKnown = DAG.computeKnownBits(A);
29271 KnownBits BKnown = DAG.computeKnownBits(B);
29281 SDValue Zero = DAG.getConstant(0, dl, VT);
29286 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
29290 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
29291 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
29296 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
29297 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
29300 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
29301 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
29303 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);
29309 SelectionDAG &DAG,
29326 SDValue Zero = DAG.getConstant(0, dl, VT);
29330 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
29331 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
29333 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
29334 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
29347 LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
29348 HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
29349 LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
29350 DAG.getConstant(8, dl, MVT::i16));
29351 HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
29352 DAG.getConstant(8, dl, MVT::i16));
29354 LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
29355 HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
29363 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
29364 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
29366 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
29367 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
29369 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
29370 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
29376 SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
29377 SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
29380 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
29382 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
29386 SelectionDAG &DAG) {
29396 return splitVectorIntBinary(Op, DAG, dl);
29399 return splitVectorIntBinary(Op, DAG, dl);
29422 DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
29425 DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
29434 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
29435 DAG.getBitcast(MulVT, A),
29436 DAG.getBitcast(MulVT, B)));
29439 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
29440 DAG.getBitcast(MulVT, Odd0),
29441 DAG.getBitcast(MulVT, Odd1)));
29448 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
29453 SDValue Zero = DAG.getConstant(0, dl, VT);
29454 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
29455 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);
29456 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
29457 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);
29459 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
29460 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);
29481 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
29482 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
29483 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
29484 Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
29485 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
29488 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG);
29493 SelectionDAG &DAG) {
29498 return LowerXALUO(Op, DAG);
29510 std::tie(LHSLo, LHSHi) = splitVector(A, DAG, dl);
29514 std::tie(RHSLo, RHSHi) = splitVector(B, DAG, dl);
29517 std::tie(LoOvfVT, HiOvfVT) = DAG.GetSplitDestVTs(OvfVT);
29518 SDVTList LoVTs = DAG.getVTList(LHSLo.getValueType(), LoOvfVT);
29519 SDVTList HiVTs = DAG.getVTList(LHSHi.getValueType(), HiOvfVT);
29522 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo);
29523 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi);
29526 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29527 SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1),
29530 return DAG.getMergeValues({Res, Ovf}, dl);
29533 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
29535 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
29542 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
29543 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
29544 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
29546 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
29555 High = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Mul, 8, DAG);
29558 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExVT, Mul, 8, DAG);
29560 15, DAG);
29564 High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High);
29565 LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign);
29569 High = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
29570 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
29572 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
29575 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
29578 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
29585 High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High);
29589 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
29593 DAG.getSetCC(dl, SetccVT, High,
29594 DAG.getConstant(0, dl, High.getValueType()), ISD::SETNE);
29597 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
29599 return DAG.getMergeValues({Low, Ovf}, dl);
29604 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low);
29610 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
29611 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
29615 DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE);
29618 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
29620 return DAG.getMergeValues({Low, Ovf}, dl);
29623 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
29631 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i64, DAG))
29632 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]);
29648 SDValue InChain = DAG.getEntryNode();
29656 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29659 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
29662 DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
29663 Entry.Ty = PointerType::get(*DAG.getContext(), 0);
29669 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
29670 getPointerTy(DAG.getDataLayout()));
29672 TargetLowering::CallLoweringInfo CLI(DAG);
29677 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
29684 return DAG.getBitcast(VT, CallInfo.first);
29688 SelectionDAG &DAG,
29710 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
29716 makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
29717 Result = DAG.getBitcast(VT, Result);
29722 SelectionDAG &DAG) const {
29743 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
29746 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29749 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
29750 Chain = DAG.getStore(Chain, dl, Arg, StackPtr, MPI, Align(16));
29754 makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain);
29755 return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
29823 static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
29835 SDValue Ex = DAG.getBitcast(ExVT, R);
29841 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);
29847 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
29849 ShiftAmt - 32, DAG);
29851 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
29853 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
29858 ShiftAmt, DAG);
29860 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
29861 Lower = DAG.getBitcast(ExVT, Lower);
29863 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
29865 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
29868 return DAG.getBitcast(VT, Ex);
29878 return DAG.getUNDEF(VT);
29892 R = DAG.getFreeze(R);
29893 return DAG.getNode(ISD::ADD, dl, VT, R, R);
29896 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
29908 DAG.ComputeNumSignBits(R) == EltSizeInBits) {
29909 SDValue Mask = DAG.getAllOnesConstant(dl, VT);
29910 Mask = DAG.getNode(Op.getOpcode(), dl, VT, Mask, Amt);
29911 return DAG.getNode(ISD::AND, dl, VT, R, Mask);
29926 R = DAG.getFreeze(R);
29927 return DAG.getNode(ISD::ADD, dl, VT, R, R);
29932 SDValue Zeros = DAG.getConstant(0, dl, VT);
29935 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
29936 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);
29938 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
29946 SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt);
29947 return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask,
29948 DAG.getTargetConstant(0, dl, MVT::i8));
29954 ShiftAmt, DAG);
29955 SHL = DAG.getBitcast(VT, SHL);
29958 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT));
29963 ShiftAmt, DAG);
29964 SRL = DAG.getBitcast(VT, SRL);
29967 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT));
29971 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29973 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
29974 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
29975 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
29984 static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
29994 if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) {
29997 Subtarget, DAG);
30012 SDValue BitMask = DAG.getAllOnesConstant(dl, ExtVT);
30014 BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
30017 8, DAG);
30018 BitMask = DAG.getBitcast(VT, BitMask);
30019 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,
30023 DAG.getBitcast(ExtVT, R), BaseShAmt,
30024 BaseShAmtIdx, Subtarget, DAG);
30025 Res = DAG.getBitcast(VT, Res);
30026 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);
30031 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
30034 BaseShAmtIdx, Subtarget, DAG);
30035 SignMask = DAG.getBitcast(VT, SignMask);
30036 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);
30037 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);
30050 SelectionDAG &DAG) {
30068 SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
30073 Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT);
30075 return DAG.getBuildVector(VT, dl, Elts);
30081 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
30082 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,
30083 DAG.getConstant(0x3f800000U, dl, VT));
30084 Amt = DAG.getBitcast(MVT::v4f32, Amt);
30085 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);
30090 SDValue Z = DAG.getConstant(0, dl, VT);
30091 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
30092 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
30093 Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG);
30094 Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG);
30096 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
30097 return getPack(DAG, Subtarget, dl, VT, Lo, Hi);
30104 SelectionDAG &DAG) {
30120 if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
30123 if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
30135 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
30136 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
30137 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
30138 R = DAG.getNode(ISD::XOR, dl, VT, R, M);
30139 R = DAG.getNode(ISD::SUB, dl, VT, R, M);
30148 Amt = DAG.getNegative(Amt, dl, VT);
30150 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
30152 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
30159 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
30160 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
30161 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
30162 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
30163 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
30214 DAG.getNode(Opc, dl, VT, R, DAG.getConstant(AmtA, dl, VT));
30216 DAG.getNode(Opc, dl, VT, R, DAG.getConstant(AmtB, dl, VT));
30217 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
30314 SDValue RWide = DAG.getBitcast(WideVT, R);
30316 SDValue AmtWide = DAG.getBuildVector(
30318 AmtWide = DAG.getZExtOrTrunc(AmtWide, dl, WideVT);
30321 SDValue ShiftedR = DAG.getNode(LogicalOpc, dl, WideVT, RWide, AmtWide);
30332 SDValue SplatFullMask = DAG.getAllOnesConstant(dl, VT);
30333 SDValue Mask = DAG.getNode(LogicalOpc, dl, VT, SplatFullMask, Amt);
30334 Mask = DAG.getBitcast(WideVT, Mask);
30336 SDValue Masked = DAG.getNode(ISD::AND, dl, WideVT, ShiftedR, Mask);
30337 Masked = DAG.getBitcast(VT, Masked);
30366 DAG.getConstant(APInt::getSignMask(EltSizeInBits), dl, VT);
30368 SDValue SignBitMask = DAG.getNode(LogicalOpc, dl, VT, SplatHighBit, Amt);
30370 DAG.getNode(ISD::XOR, dl, VT, Masked, SignBitMask);
30372 DAG.getNode(ISD::SUB, dl, VT, FlippedSignBit, SignBitMask);
30382 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
30383 return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
30389 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
30390 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
30391 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
30392 SDValue Zero = DAG.getConstant(0, dl, VT);
30393 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
30394 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
30395 return DAG.getSelect(dl, VT, ZAmt, R, Res);
30407 DAG.isKnownNeverZero(Amt))) {
30408 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
30409 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
30410 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
30412 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
30414 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
30416 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
30417 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
30418 Res = DAG.getSelect(dl, VT, Amt0, R, Res);
30419 return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
30431 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
30432 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
30433 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
30434 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
30441 SDValue Z = DAG.getConstant(0, dl, VT);
30442 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
30443 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
30444 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
30445 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
30447 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
30448 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
30450 SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG);
30451 SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG);
30452 Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02);
30453 Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13);
30454 Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02);
30455 Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13);
30460 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
30461 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
30462 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
30463 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
30468 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
30469 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
30470 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
30472 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
30473 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
30474 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
30488 KnownBits KnownLane = DAG.computeKnownBits(R, EltMask);
30504 SDValue Mask = getConstVector(LUT, Undefs, VT, DAG, dl);
30505 return DAG.getNode(X86ISD::PSHUFB, dl, VT, Mask, Amt);
30523 R = DAG.getNode(ExtOpc, dl, ExtVT, R);
30524 Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt);
30525 return DAG.getNode(ISD::TRUNCATE, dl, VT,
30526 DAG.getNode(Opc, dl, ExtVT, R, Amt));
30536 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
30541 Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT);
30542 Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt);
30543 Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt);
30549 R = DAG.getExtOrTrunc(IsSigned, R, dl, ExVT);
30550 R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt);
30551 R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8);
30552 return DAG.getZExtOrTrunc(R, dl, VT);
30563 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
30564 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
30566 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
30567 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
30568 LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8);
30569 HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8);
30570 LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA);
30571 HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA);
30572 LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8);
30573 HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8);
30574 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
30588 V0 = DAG.getBitcast(VT, V0);
30589 V1 = DAG.getBitcast(VT, V1);
30590 Sel = DAG.getBitcast(VT, Sel);
30591 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
30593 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
30597 V0 = DAG.getBitcast(VT, V0);
30598 V1 = DAG.getBitcast(VT, V1);
30599 Sel = DAG.getBitcast(VT, Sel);
30600 return DAG.getBitcast(SelVT,
30601 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1));
30606 SDValue Z = DAG.getConstant(0, dl, SelVT);
30607 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
30608 return DAG.getSelect(dl, SelVT, C, V0, V1);
30614 Amt = DAG.getBitcast(ExtVT, Amt);
30615 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG);
30616 Amt = DAG.getBitcast(VT, Amt);
30620 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
30624 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
30627 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
30631 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
30634 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
30643 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
30644 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
30645 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
30646 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
30647 ALo = DAG.getBitcast(ExtVT, ALo);
30648 AHi = DAG.getBitcast(ExtVT, AHi);
30649 RLo = DAG.getBitcast(ExtVT, RLo);
30650 RHi = DAG.getBitcast(ExtVT, RHi);
30653 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
30654 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
30659 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
30660 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
30663 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG);
30664 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG);
30669 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
30670 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
30673 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG);
30674 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG);
30680 RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG);
30681 RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG);
30682 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
30688 SDValue Z = DAG.getConstant(0, dl, VT);
30689 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
30690 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
30691 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
30692 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
30693 ALo = DAG.getBitcast(ExtVT, ALo);
30694 AHi = DAG.getBitcast(ExtVT, AHi);
30695 RLo = DAG.getBitcast(ExtVT, RLo);
30696 RHi = DAG.getBitcast(ExtVT, RHi);
30697 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
30698 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
30699 Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG);
30700 Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG);
30701 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
30715 V0 = DAG.getBitcast(ExtVT, V0);
30716 V1 = DAG.getBitcast(ExtVT, V1);
30717 Sel = DAG.getBitcast(ExtVT, Sel);
30718 return DAG.getBitcast(
30719 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1));
30725 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
30726 return DAG.getSelect(dl, VT, C, V0, V1);
30733 Amt = DAG.getNode(
30735 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
30736 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
30738 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
30742 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
30746 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
30749 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
30753 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
30756 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
30760 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
30763 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
30770 return splitVectorIntBinary(Op, DAG, dl);
30773 return splitVectorIntBinary(Op, DAG, dl);
30779 SelectionDAG &DAG) {
30802 SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);
30804 {Op0, Op1, Imm}, DAG, Subtarget);
30807 {Op0, Op1, Amt}, DAG, Subtarget);
30836 DAG.getNode(ISD::SHL, DL, WideVT, DAG.getBitcast(WideVT, Op0),
30837 DAG.getShiftAmountConstant(ShXAmt, WideVT, DL));
30839 DAG.getNode(ISD::SRL, DL, WideVT, DAG.getBitcast(WideVT, Op1),
30840 DAG.getShiftAmountConstant(ShYAmt, WideVT, DL));
30841 ShX = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShX),
30842 DAG.getConstant(MaskX, DL, VT));
30843 ShY = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShY),
30844 DAG.getConstant(MaskY, DL, VT));
30845 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
30848 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0,
30849 DAG.getShiftAmountConstant(ShXAmt, VT, DL));
30850 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1,
30851 DAG.getShiftAmountConstant(ShYAmt, VT, DL));
30852 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
30855 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
30856 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30874 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod);
30875 return splitVectorOp(Op, DAG, DL);
30881 if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) {
30886 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30887 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30889 ScalarAmtIdx, Subtarget, DAG);
30891 ScalarAmtIdx, Subtarget, DAG);
30892 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
30909 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0);
30910 Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1);
30911 AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
30913 EltSizeInBits, DAG);
30914 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1);
30915 Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod);
30918 EltSizeInBits, DAG);
30919 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
30925 SDValue Z = DAG.getConstant(0, DL, VT);
30926 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30927 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30928 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
30929 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
30930 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
30931 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
30932 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
30943 bool OptForSize = DAG.shouldOptForSize();
30950 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
30951 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
30952 Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
30953 Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
30954 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
30955 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
30956 Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
30958 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
30960 Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
30961 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
30963 return DAG.getZExtOrTrunc(Res, DL, VT);
30971 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
30972 DAG.getConstant(15, DL, Amt.getValueType()));
30974 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
30981 SelectionDAG &DAG) {
31009 return DAG.getNode(RotOpc, DL, VT, R,
31010 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
31020 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
31023 SDValue Z = DAG.getConstant(0, DL, VT);
31028 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
31029 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt);
31033 return DAG.getNode(ISD::ROTL, DL, VT, R,
31034 DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
31039 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
31041 SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt);
31042 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask,
31043 DAG.getTargetConstant(0, DL, MVT::i8));
31048 return splitVectorIntBinary(Op, DAG, DL);
31060 return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
31061 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
31075 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R,
31076 DAG.getShiftAmountConstant(ShlAmt, VT, DL));
31077 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R,
31078 DAG.getShiftAmountConstant(SrlAmt, VT, DL));
31079 return DAG.getNode(ISD::OR, DL, VT, Shl, Srl);
31084 return splitVectorIntBinary(Op, DAG, DL);
31096 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
31097 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
31104 if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) {
31107 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
31110 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
31111 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
31113 BaseRotAmtIdx, Subtarget, DAG);
31115 BaseRotAmtIdx, Subtarget, DAG);
31116 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
31130 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
31131 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
31132 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
31133 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
31134 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
31135 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
31136 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
31155 R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R);
31156 R = DAG.getNode(
31158 getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG));
31159 Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
31160 R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt);
31162 R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG);
31163 return DAG.getNode(ISD::TRUNCATE, DL, VT, R);
31171 V0 = DAG.getBitcast(VT, V0);
31172 V1 = DAG.getBitcast(VT, V1);
31173 Sel = DAG.getBitcast(VT, Sel);
31174 return DAG.getBitcast(SelVT,
31175 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
31180 SDValue Z = DAG.getConstant(0, DL, SelVT);
31181 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
31182 return DAG.getSelect(DL, SelVT, C, V0, V1);
31187 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
31197 Amt = DAG.getBitcast(ExtVT, Amt);
31198 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
31199 Amt = DAG.getBitcast(VT, Amt);
31203 M = DAG.getNode(
31205 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)),
31206 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT)));
31210 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
31213 M = DAG.getNode(
31215 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)),
31216 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT)));
31220 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
31223 M = DAG.getNode(
31225 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)),
31226 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT)));
31230 bool IsSplatAmt = DAG.isSplatValue(Amt);
31237 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
31238 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
31239 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
31240 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
31241 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
31242 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
31247 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
31252 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
31258 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
31264 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
31265 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
31266 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
31274 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
31275 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
31277 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
31278 DAG.getBitcast(MVT::v2i64, R),
31279 DAG.getBitcast(MVT::v2i64, Scale));
31280 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
31281 DAG.getBitcast(MVT::v2i64, R13),
31282 DAG.getBitcast(MVT::v2i64, Scale13));
31283 Res02 = DAG.getBitcast(VT, Res02);
31284 Res13 = DAG.getBitcast(VT, Res13);
31286 return DAG.getNode(ISD::OR, DL, VT,
31287 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
31288 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
31820 static SDValue emitLockedStackOp(SelectionDAG &DAG,
31847 auto &MF = DAG.getMachineFunction();
31852 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31854 DAG.getRegister(X86::RSP, MVT::i64), // Base
31855 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
31856 DAG.getRegister(0, MVT::i64), // Index
31857 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
31858 DAG.getRegister(0, MVT::i16), // Segment.
31861 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31866 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31868 DAG.getRegister(X86::ESP, MVT::i32), // Base
31869 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
31870 DAG.getRegister(0, MVT::i32), // Index
31871 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
31872 DAG.getRegister(0, MVT::i16), // Segment.
31876 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31882 SelectionDAG &DAG) {
31894 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
31897 return emitLockedStackOp(DAG, Subtarget, Chain, dl);
31901 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
31905 SelectionDAG &DAG) {
31920 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
31925 DAG.getTargetConstant(size, DL, MVT::i8),
31927 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
31929 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
31933 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
31934 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
31936 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
31938 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
31943 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
31949 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
31950 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
31951 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
31952 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
31953 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
31954 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
31955 DAG.getConstant(32, DL, MVT::i8));
31956 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
31960 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
31961 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
31962 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
31963 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
31964 DAG.getConstant(16, DL, MVT::i8));
31965 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
31968 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
31972 SelectionDAG &DAG) {
31984 std::tie(Lo, Hi) = DAG.SplitScalar(Src, dl, MVT::i32, MVT::i32);
31985 Lo = DAG.getBitcast(MVT::v32i1, Lo);
31986 Hi = DAG.getBitcast(MVT::v32i1, Hi);
31987 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
31995 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
31996 V = getPMOVMSKB(DL, V, DAG, Subtarget);
31997 return DAG.getZExtOrTrunc(V, DL, DstVT);
32015 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
32016 DAG.getUNDEF(SrcVT));
32020 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
32024 Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src);
32027 return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src);
32029 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src,
32030 DAG.getVectorIdxConstant(0, dl));
32041 SelectionDAG &DAG) {
32055 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
32057 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
32058 return DAG.getBitcast(VT, V);
32067 SDValue Zeros = DAG.getConstant(0, DL, VT);
32068 SDValue V32 = DAG.getBitcast(VT, V);
32069 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
32070 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
32073 Zeros = DAG.getConstant(0, DL, ByteVecVT);
32075 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
32076 DAG.getBitcast(ByteVecVT, Low), Zeros);
32077 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
32078 DAG.getBitcast(ByteVecVT, High), Zeros);
32082 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
32083 DAG.getBitcast(ShortVecVT, Low),
32084 DAG.getBitcast(ShortVecVT, High));
32086 return DAG.getBitcast(VT, V);
32096 SDValue ShifterV = DAG.getConstant(8, DL, VT);
32097 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
32098 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
32099 DAG.getBitcast(ByteVecVT, V));
32100 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
32105 SelectionDAG &DAG) {
32129 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
32130 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
32131 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
32134 SDValue FourV = DAG.getConstant(4, DL, VT);
32135 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
32138 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
32143 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
32144 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
32145 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
32152 SelectionDAG &DAG) {
32165 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
32166 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
32167 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
32173 return splitVectorIntUnary(Op, DAG, DL);
32177 return splitVectorIntUnary(Op, DAG, DL);
32182 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
32183 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
32184 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);
32191 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
32195 SelectionDAG &DAG) {
32204 KnownBits Known = DAG.computeKnownBits(Op);
32206 return DAG.getConstant(Known.getConstant().popcount(), DL, VT);
32216 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
32217 DAG.getShiftAmountConstant(TZ, VT, DL));
32218 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
32219 Op = DAG.getNode(ISD::SUB, DL, MVT::i32, Op,
32220 DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
32221 DAG.getShiftAmountConstant(1, VT, DL)));
32222 return DAG.getZExtOrTrunc(Op, DL, VT);
32228 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
32229 DAG.getShiftAmountConstant(TZ, VT, DL));
32230 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
32231 Op = DAG.getNode(ISD::SHL, DL, MVT::i32, Op,
32232 DAG.getShiftAmountConstant(1, VT, DL));
32233 Op = DAG.getNode(ISD::SRL, DL, MVT::i32,
32234 DAG.getConstant(0b1110100110010100U, DL, MVT::i32), Op);
32235 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op,
32236 DAG.getConstant(0x3, DL, MVT::i32));
32237 return DAG.getZExtOrTrunc(Op, DL, VT);
32242 DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64)) {
32243 SDValue LUT = DAG.getConstant(0x4332322132212110ULL, DL, MVT::i64);
32245 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
32246 DAG.getShiftAmountConstant(TZ, VT, DL));
32247 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
32248 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op,
32249 DAG.getConstant(4, DL, MVT::i32));
32250 Op = DAG.getNode(ISD::SRL, DL, MVT::i64, LUT,
32251 DAG.getShiftAmountOperand(MVT::i64, Op));
32252 Op = DAG.getNode(ISD::AND, DL, MVT::i64, Op,
32253 DAG.getConstant(0x7, DL, MVT::i64));
32254 return DAG.getZExtOrTrunc(Op, DL, VT);
32259 SDValue Mask11 = DAG.getConstant(0x11111111U, DL, MVT::i32);
32261 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
32262 DAG.getShiftAmountConstant(TZ, VT, DL));
32263 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
32264 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op,
32265 DAG.getConstant(0x08040201U, DL, MVT::i32));
32266 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
32267 DAG.getShiftAmountConstant(3, MVT::i32, DL));
32268 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op, Mask11);
32269 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op, Mask11);
32270 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
32271 DAG.getShiftAmountConstant(28, MVT::i32, DL));
32272 return DAG.getZExtOrTrunc(Op, DL, VT);
32280 return LowerVectorCTPOP(N, DL, Subtarget, DAG);
32283 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
32292 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
32293 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
32294 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
32295 DAG.getVectorIdxConstant(0, DL));
32303 return splitVectorIntUnary(Op, DAG, DL);
32317 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
32321 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
32322 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
32323 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
32325 return DAG.getBitcast(VT, Res);
32329 SelectionDAG &DAG) {
32333 return LowerBITREVERSE_XOP(Op, DAG);
32342 return splitVectorIntUnary(Op, DAG, DL);
32346 return splitVectorIntUnary(Op, DAG, DL);
32354 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
32355 Res = DAG.getNode(ISD::BITREVERSE, DL, MVT::v16i8,
32356 DAG.getBitcast(MVT::v16i8, Res));
32358 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, DAG.getBitcast(VecVT, Res),
32359 DAG.getVectorIdxConstant(0, DL));
32360 return (VT == MVT::i8) ? Res : DAG.getNode(ISD::BSWAP, DL, VT, Res);
32368 SDValue Res = DAG.getNode(ISD::BSWAP, DL, VT, In);
32369 Res = DAG.getBitcast(ByteVT, Res);
32370 Res = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Res);
32371 return DAG.getBitcast(VT, Res);
32380 SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT);
32381 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
32382 DAG.getTargetConstant(0, DL, MVT::i8));
32388 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
32389 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
32390 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
32405 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
32406 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
32409 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
32410 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
32411 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
32412 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
32413 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
32417 SelectionDAG &DAG) {
32424 DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) {
32425 X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
32426 SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
32427 DAG.getConstant(0, DL, MVT::i8));
32429 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32431 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
32440 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
32441 DAG.getNode(ISD::SRL, DL, MVT::i64, X,
32442 DAG.getConstant(32, DL, MVT::i8)));
32443 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
32444 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
32449 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,
32450 DAG.getConstant(16, DL, MVT::i8));
32451 X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16);
32454 X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X);
32459 SDValue Hi = DAG.getNode(
32461 DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8)));
32462 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
32463 SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);
32464 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
32467 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32469 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
32472 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
32497 return DAG.getMemIntrinsicNode(
32498 NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
32504 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
32524 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,
32525 DAG.getNegative(RHS, DL, VT), AN->getMemOperand());
32550 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
32553 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
32554 DAG.getUNDEF(VT), NewChain);
32557 SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
32560 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
32561 DAG.getUNDEF(VT), NewChain);
32564 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
32568 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
32569 DAG.getUNDEF(VT), LockOp.getValue(1));
32572 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
32580 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT);
32588 !DAG.getMachineFunction().getFunction().hasFnAttribute(
32594 SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal());
32595 Chain = DAG.getStore(Node->getChain(), dl, VecVal, Node->getBasePtr(),
32604 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
32606 SclToVec = DAG.getBitcast(StVT, SclToVec);
32607 SDVTList Tys = DAG.getVTList(MVT::Other);
32609 Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
32614 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
32617 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
32618 Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr,
32620 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
32622 SDValue Value = DAG.getMemIntrinsicNode(
32630 DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),
32639 Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
32648 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
32654 static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
32660 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
32663 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
32669 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
32670 Carry, DAG.getAllOnesConstant(DL, CarryVT));
32673 SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
32679 Sum.getValue(1), DL, DAG);
32681 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
32683 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
32687 SelectionDAG &DAG) {
32696 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
32711 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
32715 DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
32720 TargetLowering::CallLoweringInfo CLI(DAG);
32722 .setChain(DAG.getEntryNode())
32733 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, CallResult.first,
32734 DAG.getVectorIdxConstant(0, dl));
32736 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, CallResult.first,
32737 DAG.getVectorIdxConstant(1, dl));
32738 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
32739 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
32744 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
32752 return DAG.getUNDEF(NVT);
32776 FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : DAG.getUNDEF(EltVT);
32779 return DAG.getBuildVector(NVT, dl, Ops);
32782 FillWithZeroes ? DAG.getConstant(0, dl, NVT) : DAG.getUNDEF(NVT);
32783 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal, InOp,
32784 DAG.getVectorIdxConstant(0, dl));
32788 SelectionDAG &DAG) {
32808 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
32809 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
32810 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
32811 SDVTList VTs = DAG.getVTList(MVT::Other);
32813 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
32839 Src = ExtendToType(Src, VT, DAG);
32840 Index = ExtendToType(Index, IndexVT, DAG);
32841 Mask = ExtendToType(Mask, MaskVT, DAG, true);
32844 SDVTList VTs = DAG.getVTList(MVT::Other);
32846 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
32851 SelectionDAG &DAG) {
32867 SDValue NewLoad = DAG.getMaskedLoad(
32869 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(),
32873 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
32874 return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
32895 PassThru = ExtendToType(PassThru, WideDataVT, DAG);
32903 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
32904 SDValue NewLoad = DAG.getMaskedLoad(
32910 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0),
32911 DAG.getVectorIdxConstant(0, dl));
32913 return DAG.getMergeValues(RetOps, dl);
32917 SelectionDAG &DAG) {
32950 DataToStore = ExtendToType(DataToStore, WideDataVT, DAG);
32951 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
32952 return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(),
32959 SelectionDAG &DAG) {
32992 PassThru = ExtendToType(PassThru, VT, DAG);
32993 Index = ExtendToType(Index, IndexVT, DAG);
32994 Mask = ExtendToType(Mask, MaskVT, DAG, true);
32999 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
33003 SDValue NewGather = DAG.getMemIntrinsicNode(
33004 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
33006 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT, NewGather,
33007 DAG.getVectorIdxConstant(0, dl));
33008 return DAG.getMergeValues({Extract, NewGather.getValue(1)}, dl);
33011 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
33023 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
33025 Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
33027 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
33035 SelectionDAG &DAG) const {
33047 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
33048 return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
33052 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
33056 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
33058 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
33060 Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC);
33061 Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC);
33062 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33066 SelectionDAG &DAG) {
33077 static SDValue LowerFCanonicalize(SDValue Op, SelectionDAG &DAG) {
33083 SDValue One = DAG.getConstantFP(1.0, dl, VT);
33089 SDValue Chain = DAG.getEntryNode();
33090 SDValue StrictFmul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other},
33149 static SDValue getFlagsOfCmpZeroFori1(SelectionDAG &DAG, const SDLoc &DL,
33152 auto V = DAG.getBitcast(MVT::i1, Mask);
33153 auto VE = DAG.getZExtOrTrunc(V, DL, Ty);
33154 auto Zero = DAG.getConstant(0, DL, Ty);
33155 SDVTList X86SubVTs = DAG.getVTList(Ty, MVT::i32);
33156 auto CmpZero = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, VE);
33161 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO,
33170 SDVTList Tys = DAG.getVTList(Ty, MVT::Other);
33171 auto ScalarPassThru = PassThru.isUndef() ? DAG.getConstant(0, DL, Ty)
33172 : DAG.getBitcast(Ty, PassThru);
33173 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask);
33174 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
33176 NewLoad = DAG.getMemIntrinsicNode(X86ISD::CLOAD, DL, Tys, Ops, Ty, MMO);
33177 return DAG.getBitcast(VTy, NewLoad);
33180 SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL,
33189 SDVTList Tys = DAG.getVTList(MVT::Other);
33190 auto ScalarVal = DAG.getBitcast(Ty, Val);
33191 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask);
33192 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
33194 return DAG.getMemIntrinsicNode(X86ISD::CSTORE, DL, Tys, Ops, Ty, MMO);
33198 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
33202 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
33204 return LowerCMP_SWAP(Op, Subtarget, DAG);
33205 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
33210 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget);
33211 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget);
33212 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG);
33213 case ISD::PARITY: return LowerPARITY(Op, Subtarget, DAG);
33214 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
33215 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
33216 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG);
33217 case ISD::VECTOR_COMPRESS: return lowerVECTOR_COMPRESS(Op, Subtarget, DAG);
33218 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
33219 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
33220 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
33221 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
33222 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
33223 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG);
33224 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
33225 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
33226 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
33227 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
33228 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
33231 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
33233 case ISD::FSHR: return LowerFunnelShift(Op, Subtarget, DAG);
33234 case ISD::FCANONICALIZE: return LowerFCanonicalize(Op, DAG);
33236 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
33238 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
33239 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
33240 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
33241 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
33242 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
33245 return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG);
33249 case ISD::STRICT_FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
33251 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG);
33253 case ISD::STRICT_FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
33255 case ISD::STRICT_FP_ROUND: return LowerFP_ROUND(Op, DAG);
33257 case ISD::STRICT_FP16_TO_FP: return LowerFP16_TO_FP(Op, DAG);
33259 case ISD::STRICT_FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
33260 case ISD::FP_TO_BF16: return LowerFP_TO_BF16(Op, DAG);
33261 case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG);
33262 case ISD::STORE: return LowerStore(Op, Subtarget, DAG);
33264 case ISD::FSUB: return lowerFaddFsub(Op, DAG);
33265 case ISD::FROUND: return LowerFROUND(Op, DAG);
33267 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
33268 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
33269 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
33271 case ISD::LLRINT: return LowerLRINT_LLRINT(Op, DAG);
33274 case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG);
33275 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
33276 case ISD::SELECT: return LowerSELECT(Op, DAG);
33277 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
33278 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
33279 case ISD::VASTART: return LowerVASTART(Op, DAG);
33280 case ISD::VAARG: return LowerVAARG(Op, DAG);
33281 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
33282 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
33284 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
33285 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
33286 case ISD::ADDROFRETURNADDR: return LowerADDROFRETURNADDR(Op, DAG);
33287 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
33289 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
33290 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
33291 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
33292 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
33293 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
33295 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
33296 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
33297 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
33298 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
33299 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
33300 case ISD::GET_FPENV_MEM: return LowerGET_FPENV_MEM(Op, DAG);
33301 case ISD::SET_FPENV_MEM: return LowerSET_FPENV_MEM(Op, DAG);
33302 case ISD::RESET_FPENV: return LowerRESET_FPENV(Op, DAG);
33304 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG);
33306 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, Subtarget, DAG);
33307 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
33309 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG);
33311 case ISD::ROTR: return LowerRotate(Op, Subtarget, DAG);
33314 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
33318 case ISD::USUBO: return LowerXALUO(Op, DAG);
33320 case ISD::UMULO: return LowerMULO(Op, Subtarget, DAG);
33321 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
33322 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
33326 case ISD::USUBO_CARRY: return LowerADDSUBO_CARRY(Op, DAG);
33328 case ISD::SUB: return lowerAddSub(Op, DAG, Subtarget);
33332 case ISD::SSUBSAT: return LowerADDSAT_SUBSAT(Op, DAG, Subtarget);
33336 case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG);
33341 return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG);
33342 case ISD::ABS: return LowerABS(Op, Subtarget, DAG);
33344 case ISD::ABDU: return LowerABD(Op, Subtarget, DAG);
33345 case ISD::AVGCEILU: return LowerAVG(Op, Subtarget, DAG);
33346 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
33347 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG);
33348 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG);
33349 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG);
33350 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG);
33352 case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION(Op, DAG);
33353 case ISD::ADDRSPACECAST: return LowerADDRSPACECAST(Op, DAG);
33354 case X86ISD::CVTPS2PH: return LowerCVTPS2PH(Op, DAG);
33355 case ISD::PREFETCH: return LowerPREFETCH(Op, Subtarget, DAG);
33364 SelectionDAG &DAG) const {
33371 N->dump(&DAG);
33377 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
33379 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
33380 Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo);
33381 Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi);
33382 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33389 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 1);
33391 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
33392 Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other},
33394 Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other},
33396 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
33398 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33404 Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG));
33410 KnownBits Known = DAG.computeKnownBits(N->getOperand(0));
33414 SDValue Op = DAG.getNode(ISD::SRL, dl, MVT::i64, N->getOperand(0),
33415 DAG.getShiftAmountConstant(TZ, MVT::i64, dl));
33416 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Op);
33417 Op = DAG.getNode(ISD::CTPOP, dl, MVT::i32, Op);
33418 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Op);
33424 DAG.getMachineFunction().getFunction().hasFnAttribute(
33428 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));
33429 Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);
33432 Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);
33433 Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,
33434 DAG.getVectorIdxConstant(0, dl));
33435 Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);
33442 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33447 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
33448 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
33449 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
33450 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
33452 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
33454 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
33461 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33465 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
33466 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
33467 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);
33470 SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);
33471 Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1});
33472 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi,
33473 DAG.getVectorIdxConstant(0, dl));
33476 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
33481 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT));
33484 HiCmp = DAG.getConstant(0, dl, VT);
33486 SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
33489 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
33490 DAG.getUNDEF(VT));
33503 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33507 EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
33510 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
33514 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
33516 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
33518 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
33520 SDValue Res = DAG.getNode(Opc, dl, WideVT, InVec0, InVec1);
33534 SDValue UNDEF = DAG.getUNDEF(VT);
33535 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
33537 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
33541 Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
33544 Res = DAG.getNode(Opc, dl, MVT::v4f32, LHS, RHS);
33556 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33564 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
33566 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
33567 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
33568 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
33569 SDValue Res = DAG.getNode(Opc, dl, ResVT, N0, N1);
33575 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
33581 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
33587 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
33598 if (SDValue Src = matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG,
33601 truncateVectorWithPACK(PackOpcode, VT, Src, dl, DAG, Subtarget)) {
33602 Res = widenSubVector(WidenVT, Res, false, Subtarget, DAG, dl);
33616 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl, 128);
33619 WidenIn = DAG.getBitcast(WidenVT, WidenIn);
33621 DAG.getVectorShuffle(WidenVT, dl, WidenIn, WidenIn, TruncMask));
33632 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
33637 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,
33638 DAG.getUNDEF(MVT::v4i64));
33639 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
33644 getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector &&
33649 std::tie(Lo, Hi) = DAG.SplitVector(In, dl);
33651 Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
33652 Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
33653 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
33667 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl,
33669 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, WidenVT, WidenIn));
33688 assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
33695 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
33698 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
33699 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
33703 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
33705 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
33706 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
33708 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
33710 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33719 if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger)
33721 InVT = getTypeToTransformTo(*DAG.getContext(), InVT);
33727 In = DAG.getNode(Opc, dl, InVT, In);
33733 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
33736 SDValue Lo = getEXTEND_VECTOR_INREG(Opc, dl, LoVT, In, DAG);
33745 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
33746 Hi = getEXTEND_VECTOR_INREG(Opc, dl, HiVT, Hi, DAG);
33748 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
33766 Res = DAG.getNode(X86ISD::FP_TO_SINT_SAT, dl, MVT::v4i32, Op);
33768 Res = DAG.getNode(X86ISD::FP_TO_UINT_SAT, dl, MVT::v4i32, Op);
33789 DAG.getNode(Opc, dl, {VT, MVT::Other},
33790 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
33795 DAG.getNode(Opc, dl, VT, DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
33811 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
33814 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
33820 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
33824 Res = DAG.getNode(Opc, dl, ResVT, Src);
33830 Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res);
33835 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT));
33837 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
33848 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33858 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},
33862 Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
33867 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
33868 DAG.getUNDEF(MVT::v2i32));
33870 Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl,
33872 DAG.getValueType(VT.getVectorElementType()));
33875 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
33876 DAG.getVectorIdxConstant(0, dl));
33879 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
33885 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
33887 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
33899 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33904 expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget);
33923 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,
33924 DAG.getConstantFP(0.0, dl, MVT::v2f64));
33930 Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
33934 Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
33945 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
33946 DAG.getConstantFP(0.0, dl, MVT::v2f32));
33947 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
33978 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
33979 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
33980 DAG.getConstantFP(0.0, dl, VecInVT), Src,
33984 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
33985 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res);
33988 Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res);
33989 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);
33998 SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain);
34005 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
34014 if (SDValue V = LRINT_LLRINTHelper(N, DAG))
34033 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
34034 IsStrict ? DAG.getConstant(0, dl, MVT::v2i32)
34035 : DAG.getUNDEF(MVT::v2i32));
34039 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
34045 Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src));
34056 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
34062 Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));
34068 SDValue Zero = DAG.getConstant(0, dl, SrcVT);
34069 SDValue One = DAG.getConstant(1, dl, SrcVT);
34070 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
34071 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
34072 DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
34073 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
34074 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
34075 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
34077 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
34078 SignSrc, DAG.getVectorIdxConstant(i, dl));
34081 DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},
34084 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt);
34086 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
34089 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
34091 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},
34095 Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);
34097 IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);
34099 DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});
34100 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
34116 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
34117 DAG.getConstant(0, dl, MVT::v2i32));
34118 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
34126 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
34127 SDValue VBias = DAG.getConstantFP(
34129 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
34130 DAG.getBitcast(MVT::v2i64, VBias));
34131 Or = DAG.getBitcast(MVT::v2f64, Or);
34133 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
34135 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
34142 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
34143 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
34157 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)
34158 : DAG.getUNDEF(MVT::v2f32);
34159 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext);
34167 V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
34170 V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd);
34172 Results.push_back(DAG.getBitcast(MVT::v8f16, V));
34181 V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other},
34184 V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src);
34202 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)
34203 : DAG.getUNDEF(MVT::v2f16);
34204 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);
34206 V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other},
34209 V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V);
34221 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
34224 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
34227 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
34231 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
34235 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
34241 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
34252 DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT);
34253 cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
34256 DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX,
34260 DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT);
34262 DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
34272 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
34278 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO);
34280 swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL,
34285 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO);
34288 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
34291 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
34296 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
34298 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
34299 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
34301 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
34311 DAG.getMachineFunction().getFunction().hasFnAttribute(
34318 SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(),
34320 SDValue ResL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34321 DAG.getVectorIdxConstant(0, dl));
34322 SDValue ResH = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34323 DAG.getVectorIdxConstant(1, dl));
34324 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, N->getValueType(0),
34335 SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
34337 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
34340 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
34341 DAG.getVectorIdxConstant(0, dl));
34349 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
34350 DAG.getVectorIdxConstant(0, dl));
34351 Res = DAG.getBitcast(MVT::i64, Res);
34359 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
34361 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
34370 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
34373 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
34375 Chain = DAG.getMemIntrinsicNode(
34376 X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
34382 Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);
34418 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
34419 Lo = DAG.getBitcast(MVT::i32, Lo);
34420 Hi = DAG.getBitcast(MVT::i32, Hi);
34421 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
34429 assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
34431 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
34432 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
34434 Res = DAG.getBitcast(WideVT, Res);
34449 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
34451 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
34454 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
34456 DAG.getUNDEF(VT));
34460 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
34461 DAG.getUNDEF(MVT::v2i1));
34462 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
34466 SDValue Res = DAG.getMemIntrinsicNode(
34467 X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
34481 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
34488 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
34493 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
34494 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
34495 Res = DAG.getBitcast(WideVT, Res);
34501 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
34503 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
34510 SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
34519 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
34529 SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0));
34530 Split = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Split,
34532 Split = DAG.getBitcast(MVT::f16, Split);
38039 EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits);
38040 EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT,
38043 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT,
38044 Op.getOperand(1), TLO.DAG.getValueType(ExtVT));
38046 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC);
38094 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
38095 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
38102 const SelectionDAG &DAG, unsigned Depth) {
38106 Known = DAG.computeKnownBits(RHS, DemandedSrcElts, Depth + 1);
38107 Known2 = DAG.computeKnownBits(LHS, DemandedSrcElts, Depth + 1);
38119 const SelectionDAG &DAG,
38129 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1);
38130 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1);
38131 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1);
38132 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1);
38141 const SelectionDAG &DAG,
38152 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1);
38153 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1);
38154 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1);
38155 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1);
38163 const SelectionDAG &DAG,
38172 [&DAG, Depth, KnownBitsFunc](SDValue Op, APInt &DemandedEltsOp) {
38174 DAG.computeKnownBits(Op, DemandedEltsOp, Depth + 1),
38175 DAG.computeKnownBits(Op, DemandedEltsOp << 1, Depth + 1));
38190 const SelectionDAG &DAG,
38208 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38209 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38217 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38224 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38234 !DAG.isKnownNeverZero(Op.getOperand(1), Depth + 1)) {
38236 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38255 Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
38275 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38302 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
38306 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
38323 KnownBits KnownIdx = DAG.computeKnownBits(Idx, DemandedElts, Depth + 1);
38325 Known = DAG.computeKnownBits(Src, Depth + 1);
38331 Known = DAG.computeKnownBits(Src, Depth + 1);
38339 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38340 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38347 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38348 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38357 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38358 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38370 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
38376 DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38378 DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38397 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth);
38407 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth);
38412 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38413 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38421 Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
38425 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
38447 Known = DAG.computeKnownBits(Op0, Depth + 1);
38456 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38457 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
38466 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
38549 Op, DemandedElts, Depth, DAG,
38568 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth);
38580 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth);
38592 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
38642 DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
38651 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
38667 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
38690 DAG.ComputeNumSignBits(BC0, Depth + 1) == 64 &&
38691 DAG.ComputeNumSignBits(BC1, Depth + 1) == 64)
38694 return DAG.ComputeNumSignBits(V, Elts, Depth + 1);
38712 return DAG.ComputeNumSignBits(Src, Depth + 1);
38721 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
38732 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
38754 DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
38757 DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
38762 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
38764 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
38807 DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1);
38828 SelectionDAG &DAG) {
38833 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
38835 return DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, SDLoc(LN), Tys, Ops, MemVT,
38845 SDValue V1, const SelectionDAG &DAG,
38872 DAG.ComputeNumSignBits(V1) == MaskEltSize;
38902 Shuffle = DAG.getOpcode_EXTEND_VECTOR_INREG(Shuffle);
38928 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG, V1)) {
38933 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
38938 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, DAG, V1)) {
38947 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
38952 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
38958 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, DAG,
38969 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
38977 {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, DAG, V1)) {
38984 {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, DAG, V1)) {
39000 const SelectionDAG &DAG,
39149 SelectionDAG &DAG, const X86Subtarget &Subtarget,
39157 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG) &&
39160 V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);
39165 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}, DAG) &&
39172 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}, DAG) &&
39179 if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG) &&
39186 DAG) &&
39198 if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
39206 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2, 4, 6}, DAG) &&
39210 unsigned MinLZV1 = DAG.computeKnownBits(V1).countMinLeadingZeros();
39211 unsigned MinLZV2 = DAG.computeKnownBits(V2).countMinLeadingZeros();
39226 if (DAG.ComputeNumSignBits(V1) > 48 && DAG.ComputeNumSignBits(V2) > 48) {
39241 if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
39285 if (DAG.MaskedVectorIsZero(V1, DemandedZeroV1) &&
39286 DAG.MaskedVectorIsZero(V2, DemandedZeroV2)) {
39295 auto computeKnownBitsElementWise = [&DAG](SDValue V) {
39300 KnownBits PeepholeKnown = DAG.computeKnownBits(V, Mask);
39345 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
39402 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
39403 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
39409 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
39410 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
39423 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
39437 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
39438 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
39459 return DAG.getUNDEF(MaskVT);
39463 return getZeroVector(MaskVT, Subtarget, DAG, DL);
39495 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
39507 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
39524 SelectionDAG &DAG,
39538 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits());
39540 Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits());
39541 return DAG.getBitcast(VT, Op);
39548 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
39564 bool OptForSize = DAG.shouldOptForSize();
39587 DAG.isSplatValue(V1, /*AllowUndefs*/ false)) {
39602 if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, DAG, V1,
39622 Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
39623 return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
39634 SelectionDAG &DAG) {
39637 SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
39653 return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
39656 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
39675 if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG))
39676 return DAG.getBitcast(RootVT, V);
39690 Res = extract128BitVector(Res, Mask[0] * (NumRootElts / 2), DAG, DL);
39691 return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL,
39704 Hi = extractSubVector(Hi, 0, DAG, DL, 128);
39705 return insertSubVector(Lo, Hi, NumRootElts / 2, DAG, DL, 128);
39720 return DAG.getNode(
39722 DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));
39739 return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
39742 DAG.getTargetConstant(PermMask, DL, MVT::i8));
39770 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39811 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
39812 return DAG.getBitcast(RootVT, Res);
39818 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
39819 return DAG.getBitcast(RootVT, Res);
39825 DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) &&
39831 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
39832 return DAG.getBitcast(RootVT, Res);
39836 AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
39843 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
39844 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
39845 return DAG.getBitcast(RootVT, Res);
39854 !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG)) {
39858 DAG) &&
39862 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
39865 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
39866 return DAG.getBitcast(RootVT, Res);
39870 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}, DAG) &&
39876 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
39879 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
39880 return DAG.getBitcast(RootVT, Res);
39887 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
39894 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
39895 return DAG.getBitcast(RootVT, Res);
39901 AllowIntDomain, NewV1, NewV2, DL, DAG,
39908 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
39909 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
39910 return DAG.getBitcast(RootVT, Res);
39925 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
39926 DAG.getTargetConstant(BitLen, DL, MVT::i8),
39927 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
39928 return DAG.getBitcast(RootVT, Res);
39936 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
39937 DAG.getTargetConstant(BitLen, DL, MVT::i8),
39938 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
39939 return DAG.getBitcast(RootVT, Res);
39955 Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
39957 Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
39958 return DAG.getBitcast(RootVT, Res);
39979 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
39980 Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
39981 return DAG.getBitcast(RootVT, Res);
40015 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
40017 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
40018 return DAG.getBitcast(RootVT, Res);
40029 V2 = DAG.getUNDEF(MaskVT);
40030 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
40031 return DAG.getBitcast(RootVT, Res);
40052 V2 = getZeroVector(MaskVT, Subtarget, DAG, DL);
40053 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
40054 return DAG.getBitcast(RootVT, Res);
40061 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG,
40079 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
40080 return DAG.getBitcast(RootVT, Res);
40104 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
40108 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
40109 return DAG.getBitcast(RootVT, Res);
40121 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
40124 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
40126 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
40127 return DAG.getBitcast(RootVT, Res);
40159 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
40160 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
40161 DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
40162 return DAG.getBitcast(RootVT, Res);
40180 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
40184 PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
40189 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
40193 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
40194 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
40195 return DAG.getBitcast(RootVT, Res);
40212 VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));
40216 VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
40220 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
40225 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
40226 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
40227 return DAG.getBitcast(RootVT, Res);
40234 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget))
40254 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
40255 return DAG.getBitcast(RootVT, Res);
40273 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
40302 if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) &&
40402 AllowVariablePerLaneMask, DAG, Subtarget)) {
40404 extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
40405 return DAG.getBitcast(RootVT, WideShuffle);
40415 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
40453 (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) {
40465 return DAG.getUNDEF(VT0);
40467 return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL);
40479 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1);
40480 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3);
40481 return DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
40509 LHS = DAG.getBitcast(SrcVT, LHS);
40510 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
40511 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
40515 Res = DAG.getBitcast(ShuffleVT, Res);
40516 return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res,
40517 getV4X86ShuffleImm8ForMask(PostMask, DL, DAG));
40580 shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
40586 SDValue Undef = DAG.getUNDEF(SrcVT);
40587 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
40593 return DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
40609 SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL);
40610 SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL);
40611 SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1);
40612 return widenSubVector(Res, false, Subtarget, DAG, DL, 256);
40625 SelectionDAG &DAG, const SDLoc &DL,
40645 bool IsOptimizingSize = DAG.shouldOptForSize();
40690 return getZeroVector(VT, Subtarget, DAG, DL);
40700 if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
40703 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
40704 return DAG.getBitcast(VT, CstOp);
40748 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
40805 OpZero, DAG, Depth, false)) {
41010 return DAG.getUNDEF(RootVT);
41012 return getZeroVector(RootVT, Subtarget, DAG, DL);
41015 return getOnesVector(RootVT, DAG, DL);
41047 HasVariableMask, AllowCrossLaneVar, AllowPerLaneVar, DAG,
41055 RootVT, Ops, Mask, HasVariableMask, DAG, DL, Subtarget))
41075 Ops, Mask, RootSizeInBits, DL, DAG, Subtarget))
41076 return DAG.getBitcast(RootVT, HOp);
41114 DAG.getTargetLoweringInfo().SimplifyMultipleUseDemandedVectorElts(
41115 Op, OpScaledDemandedElts, DAG))
41128 Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op),
41158 AllowVariablePerLaneMask, DAG, Subtarget))
41177 AllowVariablePerLaneMask, DAG, Subtarget);
41181 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
41186 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
41239 SelectionDAG &DAG) {
41331 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
41332 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
41339 V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
41347 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
41353 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
41358 V = DAG.getBitcast(N.getValueType(), V);
41367 SelectionDAG &DAG) {
41373 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
41379 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
41384 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
41385 DAG.getTargetConstant(Imm, DL, MVT::i8));
41392 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP,
41393 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
41402 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP,
41403 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
41405 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1,
41406 DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));
41408 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP,
41409 DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));
41422 const APInt &DemandedElts, SelectionDAG &DAG,
41507 DAG.getVectorShuffle(VT, DL, DAG.getBitcast(VT, Ops0[0]),
41508 DAG.getBitcast(VT, Ops1[0]), NewBlendMask);
41509 return DAG.getVectorShuffle(VT, DL, NewBlend, DAG.getUNDEF(VT),
41526 static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
41528 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41532 auto IsMergeableWithShuffle = [Opc, &DAG](SDValue Op, bool FoldShuf = true,
41547 DAG.isSplatValue(Op, /*AllowUndefs*/ false);
41586 Op00 = DAG.getBitcast(ShuffleVT, Op00);
41587 Op01 = DAG.getBitcast(ShuffleVT, Op01);
41589 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1));
41590 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1));
41592 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00);
41593 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01);
41595 return DAG.getBitcast(ShuffleVT,
41596 DAG.getNode(SrcOpcode, DL, OpVT,
41597 DAG.getBitcast(OpVT, LHS),
41598 DAG.getBitcast(OpVT, RHS)));
41604 SDValue Op00 = DAG.getBitcast(ShuffleVT, N0.getOperand(0));
41607 ? DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1))
41608 : DAG.getNode(Opc, DL, ShuffleVT, Op00);
41609 Res = DAG.getBitcast(N0.getOperand(0).getValueType(), Res);
41610 return DAG.getBitcast(ShuffleVT, DAG.getNode(SrcOpcode, DL, OpVT, Res));
41650 Op00 = DAG.getBitcast(ShuffleVT, Op00);
41651 Op10 = DAG.getBitcast(ShuffleVT, Op10);
41652 Op01 = DAG.getBitcast(ShuffleVT, Op01);
41653 Op11 = DAG.getBitcast(ShuffleVT, Op11);
41655 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
41656 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2));
41658 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
41659 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11);
41662 return DAG.getBitcast(ShuffleVT,
41663 DAG.getNode(SrcOpcode, DL, OpVT,
41664 DAG.getBitcast(OpVT, LHS),
41665 DAG.getBitcast(OpVT, RHS)));
41675 Op00 = DAG.getBitcast(ShuffleVT, Op00);
41676 Op10 = DAG.getBitcast(ShuffleVT, Op10);
41678 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
41680 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
41683 return DAG.getBitcast(
41685 DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res)));
41698 DAG.getNode(Opc, DL, OpSrcVT, N0.getOperand(0), N1.getOperand(0));
41699 return DAG.getBitcast(ShuffleVT,
41700 DAG.getNode(SrcOpcode, DL, OpDstVT, Res));
41711 SelectionDAG &DAG,
41731 DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2));
41732 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res);
41733 return DAG.getBitcast(VT, Res);
41750 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS,
41752 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1));
41753 return DAG.getBitcast(VT, Res);
41762 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
41768 SelectionDAG &DAG,
41777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41779 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
41790 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
41791 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
41793 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
41820 /*AllowPerLaneVarMask*/ true, DAG, Subtarget))
41821 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
41822 DAG.getBitcast(SrcVT, Res));
41831 BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) {
41832 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
41834 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
41849 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
41854 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
41855 extract128BitVector(Src, 0, DAG, DL));
41860 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
41868 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
41877 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
41886 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41889 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
41895 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41898 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
41899 DAG.getVectorIdxConstant(0, DL));
41918 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41920 SDValue BcastLd = DAG.getMemIntrinsicNode(
41925 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41936 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41939 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
41942 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41961 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41962 SDValue Ptr = DAG.getMemBasePlusOffset(
41965 SDValue BcastLd = DAG.getMemIntrinsicNode(
41971 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41982 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41985 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
41988 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
42001 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
42003 SDValue BcastLd = DAG.getMemIntrinsicNode(
42008 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
42024 narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) {
42026 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
42038 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
42041 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops,
42044 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
42058 if (DAG.MaskedValueIsZero(In, Mask)) {
42059 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
42061 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
42062 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
42063 return DAG.getBitcast(VT, Movl);
42074 Type *ScalarTy = ScalarVT.getTypeForEVT(*DAG.getContext());
42080 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
42081 SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
42083 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
42085 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment,
42103 In = DAG.getBitcast(SubVT, In);
42104 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In);
42105 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
42106 getZeroVector(VT, Subtarget, DAG, DL), Movl,
42128 return DAG.getBitcast(
42129 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
42131 DAG.getTargetConstant(NewBlendMask.getZExtValue(),
42158 /*HasVariableMask=*/true, DAG, DL, Subtarget)) {
42159 SDValue NewLHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
42161 SDValue NewRHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
42163 return DAG.getNode(X86ISD::BLENDI, DL, VT,
42164 DAG.getBitcast(VT, NewLHS),
42165 DAG.getBitcast(VT, NewRHS), N.getOperand(2));
42186 if (getTargetShuffleInputs(Sub, SubOps, SubMask, DAG, 0, false) &&
42192 Ops[i] = DAG.getBitcast(VT, SubOps[0]);
42200 Ops.push_back(getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
42201 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops);
42216 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
42217 return DAG.getBitcast(VT, Res);
42233 collectConcatOps(LHS.getNode(), LHSOps, DAG) && LHSOps.size() == 2) {
42234 NewLHS = widenSubVector(LHSOps[1], false, Subtarget, DAG, DL, 512);
42238 collectConcatOps(RHS.getNode(), RHSOps, DAG) && RHSOps.size() == 2) {
42239 NewRHS = widenSubVector(RHSOps[1], false, Subtarget, DAG, DL, 512);
42243 return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS,
42245 DAG.getTargetConstant(Mask, DL, MVT::i8));
42257 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT,
42258 DAG.getBitcast(SrcVT, LHS),
42259 DAG.getBitcast(SrcVT, RHS),
42265 if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
42275 if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2)
42289 SubLo = DAG.getBitcast(SubVT, SubLo);
42290 SubHi = DAG.getBitcast(SubVT, SubHi);
42291 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi);
42314 SDValue Res = DAG.getNode(Opcode, DL, VT,
42315 DAG.getBitcast(VT, V.getOperand(0)), N1);
42316 Res = DAG.getBitcast(InnerVT, Res);
42317 Res = DAG.getNode(V.getOpcode(), DL, InnerVT, Res, V.getOperand(1));
42318 return DAG.getBitcast(VT, Res);
42348 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
42349 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
42350 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
42351 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
42352 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
42353 return DAG.getNode(Opcode, DL, VT, N0, SclVec);
42370 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
42371 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
42375 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
42376 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
42387 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
42388 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
42395 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
42396 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
42442 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
42443 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
42452 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
42455 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
42456 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
42458 DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
42459 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
42474 combineConcatVectorOps(DL, WideVT, Ops, DAG, DCI, Subtarget)) {
42475 SDValue Mask = widenSubVector(N.getOperand(1), false, Subtarget, DAG,
42477 SDValue Perm = DAG.getNode(X86ISD::VPERMV, DL, WideVT, Mask, ConcatSrc);
42478 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Perm,
42479 DAG.getVectorIdxConstant(0, DL));
42493 SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL,
42495 return DAG.getNode(X86ISD::VPERMV, DL, VT, NewMask, N.getOperand(0));
42507 combineConcatVectorOps(DL, VT, Ops, DAG, DCI, Subtarget)) {
42510 SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL,
42512 return DAG.getNode(X86ISD::VPERMV, DL, VT, NewMask, ConcatSrc);
42520 getConstVector(Mask, MaskVT, DAG, DL, /*IsMask=*/true);
42521 return DAG.getNode(X86ISD::VPERMV3, DL, VT, N.getOperand(2), NewMask,
42553 V = DAG.getBitcast(DVT, V);
42554 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
42555 getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
42556 return DAG.getBitcast(VT, V);
42585 V = DAG.getBitcast(VT, D.getOperand(0));
42586 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
42596 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DL, DAG))
42646 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
42650 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42707 SelectionDAG &DAG) {
42715 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42741 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
42749 SelectionDAG &DAG) {
42750 if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG))
42755 if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
42762 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
42764 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
42782 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
42789 SelectionDAG &DAG,
42825 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
42827 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
42833 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
42835 if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0)))
42860 HalfIdx2, false, DAG, /*UseConcat*/ true);
42863 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
42867 if (SDValue V = narrowShuffle(Shuf, DAG))
42874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42877 combineShuffleToAddSubOrFMAddSub(N, dl, Subtarget, DAG))
42882 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
42891 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, dl, DAG, Subtarget))
42896 if (SDValue Shuffle = combineTargetShuffle(Op, dl, DAG, DCI, Subtarget))
42904 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42917 if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl))
42984 SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
42985 SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
42986 SDValue NewMask = TLO.DAG.getLoad(
42987 BCVT, DL, TLO.DAG.getEntryNode(), LegalCV,
42988 MachinePointerInfo::getConstantPool(TLO.DAG.getMachineFunction()),
42990 return TLO.CombineTo(Mask, TLO.DAG.getBitcast(Mask.getValueType(), NewMask));
43059 LHS, DemandedSrcElts, TLO.DAG, Depth + 1);
43061 RHS, DemandedSrcElts, TLO.DAG, Depth + 1);
43066 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
43108 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
43113 Src, DemandedElts, TLO.DAG, Depth + 1))
43115 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
43134 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
43180 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
43182 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
43219 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
43221 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
43284 TLO.DAG, Depth + 1);
43286 TLO.DAG, Depth + 1);
43291 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
43332 TLO.DAG, Depth + 1);
43334 TLO.DAG, Depth + 1);
43339 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
43369 TLO.DAG, Depth + 1);
43371 TLO.DAG, Depth + 1);
43376 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
43400 DemandedElts, TLO.DAG, Subtarget, SDLoc(Op)))
43430 if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1))
43441 SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(),
43443 SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt);
43444 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec));
43454 Src = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op), VT, Src);
43456 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
43470 Src, SrcElts, TLO.DAG, Depth + 1))
43471 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
43512 Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
43513 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
43515 SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src);
43516 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
43517 TLO.DAG, DL, ExtSizeInBits));
43522 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
43524 SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other);
43526 SDValue Bcst = TLO.DAG.getMemIntrinsicNode(
43529 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
43531 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
43532 TLO.DAG, DL, ExtSizeInBits));
43541 TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(),
43543 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
43545 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0,
43546 TLO.DAG, DL, ExtSizeInBits));
43549 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
43552 getBROADCAST_LOAD(Opc, DL, BcstVT, MemVT, MemIntr, 0, TLO.DAG))
43554 insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0,
43555 TLO.DAG, DL, ExtSizeInBits));
43572 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
43574 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
43575 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43577 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
43588 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
43589 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43590 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
43602 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL));
43606 extractSubVector(Op.getOperand(SrcIdx), EltIdx, TLO.DAG, DL, 128);
43607 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43609 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
43625 SDValue ExtOp = TLO.DAG.getNode(
43626 Opc, DL, ExtVT, extractSubVector(SrcOp, 0, TLO.DAG, DL, SrcExtSize));
43627 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43629 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
43677 Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
43684 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
43685 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
43687 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
43696 if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
43704 OpZero, TLO.DAG, Depth, false))
43726 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
43731 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
43735 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src]));
43777 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
43820 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(LHS))
43822 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(RHS))
43837 SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT);
43838 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask));
43843 LHS, DemandedMaskLHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
43845 RHS, DemandedMaskRHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
43850 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS));
43899 SDValue NewShift = TLO.DAG.getNode(
43901 TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
43908 TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
43926 Op0, DemandedMask, OriginalDemandedElts, TLO.DAG, Depth + 1)) {
43928 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedOp0, Op1);
43957 Op0, DemandedMask, OriginalDemandedElts, TLO.DAG, Depth + 1)) {
43959 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedOp0, Op1);
43983 TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts);
44005 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));
44014 Op0, DemandedMask, OriginalDemandedElts, TLO.DAG, Depth + 1)) {
44016 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedOp0, Op1);
44029 Sel, SignMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
44031 LHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
44033 RHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
44039 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT,
44059 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
44073 Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1))
44075 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1)));
44132 Op.getOperand(0), SignMask, DemandedLHS, TLO.DAG, Depth + 1);
44134 Op.getOperand(1), SignMask, DemandedRHS, TLO.DAG, Depth + 1);
44138 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1));
44159 TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src);
44162 TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc);
44163 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst));
44182 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
44187 SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src));
44188 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
44215 Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
44216 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
44262 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0,
44263 TLO.DAG.getConstant(MaskedVal1, DL, VT)));
44294 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
44335 SelectionDAG &DAG, unsigned Depth) const {
44358 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
44383 KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1);
44395 KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1);
44396 KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1);
44411 ShuffleUndef, ShuffleZero, DAG, Depth, false)) {
44421 return DAG.getUNDEF(VT);
44423 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op));
44445 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]);
44450 Op, DemandedBits, DemandedElts, DAG, Depth);
44454 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
44478 !DAG.isGuaranteedNotToBeUndefOrPoison(
44487 Op, DemandedElts, DAG, PoisonOnly, Depth);
44491 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
44525 Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
44531 const SelectionDAG &DAG,
44544 DAG, Depth);
44591 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
44604 return DAG.getBitcast(MVT::v4f32, Op0);
44613 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
44614 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
44616 return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,
44625 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
44632 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
44636 return DAG.getNode(
44638 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
44639 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
44642 return DAG.getSelect(
44644 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
44645 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
44656 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
44666 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
44667 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,
44668 DAG.getBitcast(MVT::v4f32, V));
44669 return DAG.getZExtOrTrunc(V, DL, VT);
44702 if (collectConcatOps(Src.getNode(), SubSrcOps, DAG) &&
44709 *DAG.getContext(), LowerOp.getValueType().getVectorMinNumElements());
44710 if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) {
44711 EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
44712 return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V));
44785 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
44786 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
44789 V = getPMOVMSKB(DL, V, DAG, Subtarget);
44792 V = widenSubVector(V, false, Subtarget, DAG, DL, 256);
44793 V = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, V);
44795 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
44799 EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
44800 V = DAG.getZExtOrTrunc(V, DL, IntVT);
44801 return DAG.getBitcast(VT, V);
44805 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
44818 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth());
44819 return DAG.getConstant(Imm, SDLoc(Op), IntVT);
44822 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
44859 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0),
44860 DAG.getBitcast(DstVT, RHS));
44864 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
44865 DAG.getBitcast(DstVT, LHS), RHS.getOperand(0));
44870 RHS = combinevXi1ConstantToInteger(RHS, DAG);
44871 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
44872 DAG.getBitcast(DstVT, LHS), RHS);
44878 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
44887 return DAG.getUNDEF(MVT::x86mmx);
44890 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
44891 V = DAG.getBitcast(MVT::v2i64, V);
44892 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
44894 V = DAG.getBitcast(MVT::i32, V);
44896 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
44898 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
44904 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44909 return DAG.getUNDEF(MVT::x86mmx);
44916 Splat = DAG.getNode(
44918 DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL,
44919 TLI.getPointerTy(DAG.getDataLayout())),
44924 return DAG.getNode(
44926 DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL,
44927 TLI.getPointerTy(DAG.getDataLayout())),
44928 Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
44943 SDValue Intrin = DAG.getTargetConstant(
44944 IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout()));
44946 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
44959 SelectionDAG &DAG,
44965 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44973 return DAG.getBitcast(VT, Src);
44979 return DAG.getConstant(0, DL, VT);
44981 return DAG.getAllOnesConstant(DL, VT);
44988 EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits());
44990 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
44992 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0,
44993 DAG.getVectorIdxConstant(0, DL));
45000 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
45003 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
45005 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
45006 Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT)
45007 : DAG.getConstant(0, DL, VT),
45008 N0, DAG.getVectorIdxConstant(0, DL));
45015 if (SDValue N0 = combineBitcastToBoolVector(VT, V.getOperand(0), DL, DAG,
45017 if (SDValue N1 = combineBitcastToBoolVector(VT, V.getOperand(1), DL, DAG,
45019 return DAG.getNode(Opc, DL, VT, N0, N1);
45030 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget,
45032 return DAG.getNode(
45034 DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));
45041 if (SDNode *Alt = DAG.getNodeIfExists(ISD::BITCAST, DAG.getVTList(VT), {V}))
45047 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
45053 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45063 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
45070 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);
45071 N0 = DAG.getBitcast(MVT::v8i1, N0);
45072 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,
45073 DAG.getVectorIdxConstant(0, dl));
45093 Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT));
45094 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
45095 N0 = DAG.getBitcast(MVT::i8, N0);
45096 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
45101 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
45103 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
45104 N0 = DAG.getBitcast(MVT::i8, N0);
45105 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
45113 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget))
45127 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
45128 DAG.getBitcast(MVT::i16, N0.getOperand(0)));
45146 SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
45149 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
45151 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
45152 return DAG.getBitcast(VT, ResNode);
45169 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
45170 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
45174 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
45190 N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)
45191 : DAG.getZExtOrTrunc(N00, dl, MVT::i32);
45192 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
45202 return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget);
45210 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
45211 DAG.getBitcast(MVT::v2i64, N00));
45217 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
45218 DAG.getUNDEF(MVT::v2i32));
45219 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
45220 DAG.getBitcast(MVT::v2i64, Res));
45229 return combinevXi1ConstantToInteger(N0, DAG);
45236 return DAG.getConstant(1, SDLoc(N0), VT);
45238 return DAG.getConstant(0, SDLoc(N0), VT);
45265 MovmskIn = DAG.getBitcast(IntVT, MovmskIn);
45268 SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
45269 DAG.getConstant(0, dl, IntVT), ISD::SETLT);
45276 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
45278 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops);
45285 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
45320 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
45322 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);
45329 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
45331 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);
45338 static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
45361 DAG.computeKnownBits(Op0).countMaxActiveBits() <= 8) &&
45362 (IsFreeTruncation(Op1) && DAG.ComputeMaxSignificantBits(Op1) <= 8))
45389 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
45395 LHS = DAG.getZExtOrTrunc(LHS, DL, Vi8VT);
45396 RHS = DAG.getSExtOrTrunc(RHS, DL, Vi8VT);
45412 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT));
45415 SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45417 SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45421 auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45424 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
45427 SDValue Zero = DAG.getConstant(0, DL, DpVT);
45429 return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1},
45435 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
45445 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
45448 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45450 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
45453 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45456 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
45459 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
45465 static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
45477 SDValue Src = DAG.matchBinOpReduction(
45493 std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
45495 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
45506 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
45508 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
45510 Mask = DAG.getAllOnesConstant(DL, SrcVT);
45513 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
45520 SDValue Upper = DAG.getVectorShuffle(
45521 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
45523 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
45527 MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
45528 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
45529 MinPos = DAG.getBitcast(SrcVT, MinPos);
45532 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
45534 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
45535 DAG.getVectorIdxConstant(0, DL));
45539 static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
45553 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
45555 Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR});
45569 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45570 LLVMContext &Ctx = *DAG.getContext();
45583 SDValue LHS = DAG.getFreeze(Match.getOperand(0));
45584 SDValue RHS = DAG.getFreeze(Match.getOperand(1));
45587 DAG, X86CC))
45588 return DAG.getNode(ISD::TRUNCATE, DL, ExtractVT,
45589 getSETCC(X86CC, V, DL, DAG));
45595 Movmsk = DAG.getBitcast(MovmskVT, Match);
45600 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
45601 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
45605 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
45609 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
45625 if (DAG.ComputeNumSignBits(Match) != BitWidth)
45630 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
45631 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
45643 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
45644 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
45653 SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk);
45654 return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
45661 CmpC = DAG.getConstant(0, DL, CmpVT);
45665 CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts),
45672 EVT SetccVT = TLI.getSetCCResultType(DAG.getDataLayout(), Ctx, CmpVT);
45673 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
45674 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
45675 return DAG.getNegative(Zext, DL, ExtractVT);
45678 static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
45695 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
45713 if (!detectExtMul(DAG, Root, LHS, RHS))
45719 SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
45735 DAG.getVectorShuffle(DpVT, DL, DP, DAG.getUNDEF(DpVT), Mask);
45736 DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle);
45742 EVT::getVectorVT(*DAG.getContext(), ExtractVT,
45744 DP = DAG.getBitcast(ResVT, DP);
45745 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
45749 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
45767 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
45794 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
45809 DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
45810 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
45816 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
45818 SAD = DAG.getBitcast(ResVT, SAD);
45819 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
45833 const SDLoc &dl, SelectionDAG &DAG,
45838 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45853 DAG, LoadVec->getBasePtr(), VecVT, DAG.getVectorIdxConstant(Idx, dl));
45858 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
45860 DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
45869 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
45905 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType());
45906 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT);
45919 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
45924 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
45942 Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl,
45943 DAG.getShiftAmountConstant(Offset, SclVT, dl));
45945 Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType());
45946 Scl = DAG.getZExtOrTrunc(Scl, dl, VT);
45956 Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
45958 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src),
45965 auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT,
45975 VecVT = EVT::getVectorVT(*DAG.getContext(), VecSVT, NumEltsPerLane);
45976 Vec = extract128BitVector(Vec, LaneIdx, DAG, dl);
45981 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(),
45982 DAG.getBitcast(VecVT, Vec),
45983 DAG.getVectorIdxConstant(Idx, dl));
45988 return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec),
45989 DAG.getTargetConstant(Idx, dl, MVT::i8));
45997 if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
46044 EVT ExtractSVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltBits / Scale);
46045 ExtractVT = EVT::getVectorVT(*DAG.getContext(), ExtractSVT, Mask.size());
46052 return DAG.getUNDEF(VT);
46055 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)
46056 : DAG.getConstant(0, dl, VT);
46061 return DAG.getZExtOrTrunc(V, dl, VT);
46065 N, SrcVT, peekThroughBitcasts(SrcOp), ExtractIdx, dl, DAG, DCI))
46073 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
46096 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
46098 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
46100 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
46119 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
46122 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
46124 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
46126 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
46166 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
46167 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
46177 static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
46186 SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
46208 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
46209 DAG.getConstant(0, DL, MVT::v4i32),
46210 DAG.getBitcast(MVT::i32, V),
46211 DAG.getVectorIdxConstant(0, DL));
46212 return DAG.getBitcast(MVT::v16i8, V);
46214 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
46215 ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
46216 : DAG.getUNDEF(MVT::v4i8));
46218 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
46219 DAG.getUNDEF(MVT::v8i8));
46227 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
46228 SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
46229 SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
46230 Lo = DAG.getBitcast(WideVT, Lo);
46231 Hi = DAG.getBitcast(WideVT, Hi);
46232 Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
46234 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
46235 Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
46239 Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
46240 Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
46243 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
46244 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
46246 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
46247 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
46249 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
46250 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
46252 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
46253 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
46259 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
46260 DAG.getConstant(0, DL, MVT::v16i8));
46261 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
46262 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
46273 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
46275 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
46279 SDValue Hi = DAG.getVectorShuffle(
46282 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
46283 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
46284 getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
46285 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
46286 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
46294 DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
46298 Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
46299 DAG.getUNDEF(MVT::v8i16));
46302 Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
46308 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46311 SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
46312 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
46315 Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
46320 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
46322 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
46327 SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
46328 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
46332 Rdx = DAG.getBitcast(VecVT, Rdx);
46333 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
46337 if (!shouldUseHorizontalOp(true, DAG, Subtarget))
46349 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
46350 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
46351 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
46361 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
46363 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
46370 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
46373 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
46386 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
46389 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
46401 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
46402 return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
46412 SDValue Sub = DAG.getNode(
46414 DAG.getVectorIdxConstant(CIdx->getZExtValue() * NumEltBits, dl));
46415 return DAG.getBitcast(VT, Sub);
46432 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl);
46433 return DAG.getZExtOrTrunc(Scl, dl, VT);
46447 return DAG.getBitcast(VT, InputVector);
46454 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
46460 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
46463 if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
46467 if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
46471 if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
46475 if (SDValue V = combineArithReduction(N, DAG, Subtarget))
46478 if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget, DCI))
46484 dl, DAG, DCI))
46514 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts);
46516 combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) {
46520 SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);
46521 SDValue MaskBit = DAG.getConstant(1, dl, BCVT);
46522 SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx);
46523 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
46524 Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);
46538 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TruncSVT, TruncSrc, EltIdx);
46539 return DAG.getAnyExtOrTrunc(NewExt, dl, VT);
46549 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG,
46590 EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits);
46594 ? DAG.getSplat(BroadcastVT, DL, N00)
46595 : DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
46596 Vec = DAG.getBitcast(VT, Vec);
46602 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
46610 EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT,
46612 Vec = DAG.getBitcast(VT, DAG.getSplat(BroadcastVT, DL, N00));
46617 Vec = DAG.getSplat(VT, DL, DAG.getAnyExtOrTrunc(N00, DL, SVT));
46625 Bits.push_back(DAG.getConstant(Bit, DL, SVT));
46627 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
46628 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
46631 EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
46632 Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
46633 Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
46639 return DAG.getNode(ISD::SRL, DL, VT, Vec,
46640 DAG.getConstant(EltSizeInBits - 1, DL, VT));
46647 combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
46655 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
46672 return DAG.getConstantFP(0.0, DL, VT);
46673 return DAG.getConstant(0, DL, VT);
46691 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
46699 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
46708 if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
46713 return DAG.getBitcast(VT, Cond);
46720 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
46721 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
46722 return DAG.getBitcast(VT, Or);
46727 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
46728 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
46729 return DAG.getBitcast(VT, And);
46734 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
46738 AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
46741 AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
46742 return DAG.getBitcast(VT, AndN);
46752 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
46768 !isFreeToSplitVector(TVal.getNode(), DAG) ||
46769 !isFreeToSplitVector(FVal.getNode(), DAG))
46772 auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
46774 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
46776 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Cond, TVal, FVal}, makeBlend,
46780 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG,
46793 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
46831 Cond = DAG.getNOT(DL, Cond, MVT::i1);
46836 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
46840 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
46844 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
46857 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
46867 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
46917 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
46922 // If we changed the computation somewhere in the DAG, this change will
46931 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
46933 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
46941 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
46942 return DAG.getNode(X86ISD::BLENDV, DL, N->getValueType(0), V,
46966 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
46969 DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() &&
46974 if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT))
46990 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
47006 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
47007 return DAG.getBitcast(VT, Res);
47010 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
47035 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
47037 return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
47041 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
47051 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
47058 if (SDValue V = commuteSelect(N, DAG, DL, Subtarget))
47063 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47072 DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits())
47074 DL, DAG, Subtarget))
47083 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
47110 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0),
47111 getConstVector(LHSMask, SimpleVT, DAG, DL, true));
47112 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0),
47113 getConstVector(RHSMask, SimpleVT, DAG, DL, true));
47114 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
47137 if (DAG.isEqualTo(LHS, Op0) && DAG.isEqualTo(RHS, Op1)) {
47144 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
47145 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47146 !(DAG.isKnownNeverZeroFloat(LHS) ||
47147 DAG.isKnownNeverZeroFloat(RHS)))
47156 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47157 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
47175 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47176 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
47184 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
47185 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47186 !(DAG.isKnownNeverZeroFloat(LHS) ||
47187 DAG.isKnownNeverZeroFloat(RHS)))
47205 } else if (DAG.isEqualTo(LHS, Op1) && DAG.isEqualTo(RHS, Op0)) {
47212 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47213 !(DAG.isKnownNeverZeroFloat(LHS) ||
47214 DAG.isKnownNeverZeroFloat(RHS))) {
47215 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
47223 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
47240 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
47248 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
47249 !DAG.isKnownNeverZeroFloat(LHS) &&
47250 !DAG.isKnownNeverZeroFloat(RHS)) {
47251 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
47272 SDValue Ret = DAG.getNode(Opcode == X86ISD::FMIN ? X86ISD::STRICT_FMIN
47276 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Ret.getValue(1));
47279 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
47297 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
47298 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
47313 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
47314 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
47339 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
47341 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
47343 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
47344 DAG.getUNDEF(SrcCondVT), Cond,
47345 DAG.getVectorIdxConstant(0, DL));
47346 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
47347 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
47351 if (SDValue V = combineSelectOfTwoConstants(N, DAG, DL))
47384 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
47385 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
47389 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
47390 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
47420 Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
47421 return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2));
47435 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
47437 return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
47447 ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) {
47448 ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond);
47449 return DAG.getSelect(DL, VT, ExtCond, LHS, RHS);
47471 return DAG.getNode(LHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV
47483 return DAG.getNode(RHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV
47493 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget))
47496 if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
47499 if (SDValue V = narrowVectorSelect(N, DAG, DL, Subtarget))
47504 if (SDValue CondNot = IsNOT(Cond, DAG))
47505 return DAG.getNode(N->getOpcode(), DL, VT,
47506 DAG.getBitcast(CondVT, CondNot), RHS, LHS);
47516 Cond = DAG.getNode(X86ISD::PCMPEQ, DL, CondVT, Cond.getOperand(0),
47518 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS);
47526 Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT,
47527 DAG.getConstant(0, DL, CondVT), Cond.getOperand(0));
47528 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS);
47540 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
47550 LHS = combinevXi1ConstantToInteger(LHS, DAG);
47555 RHS = combinevXi1ConstantToInteger(RHS, DAG);
47559 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
47560 return DAG.getBitcast(VT, Select);
47581 DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
47582 return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
47606 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
47607 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
47609 DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
47610 return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
47624 SelectionDAG &DAG,
47707 auto AtomicSub = DAG.getAtomic(
47710 /*RHS*/ DAG.getConstant(NegAddend, SDLoc(CmpRHS), CmpVT),
47712 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
47713 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
47714 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
47734 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
47735 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
47736 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
47743 SelectionDAG &DAG) {
47765 if (DAG.SignBitIsZero(Cmp.getOperand(0)))
47767 else if (DAG.SignBitIsZero(Cmp.getOperand(1)))
47783 if (std::optional<uint64_t> ShiftAmt = DAG.getValidShiftAmount(Src)) {
47789 SDValue Mask = DAG.getNode(ISD::AND, DL, SrcVT, Src,
47790 DAG.getConstant(BitMask, DL, SrcVT));
47792 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Mask,
47793 DAG.getConstant(0, DL, SrcVT));
47970 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
48001 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
48014 SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType());
48019 return getBT(Carry, BitNo, DL, DAG);
48030 SelectionDAG &DAG,
48045 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48048 if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
48079 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
48080 DAG.getBitcast(OpVT, NotOp0), Op1);
48086 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
48089 return DAG.getNode(
48090 EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1),
48091 DAG.getBitcast(OpVT,
48092 DAG.getAllOnesConstant(DL, NotOp1.getValueType())));
48103 SDValue X = DAG.getBitcast(OpVT, BC0.getOperand(0));
48104 return DAG.getNode(EFLAGS.getOpcode(), DL, VT, X, X);
48111 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
48113 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
48114 DAG.getBitcast(OpVT, NotOp1), Op0);
48123 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
48124 DAG.getBitcast(OpVT, BC.getOperand(0)),
48125 DAG.getBitcast(OpVT, BC.getOperand(1)));
48131 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
48132 DAG.getBitcast(OpVT, BC.getOperand(0)),
48133 DAG.getBitcast(OpVT, BC.getOperand(1)));
48143 if (DAG.ComputeNumSignBits(BC) == EltBits) {
48147 TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) {
48155 Res = DAG.getBitcast(FloatVT, Res);
48156 return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res);
48159 Res = DAG.getBitcast(MovmskVT, Res);
48160 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
48161 Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,
48162 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
48164 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
48166 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,
48167 DAG.getConstant(0, DL, MVT::i32));
48175 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1);
48179 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0);
48193 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
48194 DAG.getBitcast(OpVT2, Src0),
48195 DAG.getBitcast(OpVT2, Src1));
48206 SelectionDAG &DAG,
48268 DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) {
48271 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
48272 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),
48273 DAG.getConstant(CmpMask, DL, MVT::i32));
48283 if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) &&
48288 SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT,
48289 DAG.getBitcast(SubVT, Ops[0]),
48290 DAG.getBitcast(SubVT, Ops[1]));
48291 V = DAG.getBitcast(VecVT.getHalfNumVectorElementsVT(), V);
48292 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
48293 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V),
48294 DAG.getConstant(CmpMask, DL, MVT::i32));
48308 SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(),
48310 V = DAG.getBitcast(TestVT, V);
48311 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
48319 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), LHS.getValueType(),
48321 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), RHS.getValueType(),
48323 LHS = DAG.getBitcast(TestVT, LHS);
48324 RHS = DAG.getBitcast(TestVT, RHS);
48325 SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS);
48326 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
48339 bool SignExt0 = DAG.ComputeNumSignBits(VecOp0) > 8;
48340 bool SignExt1 = DAG.ComputeNumSignBits(VecOp1) > 8;
48344 SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
48345 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
48346 Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);
48348 Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,
48349 DAG.getConstant(0xAAAA, DL, MVT::i16));
48351 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
48352 DAG.getConstant(0, DL, MVT::i16));
48363 SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(),
48365 V = DAG.getBitcast(MVT::v4i64, V);
48366 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
48368 Result = DAG.getBitcast(MVT::v32i8, Result);
48369 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
48374 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
48375 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
48377 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
48378 DAG.getConstant(CmpMask, DL, MVT::i32));
48402 ShuffleMask, DAG) &&
48407 SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
48408 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
48410 DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
48411 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, EFLAGS.getOperand(1));
48427 SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT);
48429 return DAG.getNode(X86ISD::TESTP, DL, MVT::i32,
48430 DAG.getBitcast(FloatVT, LHS),
48431 DAG.getBitcast(FloatVT, RHS));
48441 SelectionDAG &DAG,
48444 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
48447 if (SDValue R = checkSignTestSetCCCombine(EFLAGS, CC, DAG))
48453 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
48456 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
48459 return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
48463 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
48479 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
48484 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
48486 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
48507 Cond = getSETCC(CC, Cond, DL, DAG);
48510 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
48513 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
48514 DAG.getConstant(ShAmt, DL, MVT::i8));
48521 Cond = getSETCC(CC, Cond, DL, DAG);
48524 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
48526 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
48555 Cond = getSETCC(CC, Cond, DL ,DAG);
48557 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
48561 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
48562 DAG.getConstant(Diff, DL, Cond.getValueType()));
48566 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
48605 DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
48606 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
48628 DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0),
48629 DAG.getConstant(1, DL, CondVT));
48631 return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(VT, MVT::i32), TrueOp,
48632 DAG.getConstant(0, DL, VT), EFLAGS);
48665 DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
48666 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, VT, LOps);
48667 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
48669 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
48698 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
48700 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
48701 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
48702 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
48712 static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) {
48723 SignBits[i] = DAG.ComputeNumSignBits(Opd);
48724 IsPositive[i] = DAG.SignBitIsZero(Opd);
48774 static SDValue reduceVMULWidth(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
48785 bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
48790 if (!canReduceVMulWidth(N, DAG, Mode))
48800 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
48803 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
48804 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
48808 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
48810 return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND
48814 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2);
48818 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
48830 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
48831 ResLo = DAG.getBitcast(ResVT, ResLo);
48838 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
48839 ResHi = DAG.getBitcast(ResVT, ResHi);
48840 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
48843 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
48847 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48848 DAG.getConstant(Mult, DL, VT));
48849 Result = DAG.getNode(ISD::SHL, DL, VT, Result,
48850 DAG.getConstant(Shift, DL, MVT::i8));
48851 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
48857 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48858 DAG.getConstant(Mul1, DL, VT));
48859 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
48860 DAG.getConstant(Mul2, DL, VT));
48861 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
48880 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
48905 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
48918 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48919 DAG.getConstant(ShiftAmt, DL, MVT::i8));
48920 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48921 DAG.getConstant(ScaleShift, DL, MVT::i8));
48922 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
48933 SelectionDAG &DAG,
48983 if (DAG.ComputeMaxSignificantBits(N1) > 16 ||
48984 DAG.ComputeMaxSignificantBits(N0) > 16)
48991 if (DAG.MaskedValueIsZero(Op, Mask17))
48995 return DAG.getNode(ISD::AND, DL, VT, Op, DAG.getConstant(0xFFFF, DL, VT));
49000 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src);
49005 Src = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, Src);
49006 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src);
49014 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Src);
49019 return DAG.getNode(X86ISD::VSRLI, DL, VT, Op.getOperand(0),
49032 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
49036 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
49037 DAG.getBitcast(OpVT, Ops[0]),
49038 DAG.getBitcast(OpVT, Ops[1]));
49040 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDWDBuilder);
49043 static SDValue combineMulToPMULDQ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
49061 if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 &&
49062 DAG.ComputeNumSignBits(N1) > 32) {
49063 auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
49065 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
49067 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULDQBuilder,
49073 if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) {
49074 auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
49076 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
49078 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULUDQBuilder,
49085 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
49091 if (SDValue V = combineMulToPMADDWD(N, DL, DAG, Subtarget))
49094 if (SDValue V = combineMulToPMULDQ(N, DL, DAG, Subtarget))
49098 return reduceVMULWidth(N, DL, DAG, Subtarget);
49104 KnownBits Known1 = DAG.computeKnownBits(N->getOperand(1));
49110 return DAG.getConstant(0, DL, VT);
49113 return DAG.getNegative(N->getOperand(0), DL, VT);
49124 if (DAG.getMachineFunction().getFunction().hasMinSize())
49137 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
49138 DAG.getConstant(AbsMulAmt, DL, VT));
49140 NewMul = DAG.getNegative(NewMul, DL, VT);
49172 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49173 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
49175 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
49176 DAG.getConstant(MulAmt1, DL, VT));
49179 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
49180 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
49182 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
49183 DAG.getConstant(MulAmt2, DL, VT));
49187 NewMul = DAG.getNegative(NewMul, DL, VT);
49189 NewMul = combineMulSpecial(C.getZExtValue(), N, DAG, VT, DL);
49195 NewMul = DAG.getNode(
49197 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49198 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT)));
49200 NewMul = DAG.getNegative(NewMul, DL, VT);
49204 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49205 DAG.getConstant(Log2_64(AbsMulAmt + 1), DL, ShiftVT));
49208 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
49210 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
49215 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49216 DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT));
49217 NewMul = DAG.getNode(
49219 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
49224 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49225 DAG.getConstant(Log2_64(AbsMulAmt + 2), DL, ShiftVT));
49226 NewMul = DAG.getNode(
49228 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
49244 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49245 DAG.getConstant(Log2_64(ShiftAmt1), DL, ShiftVT));
49247 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
49248 DAG.getConstant(Log2_64(AbsMulAmtLowBit), DL, ShiftVT));
49249 NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2);
49262 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
49264 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
49307 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
49310 return DAG.getNode(ExtOpc, DL, VT, Mulh);
49313 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG,
49334 return DAG.getNode(X86ISD::VSHLV, DL, VT, N00, N1);
49340 return DAG.getNode(X86ISD::VSHLV, DL, VT, N01, N1);
49375 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
49381 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
49390 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
49398 return DAG.getNode(X86ISD::VSRAV, DL, VT, N0, ShrAmtVal);
49437 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
49441 return DAG.getNode(ISD::SHL, DL, VT, NN,
49442 DAG.getConstant(ShlConst - SraConst, DL, CVT));
49443 return DAG.getNode(ISD::SRA, DL, VT, NN,
49444 DAG.getConstant(SraConst - ShlConst, DL, CVT));
49449 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
49459 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
49473 return DAG.getNode(X86ISD::VSRLV, DL, VT, N00, N1);
49479 return DAG.getNode(X86ISD::VSRLV, DL, VT, N01, N1);
49483 // Only do this on the last DAG combine as it can interfere with other
49489 // TODO: This is a generic DAG combine that became an x86-only combine to
49518 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
49519 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
49520 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
49525 static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
49551 if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) {
49560 std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL);
49561 Lo = DAG.getBitcast(SrcVT, Lo);
49562 Hi = DAG.getBitcast(SrcVT, Hi);
49563 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
49564 Res = DAG.getBitcast(ShufVT, Res);
49565 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask);
49566 return DAG.getBitcast(VT, Res);
49579 getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
49583 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
49618 LHS = DAG.getBitcast(SrcVT, LHS);
49619 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
49621 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
49622 Res = DAG.getBitcast(ShufVT, Res);
49623 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle);
49624 return DAG.getBitcast(VT, Res);
49634 if (getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
49635 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
49657 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00),
49658 DAG.getBitcast(SrcVT, Op01));
49659 Res = DAG.getBitcast(ShufVT, Res);
49660 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask);
49661 return DAG.getBitcast(VT, Res);
49669 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
49739 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
49743 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
49749 (N0.isUndef() || DAG.ComputeNumSignBits(N0) == SrcBitsPerElt) &&
49750 (N1.isUndef() || DAG.ComputeNumSignBits(N1) == SrcBitsPerElt)) {
49751 SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG);
49752 SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG);
49757 DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0),
49758 DAG.getBitcast(SrcVT, Not1));
49759 return DAG.getNOT(DL, Pack, VT);
49768 if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
49770 DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) {
49772 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));
49776 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
49777 N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));
49778 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);
49798 Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType());
49799 Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType());
49800 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1);
49809 DAG);
49814 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49820 static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
49827 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) {
49844 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
49848 Res = DAG.getBitcast(ShufVT, Res);
49850 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
49851 getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG));
49853 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
49854 getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG));
49855 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS),
49856 DAG.getBitcast(VT, NewRHS));
49862 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
49868 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
49880 return DAG.getConstant(0, SDLoc(N), VT);
49890 EltBits[0].getZExtValue(), DAG);
49893 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49901 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
49919 return DAG.getConstant(0, SDLoc(N), VT);
49926 return DAG.getConstant(0, SDLoc(N), VT);
49938 return DAG.getConstant(0, SDLoc(N), VT);
49944 return DAG.getAllOnesConstant(SDLoc(N), VT);
49952 return DAG.getConstant(0, SDLoc(N), VT);
49955 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0),
49956 DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
49971 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49989 Src = DAG.getBitcast(VT, Src);
49990 Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src,
49991 getV4X86ShuffleImm8ForMask({0, 0, 2, 2}, DL, DAG));
49992 Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1);
49993 Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1);
50023 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));
50039 SDValue LHS = DAG.getNode(Opcode, DL, VT,
50040 DAG.getBitcast(VT, BC.getOperand(0)), N1);
50041 return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS);
50046 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50054 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
50070 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl);
50074 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50083 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
50093 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
50155 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
50156 DAG.getTargetConstant(x86cc, DL, MVT::i8));
50159 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
50160 DAG.getConstant(0, DL, MVT::v16i1),
50161 FSetCC, DAG.getVectorIdxConstant(0, DL));
50162 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
50166 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
50167 CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
50178 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
50180 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
50182 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Vector32,
50183 DAG.getVectorIdxConstant(0, DL));
50187 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
50188 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
50189 DAG.getConstant(1, DL, IntVT));
50190 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
50201 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
50212 if (SDValue Not = IsNOT(N0, DAG)) {
50215 } else if (SDValue Not = IsNOT(N1, DAG)) {
50221 X = DAG.getBitcast(VT, X);
50222 Y = DAG.getBitcast(VT, Y);
50223 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
50232 static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
50243 auto GetNot = [&DAG](SDValue V) {
50260 if (SDValue Not = IsNOT(Src, DAG)) {
50261 SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
50263 DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(),
50265 return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN,
50274 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50285 X = DAG.getBitcast(VT, X);
50286 Y = DAG.getBitcast(VT, Y);
50292 TLI.isTypeLegal(VT.getHalfNumVectorElementsVT(*DAG.getContext()))) {
50294 std::tie(LoX, HiX) = splitVector(X, DAG, DL);
50296 std::tie(LoY, HiY) = splitVector(Y, DAG, DL);
50298 SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
50299 SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
50300 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV});
50304 return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y});
50319 SelectionDAG &DAG, unsigned Depth) {
50330 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50334 if (SDValue NN0 = PromoteMaskArithmetic(N0, DL, VT, DAG, Depth + 1))
50348 if (SDValue NN1 = PromoteMaskArithmetic(N1, DL, VT, DAG, Depth + 1))
50357 DAG.FoldConstantArithmetic(ISD::ZERO_EXTEND, DL, VT, {N1}))
50363 return DAG.getNode(N.getOpcode(), DL, VT, N0, N1);
50373 SelectionDAG &DAG,
50385 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, 0);
50393 return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
50395 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
50396 Op, DAG.getValueType(NarrowVT));
50418 SelectionDAG &DAG,
50441 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
50442 return DAG.getBitcast(VT, FPLogic);
50463 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts);
50464 EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
50465 SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL);
50468 SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00);
50469 SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01);
50470 SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10);
50471 SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11);
50472 SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0);
50473 SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1);
50474 SDValue Logic = DAG.getNode(Opc, DL, BoolVecVT, Setcc0, Setcc1);
50475 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex);
50481 SDValue N1, SelectionDAG &DAG) {
50504 DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
50505 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
50513 SelectionDAG &DAG) {
50537 DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0));
50538 SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1));
50539 return DAG.getBitcast(VT, Shift);
50550 SDValue N0, SDValue N1, SelectionDAG &DAG) {
50573 if (DAG.ComputeNumSignBits(N0.getOperand(0)) != NumSrcBits ||
50574 DAG.ComputeNumSignBits(N0.getOperand(1)) != NumSrcBits ||
50575 DAG.ComputeNumSignBits(N1.getOperand(0)) != NumSrcBits ||
50576 DAG.ComputeNumSignBits(N1.getOperand(1)) != NumSrcBits)
50579 SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0));
50580 SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1));
50581 return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS));
50587 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
50620 VT.getScalarSizeInBits() - 1, DAG);
50621 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y);
50637 if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
50642 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
50643 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
50644 return DAG.getBitcast(N->getValueType(0), Shift);
50647 // Get the index node from the lowered DAG of a GEP IR instruction with one
50671 static SDValue combineAndNotOrIntoAndNotAnd(SDNode *N, SelectionDAG &DAG) {
50676 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50684 if (!DAG.isConstantIntBuildVectorOrConstantInt(Y) &&
50685 !DAG.isConstantIntBuildVectorOrConstantInt(Z))
50686 return DAG.getNode(
50688 DAG.getNOT(
50689 DL, DAG.getNode(ISD::AND, DL, VT, DAG.getNOT(DL, Y, VT), Z), VT));
50705 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
50759 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
50761 Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);
50762 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
50763 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);
50765 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
50766 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
50767 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);
50780 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
50812 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50852 DAG.getConstant(0, dl, SubVecVT));
50854 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
50856 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
50857 return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
50860 static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG,
50879 getBMIMatchingOp(Opc, DAG, OpMustEq, Op.getOperand(OpIdx), Depth))
50880 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), R,
50887 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
50893 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
50900 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
50905 static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG,
50918 getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx),
50925 SelectionDAG &DAG,
50957 // sub has two results while X only have one. DAG combine assumes the value
50960 X = DAG.getMergeValues({N->getOperand(0), X}, SDLoc(N));
50974 Ops[CondNo] = DAG.getTargetConstant(OppositeCC, SDLoc(BrCond), MVT::i8);
50979 DAG.getNode(X86ISD::BRCOND, SDLoc(BrCond), BrCond->getValueType(0), Ops);
50986 static SDValue combineAndOrForCcmpCtest(SDNode *N, SelectionDAG &DAG,
51040 IsOR ? DAG.getTargetConstant(X86::GetOppositeBranchCondition(CC0),
51049 SDValue CFlags = DAG.getTargetConstant(
51056 ? DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
51059 : DAG.getNode(X86ISD::CTEST, DL, MVT::i32,
51063 return DAG.getNode(X86ISD::SETCC, DL, MVT::i8, {CC1N, CCMP});
51066 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
51073 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51077 return DAG.getBitcast(MVT::v4i32,
51078 DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,
51079 DAG.getBitcast(MVT::v4f32, N0),
51080 DAG.getBitcast(MVT::v4f32, N1)));
51086 if (DAG.MaskedValueIsZero(N1, HiMask) ||
51087 DAG.MaskedValueIsZero(N0, HiMask)) {
51088 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
51089 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
51090 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
51091 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
51103 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
51104 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
51106 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
51110 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
51111 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
51112 return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ);
51138 SDValue Neg = DAG.getNegative(N0.getOperand(0), dl, VT);
51142 SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg,
51143 DAG.getConstant(MulCLowBitLog, dl, VT));
51144 return DAG.getNode(ISD::AND, dl, VT, Shift, N1);
51149 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
51152 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
51155 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), dl, N0, N1, DAG))
51158 if (SDValue R = combineBitOpWithShift(N->getOpcode(), dl, VT, N0, N1, DAG))
51161 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), dl, VT, N0, N1, DAG))
51165 DAG, DCI, Subtarget))
51168 if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
51174 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
51177 if (SDValue R = combineAndNotIntoANDNP(N, DAG))
51180 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
51183 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
51186 if (SDValue R = combineAndNotOrIntoAndNotAnd(N, DAG))
51196 DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() &&
51198 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1);
51199 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul);
51231 if (SDValue BT = getBT(Src, BitNo, dl, DAG))
51232 return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT);
51239 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
51283 SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG);
51284 SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG);
51286 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0,
51323 /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
51324 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
51329 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
51337 SelectionDAG &DAG,
51383 SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1));
51384 SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0));
51385 SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0));
51386 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
51388 DAG, Subtarget);
51389 return DAG.getBitcast(VT, Res);
51394 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
51395 DAG.getBitcast(VT, N1.getOperand(0)));
51396 return DAG.getNode(ISD::OR, DL, VT, X, Y);
51440 SelectionDAG &DAG,
51462 if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits)
51467 DAG, Subtarget))
51480 X = DAG.getBitcast(BlendVT, X);
51481 Y = DAG.getBitcast(BlendVT, Y);
51482 Mask = DAG.getBitcast(BlendVT, Mask);
51483 Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
51484 return DAG.getBitcast(VT, Mask);
51493 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
51498 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
51501 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
51502 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
51503 DAG.getConstant(Log2b, dl, MVT::i8));
51514 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
51564 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
51566 if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG)))
51567 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);
51580 NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG);
51583 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);
51586 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
51591 const SDLoc &DL, SelectionDAG &DAG) {
51603 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
51604 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
51605 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
51606 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
51615 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
51631 if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
51633 if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
51635 if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
51637 if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
51648 SelectionDAG &DAG,
51650 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
51664 EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC);
51679 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
51680 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51692 SDValue NewSub = DAG.getNode(
51696 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
51697 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51706 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
51707 DAG.getVTList(VT, MVT::i32), X,
51708 DAG.getConstant(0, DL, VT), EFLAGS);
51725 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
51728 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
51729 DAG.getVTList(VT, MVT::i32), X,
51730 DAG.getConstant(0, DL, VT), NewEFLAGS);
51737 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
51738 DAG.getVTList(VT, MVT::i32), X,
51739 DAG.getAllOnesConstant(DL, VT), EFLAGS);
51755 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
51758 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
51759 DAG.getVTList(VT, MVT::i32), X,
51760 DAG.getAllOnesConstant(DL, VT), NewEFLAGS);
51784 SDValue Zero = DAG.getConstant(0, DL, ZVT);
51785 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
51786 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
51787 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
51788 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51798 SDValue One = DAG.getConstant(1, DL, ZVT);
51799 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
51800 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
51801 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
51802 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51808 SDValue One = DAG.getConstant(1, DL, ZVT);
51809 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
51810 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
51813 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
51818 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
51819 DAG.getAllOnesConstant(DL, VT), Cmp1.getValue(1));
51823 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
51824 DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
51831 SelectionDAG &DAG) {
51837 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG))
51841 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) {
51843 ADCOrSBB = DAG.getNegative(ADCOrSBB, DL, VT);
51852 SelectionDAG &DAG) {
51867 if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG))
51879 return DAG.getNode(X86ISD::PCMPEQ, DL, VT, N0.getOperand(0),
51886 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
51893 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51897 return DAG.getBitcast(MVT::v4i32,
51898 DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,
51899 DAG.getBitcast(MVT::v4f32, N0),
51900 DAG.getBitcast(MVT::v4f32, N1)));
51911 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
51912 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
51914 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
51918 SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
51919 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
51920 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
51921 return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE);
51926 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
51929 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), dl, N0, N1, DAG))
51932 if (SDValue R = combineBitOpWithShift(N->getOpcode(), dl, VT, N0, N1, DAG))
51935 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), dl, VT, N0, N1, DAG))
51939 DAG, DCI, Subtarget))
51945 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
51948 if (SDValue R = canonicalizeBitSelect(N, dl, DAG, Subtarget))
51951 if (SDValue R = combineLogicBlendIntoPBLENDV(N, dl, DAG, Subtarget))
51968 SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG);
51970 SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
51971 R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
51972 R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
51989 DAG.MaskedVectorIsZero(N0, UpperElts)) {
51990 return DAG.getNode(
51992 extractSubVector(N0, 0, DAG, dl, HalfElts),
51993 extractSubVector(N1.getOperand(0), 0, DAG, dl, HalfElts));
51997 DAG.MaskedVectorIsZero(N1, UpperElts)) {
51998 return DAG.getNode(
52000 extractSubVector(N1, 0, DAG, dl, HalfElts),
52001 extractSubVector(N0.getOperand(0), 0, DAG, dl, HalfElts));
52008 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
52037 if (SDValue R = foldMaskedMerge(N, DAG))
52040 if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), dl, VT, N0, N1, DAG))
52050 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
52088 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52089 EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
52090 *DAG.getContext(), ResultType);
52092 DAG.getSetCC(DL, SetCCResultType, ShiftOp,
52093 DAG.getAllOnesConstant(DL, ShiftOpTy), ISD::SETGT);
52095 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
52106 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
52143 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT);
52162 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
52188 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
52230 SelectionDAG &DAG,
52248 DL, DAG, Subtarget);
52250 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
52275 DAG, Subtarget);
52277 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
52282 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
52286 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
52290 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52299 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) {
52310 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
52312 InVT = EVT::getVectorVT(*DAG.getContext(), InSVT,
52314 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
52319 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts);
52320 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
52321 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
52322 DAG.getVectorIdxConstant(0, DL));
52330 SelectionDAG &DAG,
52391 SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
52392 Extract = DAG.getBitcast(RegVT, Extract);
52404 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
52411 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52422 (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT,
52432 DAG.getMemBasePlusOffset(Ptr1, TypeSize::getFixed(HalfOffset), dl);
52433 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
52436 DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(),
52439 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
52443 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
52446 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
52455 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
52457 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
52461 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
52481 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl,
52483 Extract = DAG.getBitcast(RegVT, Extract);
52489 if (SDValue V = combineConstantPoolLoads(Ld, dl, DAG, DCI, Subtarget))
52496 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
52499 DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
52500 return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast,
52548 SelectionDAG &DAG, SDValue &Addr,
52562 Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::getFixed(Offset),
52566 Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp));
52577 reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
52588 if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset))
52604 DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
52608 SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
52612 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex);
52613 Insert = DAG.getBitcast(VT, Insert);
52618 combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
52635 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
52637 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
52656 SDValue NewML = DAG.getMaskedLoad(
52658 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
52660 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
52666 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
52677 reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget))
52682 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
52691 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52699 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
52700 return DAG.getMaskedLoad(
52714 SelectionDAG &DAG,
52723 if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset))
52734 Value = DAG.getBitcast(CastVT, Value);
52737 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex);
52740 return DAG.getStore(MS->getChain(), DL, Extract, Addr,
52745 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
52754 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52759 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget))
52773 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
52774 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(),
52784 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
52793 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
52801 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52807 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
52808 StoredVal = DAG.getBitcast(NewVT, StoredVal);
52810 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
52822 Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);
52823 return DAG.getStore(St->getChain(), dl, Val,
52834 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
52836 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
52837 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
52848 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
52850 Lo = combinevXi1ConstantToInteger(Lo, DAG);
52851 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
52853 Hi = combinevXi1ConstantToInteger(Hi, DAG);
52856 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl);
52859 DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(),
52863 DAG.getStore(St->getChain(), dl, Hi, Ptr1,
52867 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
52870 StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG);
52871 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
52890 SDValue LogicOp = DAG.getNode(
52891 SignOp, dl, IntVT, DAG.getBitcast(IntVT, StoredVal.getOperand(0)),
52892 DAG.getConstant(SignMask, dl, IntVT));
52893 return DAG.getStore(St->getChain(), dl, LogicOp, St->getBasePtr(),
52903 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
52910 return splitVectorStore(St, DAG);
52922 return splitVectorStore(St, DAG);
52931 return scalarizeVectorStore(St, NTVT, DAG);
52942 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,
52944 return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(),
52957 VT, St->getMemOperand(), DAG);
52983 return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(),
52998 St->getMemoryVT(), St->getMemOperand(), DAG);
53000 DAG, dl))
53003 St->getMemoryVT(), St->getMemOperand(), DAG);
53013 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
53016 DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
53017 return DAG.getTruncStore(
53048 SDVTList Tys = DAG.getVTList(MVT::Other);
53050 SDValue Src = DAG.getAnyExtOrTrunc(Cmov.getOperand(!InvertCC), dl, VT);
53052 CC = DAG.getTargetConstant(
53058 return DAG.getMemIntrinsicNode(X86ISD::CSTORE, dl, Tys, Ops, VT,
53071 const Function &F = DAG.getMachineFunction().getFunction();
53094 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
53098 DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
53099 return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(),
53114 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
53115 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
53116 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
53118 return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
53126 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
53139 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53163 SelectionDAG &DAG, const X86Subtarget &Subtarget,
53197 if (getTargetShuffleInputs(BC, SrcOps, SrcMask, DAG) &&
53210 std::tie(N0, N1) = DAG.SplitVector(SrcOps[0], SDLoc(Op));
53341 DAG, Subtarget))
53344 LHS = DAG.getBitcast(VT, NewLHS);
53345 RHS = DAG.getBitcast(VT, NewRHS);
53350 static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
53372 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
53374 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
53376 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
53377 DAG.getUNDEF(VT), PostShuffleMask);
53389 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
53391 auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL,
53393 return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops);
53395 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
53398 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
53399 DAG.getUNDEF(VT), PostShuffleMask);
53422 static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
53433 KnownBits XORRHS = DAG.computeKnownBits(XOR.getOperand(1));
53441 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
53442 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0));
53443 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F);
53444 r = DAG.getBitcast(VT, FCMulC);
53463 static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
53465 auto AllowContract = [&DAG](const SDNodeFlags &Flags) {
53466 return DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
53470 auto HasNoSignedZero = [&DAG](const SDNodeFlags &Flags) {
53471 return DAG.getTarget().Options.NoSignedZerosFPMath ||
53474 auto IsVectorAllNegativeZero = [&DAG](SDValue Op) {
53476 KnownBits Bits = DAG.computeKnownBits(Op);
53528 FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
53533 DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags());
53534 return DAG.getBitcast(VT, CFmul);
53538 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
53540 if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget))
53543 if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
53549 static SDValue combineLRINT_LLRINT(SDNode *N, SelectionDAG &DAG,
53560 return DAG.getNode(X86ISD::CVTP2SI, DL, VT,
53561 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, Src,
53562 DAG.getUNDEF(SrcVT)));
53570 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
53576 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53601 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
53602 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
53603 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
53649 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
53652 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
53678 auto IsSext = [&DAG](SDValue V) {
53679 return DAG.ComputeMaxSignificantBits(V) <= 16;
53681 auto IsZext = [&DAG](SDValue V) {
53682 return DAG.computeKnownBits(V).countMaxActiveBits() <= 16;
53707 EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
53709 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS),
53710 DAG.getBitcast(BCVT, RHS));
53711 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res));
53715 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS);
53716 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS);
53719 return DAG.getNode(Opc, DL, VT, LHS, RHS);
53730 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
53845 auto ExtractVec = [&DAG, &DL, NumElems](SDValue &Ext) {
53849 Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, Ext,
53850 DAG.getVectorIdxConstant(0, DL));
53856 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
53864 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
53866 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
53868 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
53872 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
53879 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
53883 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
53887 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
53891 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
53899 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
53905 return DAG.getNode(ISD::LRINT, DL, VT, Src.getOperand(0));
53910 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
53917 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
53918 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL))
53919 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
53921 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53938 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
53962 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
53964 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT),
53975 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
53977 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,
54084 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
54088 SDValue Arg = isFNEG(DAG, N);
54092 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54106 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
54107 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
54109 return DAG.getBitcast(OrigVT, NewNode);
54112 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
54115 TLI.getNegatedExpression(Arg, DAG, LegalOperations, CodeSize))
54116 return DAG.getBitcast(OrigVT, NegArg);
54121 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
54127 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
54129 return DAG.getBitcast(Op.getValueType(), Arg);
54160 Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1);
54174 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
54178 getNegatedExpression(Op.getOperand(0), DAG, LegalOperations,
54180 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0);
54184 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
54188 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
54201 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
54202 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
54213 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
54214 return DAG.getBitcast(VT, IntOp);
54219 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
54230 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
54233 static SDValue combineXorSubCTLZ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
54278 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, OpVT, Op);
54281 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
54282 Op = DAG.getNode(X86ISD::BSR, DL, VTs, DAG.getUNDEF(OpVT), Op);
54284 Op = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Op);
54289 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
54299 return DAG.getBitcast(MVT::v4i32,
54300 DAG.getNode(X86ISD::FXOR, DL, MVT::v4f32,
54301 DAG.getBitcast(MVT::v4f32, N0),
54302 DAG.getBitcast(MVT::v4f32, N1)));
54305 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
54308 if (SDValue R = combineBitOpWithMOVMSK(N->getOpcode(), DL, N0, N1, DAG))
54311 if (SDValue R = combineBitOpWithShift(N->getOpcode(), DL, VT, N0, N1, DAG))
54314 if (SDValue R = combineBitOpWithPACK(N->getOpcode(), DL, VT, N0, N1, DAG))
54318 DAG, DCI, Subtarget))
54321 if (SDValue R = combineXorSubCTLZ(N, DL, DAG, Subtarget))
54327 if (SDValue SetCC = foldXor1SetCC(N, DAG))
54330 if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), DL, VT, N0, N1, DAG))
54333 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
54337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54342 return DAG.getBitcast(
54343 VT, DAG.getNOT(DL, N0.getOperand(0), N0.getOperand(0).getValueType()));
54352 return DAG.getNode(
54354 DAG.getNOT(DL, N0.getOperand(1), N0.getOperand(1).getValueType()),
54367 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT);
54368 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT);
54369 return DAG.getNode(ISD::XOR, DL, VT, LHS,
54370 DAG.getNode(ISD::XOR, DL, VT, RHS, N1));
54374 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
54377 return combineFneg(N, DAG, DCI, Subtarget);
54380 static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG,
54392 DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) &&
54399 DAG.getVectorShuffle(SrcVT, SDLoc(N), Src, Src, ReverseMask);
54400 return DAG.getBitcast(VT, Rev);
54408 static SDValue combineAVG(SDNode *N, SelectionDAG &DAG,
54422 SDValue SignMask = DAG.getConstant(SignBit, DL, VT);
54423 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, SignMask);
54424 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, SignMask);
54425 return DAG.getNode(ISD::XOR, DL, VT,
54426 DAG.getNode(ISD::AVGCEILU, DL, VT, N0, N1), SignMask);
54432 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
54441 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54459 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
54465 return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V));
54470 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
54492 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
54496 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
54502 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
54505 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
54509 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
54512 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
54515 return lowerX86FPLogicOp(N, DAG, Subtarget);
54519 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
54526 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
54529 return lowerX86FPLogicOp(N, DAG, Subtarget);
54533 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
54546 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
54549 return lowerX86FPLogicOp(N, DAG, Subtarget);
54553 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
54557 if (!DAG.getTarget().Options.NoNaNsFPMath ||
54558 !DAG.getTarget().Options.NoSignedZerosFPMath)
54570 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
54574 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
54580 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54595 if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs())
54596 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
54600 if (DAG.isKnownNeverNaN(Op1))
54601 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
54602 if (DAG.isKnownNeverNaN(Op0))
54603 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
54607 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
54610 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
54632 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
54633 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
54637 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
54640 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
54643 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54659 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
54661 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
54662 DAG.getBitcast(InVT, VZLoad));
54664 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54673 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
54675 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
54689 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
54693 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
54694 {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)});
54698 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
54701 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54711 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
54719 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54725 return DAG.getConstant(0, DL, VT);
54733 return DAG.getConstant(0, DL, VT);
54737 return DAG.getNOT(DL, N0, VT);
54740 if (SDValue Not = IsNOT(N0, DAG))
54741 return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
54751 return DAG.getSelect(DL, VT, DAG.getNOT(DL, Src, SrcVT), N1,
54752 getZeroVector(VT, Subtarget, DAG, DL));
54767 return getConstVector(ResultBits, VT, DAG, DL);
54778 SDValue Not = getConstVector(EltBits0, VT, DAG, DL);
54779 return DAG.getNode(ISD::AND, DL, VT, Not, N1);
54787 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
54834 if (SDValue Not = IsNOT(N1, DAG))
54835 return DAG.getNOT(
54836 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT);
54840 if (DAG.ComputeNumSignBits(N0) == EltSizeInBits) {
54844 SDValue NewMask = DAG.getNode(ISD::OR, DL, ShufVT, BC1.getOperand(1),
54845 DAG.getBitcast(ShufVT, N0));
54847 DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, BC1.getOperand(0), NewMask);
54848 return DAG.getBitcast(VT, NewShuf);
54856 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
54863 if (DAG.getTargetLoweringInfo().SimplifyDemandedBits(N1, DemandedMask, DCI)) {
54872 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
54878 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54889 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
54892 SDValue Convert = DAG.getNode(
54894 {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)});
54897 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
54898 DAG.getBitcast(MVT::v8i16, VZLoad));
54902 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
54913 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
54950 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
54951 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
54954 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
54955 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
54961 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
54962 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
54965 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
54969 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
54974 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
54978 if (SDValue V = combineSextInRegCmov(N, DAG))
55004 if (SDValue Promote = PromoteMaskArithmetic(N0, dl, DAG, Subtarget))
55005 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1);
55009 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1);
55010 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
55021 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
55041 NSW = NSW || (Sext && DAG.willNotOverflowAdd(true, AddOp0, AddOp1));
55042 NUW = NUW || (!Sext && DAG.willNotOverflowAdd(false, AddOp0, AddOp1));
55073 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
55074 SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT);
55081 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
55096 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
55128 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
55129 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
55131 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
55136 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
55143 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
55179 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
55182 Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType());
55187 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
55197 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
55203 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
55211 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
55217 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
55221 DAG, DCI, Subtarget))
55225 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), DL, DAG, Subtarget))
55229 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
55232 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
55239 // inverted vector is already present in DAG. Otherwise, it will be loaded
55244 static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) {
55262 Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT));
55265 Ops.push_back(DAG.getUNDEF(EltVT));
55269 SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops);
55292 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
55297 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
55303 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55316 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags);
55317 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags);
55327 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
55328 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
55330 if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
55341 Vec, DAG, LegalOperations, CodeSize)) {
55342 V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(),
55347 // Lookup if there is an inverted version of constant vector V in DAG.
55349 if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) {
55370 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
55373 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other},
55377 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));
55378 return DAG.getNode(NewOpcode, dl, VT, A, B, C);
55384 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
55388 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55389 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
55395 TLI.getCheaperNegatedExpression(N2, DAG, LegalOperations, CodeSize);
55401 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
55403 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
55407 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
55418 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
55424 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
55432 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
55436 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
55440 DAG, DCI, Subtarget))
55444 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), dl, DAG, Subtarget))
55447 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
55450 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
55460 if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) &&
55461 (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) {
55462 return concatSubVectors(N00, N01, DAG, dl);
55474 const SDLoc &DL, SelectionDAG &DAG,
55480 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
55481 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
55486 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
55497 if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
55504 MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
55505 return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
55514 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
55517 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
55523 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
55525 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
55532 return DAG.getNode(ISD::AND, DL, OpVT, N1,
55533 DAG.getNOT(DL, N0.getOperand(1), OpVT));
55535 return DAG.getNode(ISD::AND, DL, OpVT, N1,
55536 DAG.getNOT(DL, N0.getOperand(0), OpVT));
55541 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
55543 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
55554 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55555 if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) &&
55557 return DAG.getSetCC(DL, VT, LHS.getOperand(0),
55558 DAG.getZExtOrTrunc(RHS, DL, SrcVT), CC);
55575 SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC);
55576 SDValue SETCC1 = DAG.getSetCC(
55577 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC);
55578 return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT,
55608 return DAG.getConstant(0, DL, VT);
55610 return DAG.getConstant(1, DL, VT);
55612 return DAG.getNOT(DL, Op0.getOperand(0), VT);
55628 DAG.computeKnownBits(LHS).intersectWith(DAG.computeKnownBits(RHS));
55647 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
55651 RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
55663 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
55666 else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
55682 NewCC, DL, DAG, Subtarget))
55684 return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC);
55690 truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget))
55717 DAG.isConstantIntBuildVectorOrConstantInt(AddC)) {
55727 DAG.getTargetLoweringInfo().isOperationLegal(ISD::UMIN, OpVT)) {
55734 C0 = DAG.getNegative(AddC, DL, OpVT);
55735 C1 = DAG.getNode(ISD::SUB, DL, OpVT, C0,
55736 DAG.getAllOnesConstant(DL, OpVT));
55742 C0 = DAG.getNOT(DL, AddC, OpVT);
55743 C1 = DAG.getNode(ISD::ADD, DL, OpVT, C0,
55744 DAG.getAllOnesConstant(DL, OpVT));
55748 DAG.getSetCC(DL, VT, LHS.getOperand(0), C0, ISD::SETEQ);
55750 DAG.getSetCC(DL, VT, LHS.getOperand(0), C1, ISD::SETEQ);
55751 return DAG.getNode(ISD::OR, DL, VT, NewLHS, NewRHS);
55760 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
55767 SDVTList FNegVT = DAG.getVTList(OpVT);
55768 if (SDNode *FNeg = DAG.getNodeIfExists(ISD::FNEG, FNegVT, {LHS}))
55769 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC);
55775 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
55797 return DAG.getConstant(Imm, SDLoc(N), VT);
55804 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
55808 if (SDValue NotSrc = IsNOT(Src, DAG)) {
55811 NotSrc = DAG.getBitcast(SrcVT, NotSrc);
55812 return DAG.getNode(ISD::XOR, DL, VT,
55813 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
55814 DAG.getConstant(NotMask, DL, VT));
55823 return DAG.getNode(ISD::XOR, DL, VT,
55824 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)),
55825 DAG.getConstant(NotMask, DL, VT));
55834 KnownBits KnownLHS = DAG.computeKnownBits(Src.getOperand(0));
55835 KnownBits KnownRHS = DAG.computeKnownBits(Src.getOperand(1));
55847 ShiftLHS = DAG.getBitcast(ShiftVT, ShiftLHS);
55848 ShiftRHS = DAG.getBitcast(ShiftVT, ShiftRHS);
55851 ShiftLHS, ShiftAmt, DAG);
55853 ShiftRHS, ShiftAmt, DAG);
55854 ShiftLHS = DAG.getBitcast(SrcVT, ShiftLHS);
55855 ShiftRHS = DAG.getBitcast(SrcVT, ShiftRHS);
55856 SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS);
55857 return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT));
55875 SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0));
55876 SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc);
55877 return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk,
55878 DAG.getConstant(Mask, DL, VT));
55884 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55892 static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG,
55899 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55907 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
55914 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55928 SelectionDAG &DAG) {
55934 return DAG.getMaskedGather(Gather->getVTList(),
55943 return DAG.getMaskedScatter(Scatter->getVTList(),
55950 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
55957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55971 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
55973 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
55974 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
55985 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
55987 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
55988 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
55992 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
56009 Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
56010 DAG.getConstant(Adder, DL, PtrVT));
56012 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
56020 SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base);
56022 Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
56025 Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
56027 Base = DAG.getConstant(0, DL, Base.getValueType());
56028 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
56040 Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
56041 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
56060 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
56067 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
56068 return getSETCC(CC, Flags, DL, DAG);
56074 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
56083 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
56084 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
56085 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
56094 SelectionDAG &DAG) {
56111 DAG.ComputeNumSignBits(Op0.getOperand(0)) != NumEltBits ||
56128 // DAG.
56131 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
56134 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
56136 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
56137 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
56139 SDValue Res = DAG.getBitcast(VT, NewAnd);
56141 return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
56151 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
56175 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts);
56176 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
56178 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
56180 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
56183 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
56205 EVT::getVectorVT(*DAG.getContext(),
56210 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
56212 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
56214 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
56224 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
56228 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
56230 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
56237 if (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0)) {
56239 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},
56241 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
56247 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
56253 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
56276 EVT::getVectorVT(*DAG.getContext(),
56281 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
56283 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
56285 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
56295 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
56297 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
56299 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
56307 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0);
56314 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
56316 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
56318 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);
56323 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
56324 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
56327 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
56329 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf);
56353 Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG);
56354 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
56362 if (SDValue V = combineToFPTruncExtElt(N, DAG))
56369 static SDValue combineFP_TO_xINT_SAT(SDNode *N, SelectionDAG &DAG,
56380 SDValue V2F32Value = DAG.getUNDEF(SrcVT);
56383 SDValue NewSrc = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
56388 return DAG.getNode(X86ISD::FP_TO_SINT_SAT, dl, MVT::v2i64, NewSrc);
56390 return DAG.getNode(X86ISD::FP_TO_UINT_SAT, dl, MVT::v2i64, NewSrc);
56457 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
56471 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
56474 combineX86SubCmpForFlags(N, SDValue(N, 0), DAG, DCI, Subtarget))
56491 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),
56492 DAG.getConstant(Mask, dl, VT));
56493 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
56494 DAG.getConstant(0, dl, VT));
56514 BoolVec = widenMaskVector(BoolVec, false, Subtarget, DAG, dl);
56517 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth);
56520 Op = DAG.getBitcast(BCVT, BoolVec);
56521 Op = DAG.getNode(ISD::AND, dl, BCVT, Op,
56522 DAG.getConstant(Mask, dl, BCVT));
56523 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
56524 DAG.getConstant(0, dl, BCVT));
56534 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,
56535 DAG.getConstant(0, dl, SrcVT));
56551 if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) &&
56553 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
56554 DAG.getConstant(0, dl, OpVT));
56588 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
56589 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
56591 // Use a X86 specific opcode to avoid DAG combine messing with it.
56592 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
56593 Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1);
56597 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
56598 DAG.getConstant(0, dl, VT));
56604 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
56618 if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST))
56623 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
56624 return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
56630 SDVTList VTs = DAG.getVTList(N->getValueType(0));
56631 if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) {
56638 Op = DAG.getNegative(Op, DL, VT);
56648 return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG,
56652 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
56657 if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) {
56659 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
56660 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags);
56667 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0),
56674 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
56684 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS,
56696 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
56697 SDValue Res1 = DAG.getNode(
56699 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
56700 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),
56701 DAG.getConstant(1, DL, VT));
56711 return DAG.getNode(X86ISD::ADC, DL, N->getVTList(),
56712 DAG.getConstant(0, DL, LHS.getValueType()),
56713 DAG.getConstant(Sum, DL, LHS.getValueType()), CarryIn);
56716 if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) {
56718 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
56719 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags);
56726 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0),
56732 static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
56816 if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) ||
56820 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
56822 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
56823 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
56825 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
56829 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
56831 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
56833 SDValue R = SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDBuilder);
56835 R = DAG.getNode(ISD::ADD, DL, VT, R, Accum);
56842 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
56926 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
56932 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
56934 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
56939 EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
56942 In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
56943 DAG.getVectorIdxConstant(0, DL));
56946 In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
56947 DAG.getVectorIdxConstant(0, DL));
56949 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
56957 static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
56972 DAG.MaskedValueIsZero(N0.getOperand(0), DemandedBits, DemandedHiElts) ||
56973 DAG.MaskedValueIsZero(N0.getOperand(1), DemandedBits, DemandedHiElts);
56975 DAG.MaskedValueIsZero(N1.getOperand(0), DemandedBits, DemandedHiElts) ||
56976 DAG.MaskedValueIsZero(N1.getOperand(1), DemandedBits, DemandedHiElts);
56991 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
56993 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
56994 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
57002 SelectionDAG &DAG,
57052 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp);
57053 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp);
57054 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp,
57056 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y);
57060 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp);
57061 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp);
57062 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2),
57066 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
57074 if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget))
57077 if (SDValue MAdd = matchPMADDWD(DAG, N, DL, VT, Subtarget))
57079 if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
57081 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
57085 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
57093 if (DAG.willNotOverflowAdd(false, Op0.getOperand(0), Op1.getOperand(0))) {
57096 DAG.getNode(ISD::ADD, DL, OpVT, Op0.getOperand(0), Op1.getOperand(0));
57097 return DAG.getNode(X86ISD::PSADBW, DL, VT, Sum,
57098 getZeroVector(OpVT, Subtarget, DAG, DL));
57105 // generic DAG combine without a legal type check, but adding this there
57108 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57112 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
57113 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
57119 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
57120 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
57135 return DAG.getNode(X86ISD::VPDPWSSD, DL, VT, Accum,
57136 concatSubVectors(Lo0, Hi0, DAG, DL),
57137 concatSubVectors(Lo1, Hi1, DAG, DL));
57145 return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1,
57149 return combineAddOrSubToADCOrSBB(N, DL, DAG);
57157 SelectionDAG &DAG) {
57182 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp,
57185 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov);
57198 return DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp, N1.getOperand(2),
57205 static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
57223 SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG);
57224 NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC);
57225 return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC,
57226 DAG.getConstant(NewImm, DL, VT));
57232 static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) {
57253 return DAG.getMemIntrinsicNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops,
57258 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
57267 return DAG.isConstantIntBuildVectorOrConstantInt(Op,
57279 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0),
57280 DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT));
57282 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT));
57283 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd);
57286 if (SDValue V = combineSubABS(VT, DL, Op0, Op1, DAG))
57290 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
57297 return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0,
57306 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0,
57308 return DAG.getNode(ISD::SUB, DL, VT, ADC.getValue(0), Op1.getOperand(0));
57311 if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget))
57314 if (SDValue V = combineAddOrSubToADCOrSBB(N, DL, DAG))
57317 return combineSubSetcc(N, DAG);
57320 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
57334 return (Opcode == X86ISD::PCMPEQ) ? DAG.getAllOnesConstant(DL, VT)
57335 : DAG.getConstant(0, DL, VT);
57357 return getConstVector(Results, LHSUndefs | RHSUndefs, VT, DAG, DL);
57358 return getConstVector(Results, VT, DAG, DL);
57390 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
57397 return DAG.getUNDEF(VT);
57402 return getZeroVector(VT, Subtarget, DAG, DL);
57407 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57408 LLVMContext &Ctx = *DAG.getContext();
57415 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
57422 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
57423 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
57425 DAG.getVectorIdxConstant(0, DL)));
57433 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
57453 return DAG.getNode(Op0.getOpcode(), DL, VT,
57454 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
57475 return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
57476 DAG.getBitcast(VT, Src0.getOperand(0)),
57477 DAG.getBitcast(VT, Src1.getOperand(0)),
57478 DAG.getTargetConstant(0x31, DL, MVT::i8));
57485 return DAG.getBitcast(VT, extractSubVector(Src0.getOperand(0), 0, DAG,
57505 EVT::getVectorVT(*DAG.getContext(), SubVT.getScalarType(),
57508 Sub = DAG.getBitcast(SubVT, Sub);
57509 return DAG.getBitcast(
57510 VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Subs));
57512 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
57552 return DAG.getVectorShuffle(VT, DL, ConcatSubOperand(VT, Ops, 0),
57562 return DAG.getNode(X86ISD::UNPCKL, DL, VT,
57567 return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI
57570 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
57578 return DAG.getNode(Op0.getOpcode(), DL, VT,
57588 return DAG.getNode(Op0.getOpcode(), DL, VT,
57606 return DAG.getNode(Op0.getOpcode(), DL, VT,
57617 return DAG.getNode(Op0.getOpcode(), DL, VT,
57629 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0));
57631 DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1));
57632 return DAG.getBitcast(VT, Res);
57638 return DAG.getNode(Op0.getOpcode(), DL, VT,
57640 DAG.getTargetConstant(Idx, DL, MVT::i8));
57652 return DAG.getNode(Op0.getOpcode(), DL, VT,
57676 Ops[1].getOperand(1), DAG, DL);
57679 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
57680 return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src);
57708 Ops[1].getOperand(0), DAG, DL);
57710 Ops[1].getOperand(2), DAG, DL);
57713 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
57714 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1);
57729 Ops[0].getOperand(1), DAG, DL);
57731 Ops[1].getOperand(1), DAG, DL);
57732 SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
57733 DAG.getBitcast(ShuffleVT, LHS),
57734 DAG.getBitcast(ShuffleVT, RHS),
57735 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
57736 return DAG.getBitcast(VT, Res);
57748 Ops[0].getOperand(1), DAG, DL);
57750 Ops[1].getOperand(1), DAG, DL);
57751 return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS,
57752 DAG.getTargetConstant(Imm, DL, MVT::i8));
57765 return DAG.getNode(ISD::TRUNCATE, DL, VT,
57782 return DAG.getNode(Op0.getOpcode(), DL, VT,
57795 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
57796 SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
57798 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
57801 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
57804 return DAG.getBitcast(VT, Res);
57817 return DAG.getNode(Op0.getOpcode(), DL, VT,
57828 return DAG.getNode(Op0.getOpcode(), DL, VT,
57838 return DAG.getNode(Op0.getOpcode(), DL, VT,
57849 return DAG.getNode(Op0.getOpcode(), DL, VT,
57859 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(0)));
57862 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(1)));
57879 LHS = DAG.getNode(*CastOpc, DL, FpVT, LHS);
57880 RHS = DAG.getNode(*CastOpc, DL, FpVT, RHS);
57885 return DAG.getBitcast(
57886 VT, DAG.getNode(X86ISD::CMPP, DL, FpVT, LHS, RHS,
57887 DAG.getTargetConstant(FSETCC, DL, MVT::i8)));
57898 return DAG.getNode(Op0.getOpcode(), DL, VT,
57909 return DAG.getNode(Op0.getOpcode(), DL, VT,
57920 return DAG.getNode(Op0.getOpcode(), DL, VT,
57934 return DAG.getNode(Op0.getOpcode(), DL, VT,
57942 return DAG.getNode(Op0.getOpcode(), DL, VT,
57953 return DAG.getNode(Op0.getOpcode(), DL, VT,
57965 return DAG.getNode(Op0.getOpcode(), DL, VT,
57977 return DAG.getNode(Op0.getOpcode(), DL, VT,
57995 DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT));
57996 return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1),
58010 return DAG.getNode(Op0.getOpcode(), DL, VT,
58024 return DAG.getNode(Op0.getOpcode(), DL, VT,
58038 if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT,
58042 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
58063 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
58064 SDValue CV = DAG.getConstantPool(C, PVT);
58065 MachineFunction &MF = DAG.getMachineFunction();
58067 SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI);
58068 SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits());
58069 DAG.ReplaceAllUsesOfValueWith(Op0, Sub);
58087 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) {
58089 extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits());
58090 DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc);
58100 SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
58101 Res = DAG.getBitcast(ShuffleVT, Res);
58102 Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, Res, Res,
58103 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
58104 return DAG.getBitcast(VT, Res);
58110 static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
58115 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58127 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
58129 return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT));
58138 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
58146 static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
58164 return DAG.getUNDEF(OpVT);
58169 return getZeroVector(OpVT, Subtarget, DAG, dl);
58177 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
58178 getZeroVector(OpVT, Subtarget, DAG, dl),
58180 DAG.getVectorIdxConstant(IdxVal + Idx2Val, dl));
58195 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
58196 getZeroVector(OpVT, Subtarget, DAG, dl),
58212 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec,
58233 return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask);
58239 if (collectConcatOps(N, SubVectorOps, DAG)) {
58241 combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
58251 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
58252 getZeroVector(OpVT, Subtarget, DAG, dl),
58253 SubVectorOps[0], DAG.getVectorIdxConstant(0, dl));
58260 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
58267 return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
58274 SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
58277 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
58280 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
58291 DAG.areNonVolatileConsecutiveLoads(SubLd, VecLd,
58294 SubLd, 0, DAG);
58307 SelectionDAG &DAG) {
58310 !isFreeToSplitVector(Sel.getOperand(0).getNode(), DAG))
58350 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
58351 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
58352 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
58353 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
58354 return DAG.getBitcast(VT, NarrowSel);
58357 static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
58381 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58397 SDValue Concat = splitVectorIntBinary(InVecBC, DAG, SDLoc(InVecBC));
58398 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
58399 DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
58406 if (SDValue V = narrowExtractedVectorSelect(N, DL, DAG))
58410 return getZeroVector(VT, Subtarget, DAG, DL);
58414 return DAG.getConstant(1, DL, VT);
58415 return getOnesVector(VT, DAG, DL);
58419 return DAG.getBuildVector(VT, DL, InVec->ops().slice(IdxVal, NumSubElts));
58426 return extractSubVector(InVec.getOperand(0), NewIdx, DAG, DL, SizeInBits);
58437 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
58440 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt,
58442 DAG.getVectorIdxConstant(NewIdxVal, DL));
58450 DAG.isSplatValue(InVec, /*AllowUndefs*/ false)))
58451 return extractSubVector(InVec, 0, DAG, DL, SizeInBits);
58456 return extractSubVector(InVec, 0, DAG, DL, SizeInBits);
58465 if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
58469 return DAG.getUNDEF(VT);
58471 return getZeroVector(VT, Subtarget, DAG, DL);
58476 return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
58504 return DAG.getNode(X86ISD::CVTSI2P, DL, VT, InVec.getOperand(0));
58509 return DAG.getNode(X86ISD::CVTUI2P, DL, VT, InVec.getOperand(0));
58514 return DAG.getNode(X86ISD::VFPEXT, DL, VT, InVec.getOperand(0));
58525 return DAG.getNode(InOpcode, DL, VT,
58526 extractSubVector(Src, IdxVal, DAG, DL, SizeInBits));
58534 Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits);
58535 unsigned ExtOp = DAG.getOpcode_EXTEND_VECTOR_INREG(InOpcode);
58536 return DAG.getNode(ExtOp, DL, VT, Ext);
58542 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
58543 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
58544 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
58545 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
58551 SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
58552 return DAG.getNode(InOpcode, DL, VT, Ext);
58558 return DAG.getNode(
58560 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits));
58566 return DAG.getNode(InOpcode, DL, VT,
58567 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
58569 DAG.getTargetConstant(M, DL, MVT::i8));
58578 return DAG.getNode(InOpcode, DL, VT,
58579 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
58581 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
58587 return DAG.getNode(InOpcode, DL, VT,
58588 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
58590 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
58599 return DAG.getNode(InOpcode, DL, VT,
58600 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
58602 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
58604 DAG.getTargetConstant(M, DL, MVT::i8));
58612 Mask = extractSubVector(Mask, IdxVal, DAG, DL, SizeInBits);
58613 Mask = widenSubVector(Mask, /*ZeroNewElements=*/false, Subtarget, DAG,
58616 DAG.getNode(InOpcode, DL, InVecVT, Src0, Mask, Src1);
58617 return extractSubVector(Shuffle, 0, DAG, DL, SizeInBits);
58630 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
58631 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1));
58637 static SDValue combineSCALAR_TO_VECTOR(SDNode *N, SelectionDAG &DAG,
58650 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Src.getOperand(0));
58657 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
58663 auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) {
58676 KnownBits Known = DAG.computeKnownBits(Op);
58684 return DAG.getBitcast(
58685 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
58686 DAG.getAnyExtOrTrunc(AnyExt, DL, MVT::i32)));
58689 return DAG.getBitcast(
58691 DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32,
58692 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
58693 DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32))));
58700 return DAG.getBitcast(
58701 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, SrcOp));
58704 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, SrcOp);
58713 return DAG.getBitcast(
58714 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, HalfSrc));
58729 return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
58744 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Src.getOperand(0));
58747 Amt->getZExtValue(), DAG);
58761 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Src.getOperand(0));
58763 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Src.getOperand(1));
58764 return DAG.getNode(Src.getOpcode(), DL, VT, SrcVec0, SrcVec1,
58765 DAG.getConstant(AmtVal, DL, VT));
58775 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
58782 if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) &&
58783 !DAG.isConstantIntBuildVectorOrConstantInt(RHS))
58784 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS);
58789 return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
58792 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58808 LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
58810 LHS = DAG.getBitcast(MVT::v2i64, LHS);
58811 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
58818 RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
58820 RHS = DAG.getBitcast(MVT::v2i64, RHS);
58821 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
58828 static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG,
58842 return DAG.getConstant(0, SDLoc(N), VT);
58862 return getConstVector(Result, VT, DAG, SDLoc(N));
58865 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58873 static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
58880 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58894 SDValue Load = DAG.getExtLoad(
58897 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
58905 return DAG.getNode(Opcode, DL, VT, In.getOperand(0));
58911 In.getOperand(0).getOpcode() == DAG.getOpcode_EXTEND(Opcode) &&
58914 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0));
58924 SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT));
58927 return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts));
58934 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
58941 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
58944 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
58946 return DAG.getConstant(0, SDLoc(N), VT);
58960 SDValue Shift = DAG.getNode(X86ISD::KSHIFTR, DL, SrcVT, Src,
58961 DAG.getTargetConstant(Amt, DL, MVT::i8));
58962 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shift,
58963 DAG.getVectorIdxConstant(0, DL));
58978 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
58991 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
58993 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
58994 DAG.getTargetConstant(4, dl, MVT::i32));
58995 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
58996 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
58997 DAG.getVectorIdxConstant(0, dl));
59000 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
59020 return DAG.getNode(ISD::FP_EXTEND, dl, VT,
59021 DAG.getNode(ISD::FP_EXTEND, dl, TmpVT, Src));
59025 Src = DAG.getBitcast(SrcVT.changeTypeToInteger(), Src);
59026 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Src);
59027 Src = DAG.getNode(ISD::SHL, dl, NVT, Src, DAG.getConstant(16, dl, NVT));
59028 return DAG.getBitcast(VT, Src);
59050 Src = DAG.getBitcast(IntVT, Src);
59055 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
59056 : DAG.getConstant(0, dl, IntVT);
59059 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
59063 EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
59067 Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
59071 Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src);
59076 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt,
59077 DAG.getVectorIdxConstant(0, dl));
59083 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other},
59087 return DAG.getMergeValues({Cvt, Chain}, dl);
59091 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt);
59097 static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
59124 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
59126 Extract = DAG.getBitcast(VT, Extract);
59133 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
59173 Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
59175 Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
59177 Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
59178 return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl);
59182 Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0));
59183 Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0));
59184 return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
59194 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
59195 DAG.getConstantFP(0.0, dl, SrcVT));
59199 EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts));
59200 SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
59202 Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},
59206 Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd);
59212 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt,
59213 DAG.getVectorIdxConstant(0, dl));
59216 Cvt = DAG.getBitcast(VT, Cvt);
59219 return DAG.getMergeValues({Cvt, Chain}, dl);
59224 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
59232 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
59237 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
59245 static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG,
59248 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
59258 static SDValue FixupMMXIntrinsicTypes(SDNode *N, SelectionDAG &DAG) {
59266 Args.push_back(DAG.getBitcast(MVT::x86mmx, Arg));
59275 NewVTs = DAG.getVTList(NewVTArr);
59281 SDValue Result = DAG.getNode(N->getOpcode(), dl, NewVTs, Args);
59286 Returns[0] = DAG.getBitcast(MVT::v1i64, Returns[0]);
59287 return DAG.getMergeValues(Returns, dl);
59293 static SDValue combineINTRINSIC_WO_CHAIN(SDNode *N, SelectionDAG &DAG,
59302 return FixupMMXIntrinsicTypes(N, DAG);
59307 static SDValue combineINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
59316 return FixupMMXIntrinsicTypes(N, DAG);
59321 static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
59330 return FixupMMXIntrinsicTypes(N, DAG);
59337 SelectionDAG &DAG = DCI.DAG;
59342 return combineSCALAR_TO_VECTOR(N, DAG, Subtarget);
59346 return combineExtractVectorElt(N, DAG, DCI, Subtarget);
59348 return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget);
59350 return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget);
59352 return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget);
59355 case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget);
59356 case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget);
59357 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
59358 case X86ISD::CMP: return combineCMP(N, DAG, DCI, Subtarget);
59359 case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget);
59360 case ISD::SUB: return combineSub(N, DAG, DCI, Subtarget);
59362 case X86ISD::SUB: return combineX86AddSub(N, DAG, DCI, Subtarget);
59364 case X86ISD::CSTORE: return combineX86CloadCstore(N, DAG);
59365 case X86ISD::SBB: return combineSBB(N, DAG);
59366 case X86ISD::ADC: return combineADC(N, DAG, DCI);
59367 case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
59368 case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget);
59369 case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget);
59370 case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget);
59371 case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget);
59372 case ISD::OR: return combineOr(N, DAG, DCI, Subtarget);
59373 case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget);
59374 case ISD::BITREVERSE: return combineBITREVERSE(N, DAG, DCI, Subtarget);
59378 case ISD::AVGFLOORU: return combineAVG(N, DAG, DCI, Subtarget);
59380 case X86ISD::BEXTRI: return combineBEXTR(N, DAG, DCI, Subtarget);
59381 case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget);
59382 case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget);
59383 case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget);
59384 case ISD::MSTORE: return combineMaskedStore(N, DAG, DCI, Subtarget);
59386 return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget);
59389 return combineSIntToFP(N, DAG, DCI, Subtarget);
59392 return combineUIntToFP(N, DAG, Subtarget);
59394 case ISD::LLRINT: return combineLRINT_LLRINT(N, DAG, Subtarget);
59396 case ISD::FSUB: return combineFaddFsub(N, DAG, Subtarget);
59398 case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
59399 case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
59400 case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
59401 case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
59402 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget);
59403 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
59404 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
59406 case X86ISD::FOR: return combineFOr(N, DAG, DCI, Subtarget);
59408 case X86ISD::FMAX: return combineFMinFMax(N, DAG);
59410 case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget);
59412 case X86ISD::CVTUI2P: return combineX86INT_TO_FP(N, DAG, DCI);
59419 return combineCVTP2I_CVTTP2I(N, DAG, DCI);
59421 case X86ISD::CVTPH2PS: return combineCVTPH2PS(N, DAG, DCI);
59422 case X86ISD::BT: return combineBT(N, DAG, DCI);
59424 case ISD::ZERO_EXTEND: return combineZext(N, DAG, DCI, Subtarget);
59425 case ISD::SIGN_EXTEND: return combineSext(N, DAG, DCI, Subtarget);
59426 case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget);
59430 return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget);
59431 case ISD::SETCC: return combineSetCC(N, DAG, DCI, Subtarget);
59432 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget);
59433 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget);
59435 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget);
59439 case X86ISD::FHSUB: return combineVectorHADDSUB(N, DAG, DCI, Subtarget);
59443 return combineVectorShiftVar(N, DAG, DCI, Subtarget);
59447 return combineVectorShiftImm(N, DAG, DCI, Subtarget);
59450 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget);
59485 case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget);
59497 case ISD::STRICT_FMA: return combineFMA(N, DAG, DCI, Subtarget);
59501 case X86ISD::FMSUBADD: return combineFMADDSUB(N, DAG, DCI);
59502 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI, Subtarget);
59503 case X86ISD::TESTP: return combineTESTP(N, DAG, DCI, Subtarget);
59505 case X86ISD::MSCATTER: return combineX86GatherScatter(N, DAG, DCI);
59507 case ISD::MSCATTER: return combineGatherScatter(N, DAG, DCI);
59509 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget);
59511 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI, Subtarget);
59513 case X86ISD::VPMADDWD: return combineVPMADD(N, DAG, DCI);
59515 case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI);
59516 case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget);
59518 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, DCI, Subtarget);
59520 case ISD::FP_ROUND: return combineFP_ROUND(N, DAG, Subtarget);
59522 case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI);
59523 case X86ISD::MOVDQ2Q: return combineMOVDQ2Q(N, DAG);
59524 case X86ISD::PDEP: return combinePDEP(N, DAG, DCI);
59525 case ISD::INTRINSIC_WO_CHAIN: return combineINTRINSIC_WO_CHAIN(N, DAG, DCI);
59526 case ISD::INTRINSIC_W_CHAIN: return combineINTRINSIC_W_CHAIN(N, DAG, DCI);
59527 case ISD::INTRINSIC_VOID: return combineINTRINSIC_VOID(N, DAG, DCI);
59529 case ISD::FP_TO_UINT_SAT: return combineFP_TO_xINT_SAT(N, DAG, Subtarget);
59599 SelectionDAG &DAG) const {
59600 const Module *M = DAG.getMachineFunction().getFunction().getParent();
59609 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF())
59610 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, dl);
59611 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Chain, Addr);
59614 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
60131 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
60142 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue);
60145 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
60147 SDValue CC = getSETCC(Cond, Glue, DL, DAG);
60149 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
60159 SelectionDAG &DAG) const {
60167 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60176 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60185 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60195 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
60204 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60213 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60222 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60231 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
60234 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
60247 Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
60257 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
60265 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
60267 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
60285 Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);
60312 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);