Lines Matching defs:Zeroable
3876 const APInt &Zeroable,
3883 assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
3885 if (Mask[i] != SM_SentinelUndef && Zeroable[i])
6882 std::bitset<4> Zeroable, Undefs;
6886 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
6888 assert(Zeroable.size() - Zeroable.count() > 1 &&
6896 if (Zeroable[i])
6921 if (Zeroable[EltIdx]) {
6937 SDValue VZeroOrUndef = (Zeroable == Undefs)
6955 if (Zeroable[i])
6975 unsigned ZMask = Zeroable.to_ulong();
10065 // Each Zeroable's element correspond to a particular Mask's element.
10070 static bool isNonZeroElementsInOrder(const APInt &Zeroable,
10080 if (Zeroable[i])
10098 SDValue V2, const APInt &Zeroable,
10121 if (Zeroable[i / NumEltBytes]) {
10156 const APInt &Zeroable,
10160 if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
10163 unsigned VEXPANDMask = (~Zeroable).getZExtValue();
10194 "Zeroable shuffle detected");
10320 ArrayRef<int> Mask, const APInt &Zeroable,
10337 if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10420 const APInt &Zeroable,
10435 !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10470 const APInt &Zeroable,
10499 !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10745 const APInt &Zeroable,
10772 if (Zeroable[i])
10823 const APInt &Zeroable, bool &ForceV1Zero,
10870 if (Zeroable[Elt]) {
10907 const APInt &Zeroable,
10913 if (!matchShuffleAsBlend(VT, V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero,
10978 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11045 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11418 const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11435 V1Zero &= Zeroable[i];
11440 V2Zero &= Zeroable[i];
11767 const APInt &Zeroable,
11788 unsigned ZeroLo = Zeroable.countr_one();
11789 unsigned ZeroHi = Zeroable.countl_one();
11790 assert((ZeroLo + ZeroHi) < NumElts && "Zeroable shuffle detected");
11818 const APInt &Zeroable,
11826 unsigned ZeroLo = Zeroable.countr_one();
11827 unsigned ZeroHi = Zeroable.countl_one();
11905 int MaskOffset, const APInt &Zeroable,
11913 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
11967 const APInt &Zeroable,
11979 Mask, 0, Zeroable, Subtarget);
11984 Mask, Size, Zeroable, Subtarget);
12006 uint64_t &BitIdx, const APInt &Zeroable) {
12010 assert(!Zeroable.isAllOnes() && "Fully zeroable shuffle mask");
12020 if (!Zeroable[Len - 1])
12022 assert(Len > 0 && "Zeroable shuffle mask");
12128 const APInt &Zeroable, SelectionDAG &DAG) {
12130 if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
12319 const APInt &Zeroable, const X86Subtarget &Subtarget,
12342 if (!Zeroable[i])
12414 if (!Zeroable[i])
12482 const APInt &Zeroable, const X86Subtarget &Subtarget,
12498 if (i != V2Index && !Zeroable[i]) {
12958 const APInt &Zeroable,
12976 if (Zeroable[i]) {
13045 ArrayRef<int> Mask, const APInt &Zeroable,
13052 if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
13068 const APInt &Zeroable, SDValue V1, SDValue V2,
13110 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13117 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13133 Zeroable, Subtarget, DAG))
13152 const APInt &Zeroable, SDValue V1, SDValue V2,
13189 lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget,
13196 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13202 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13210 Zeroable, Subtarget, DAG))
13222 Zeroable, Subtarget, DAG))
13234 Zeroable, Subtarget, DAG);
13344 const APInt &Zeroable, SDValue V1, SDValue V2,
13353 Zeroable, Subtarget, DAG))
13396 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) {
13412 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13417 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13448 const APInt &Zeroable, SDValue V1, SDValue V2,
13459 Zeroable, Subtarget, DAG))
13467 lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable,
13506 lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget,
13513 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13521 Zeroable, Subtarget, DAG))
13525 Zeroable, Subtarget, DAG))
13537 Zeroable, Subtarget, DAG))
13553 Zeroable, Subtarget, DAG);
14084 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
14105 if (Zeroable[i / Scale])
14146 const APInt &Zeroable, SDValue V1, SDValue V2,
14156 Zeroable, Subtarget, DAG))
14160 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
14169 lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, Zeroable,
14209 lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget,
14216 Zeroable, DAG))
14222 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14230 Zeroable, Subtarget, DAG))
14234 Zeroable, Subtarget, DAG))
14247 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
14262 Zeroable, Subtarget, DAG))
14339 Zeroable, DAG, V1InUse, V2InUse);
14345 Zeroable, Subtarget, DAG);
14350 const APInt &Zeroable, SDValue V1, SDValue V2,
14367 DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14433 const APInt &Zeroable, SDValue V1, SDValue V2,
14442 lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget,
14458 Zeroable, Subtarget, DAG))
14462 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14466 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14473 Zeroable, DAG))
14590 Zeroable, Subtarget, DAG))
14599 Zeroable, Subtarget, DAG))
14627 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14635 Zeroable, Subtarget, DAG))
14674 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14731 Zeroable, Subtarget, DAG);
14788 const APInt &Zeroable,
14800 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14802 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14804 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14806 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14808 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14810 return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14812 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14955 const APInt &Zeroable,
14982 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
15002 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
15236 const APInt &Zeroable,
15261 if (!canWidenShuffleElements(Mask, Zeroable, V2IsZero, WidenedMask))
15264 bool IsLowZero = (Zeroable & 0x3) == 0x3;
15265 bool IsHighZero = (Zeroable & 0xc) == 0xc;
15282 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
15935 const APInt &Zeroable) {
15945 ZeroLane[i & 1] &= Zeroable[i];
15980 const APInt &Zeroable,
15989 Mask, Zeroable))
16008 const APInt &Zeroable,
16017 if (Zeroable.countl_one() < (Mask.size() - 8))
16127 const APInt &Zeroable, SDValue V1, SDValue V2,
16134 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
16183 Zeroable, Subtarget, DAG))
16188 Zeroable, Subtarget, DAG))
16208 Zeroable, Subtarget, DAG);
16228 Zeroable, Subtarget, DAG))
16235 Zeroable, Subtarget, DAG);
16238 return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
16247 const APInt &Zeroable, SDValue V1, SDValue V2,
16255 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
16260 Zeroable, Subtarget, DAG))
16271 lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
16297 lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable, Subtarget,
16304 Zeroable, Subtarget, DAG))
16308 Zeroable, Subtarget, DAG))
16328 Zeroable, Subtarget, DAG);
16352 Zeroable, Subtarget, DAG);
16360 const APInt &Zeroable, SDValue V1, SDValue V2,
16368 Zeroable, Subtarget, DAG))
16386 Zeroable, Subtarget, DAG))
16446 Zeroable, Subtarget, DAG))
16463 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
16470 Zeroable, Subtarget, DAG);
16473 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
16482 const APInt &Zeroable, SDValue V1, SDValue V2,
16496 Zeroable, Subtarget, DAG))
16511 return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16515 Zeroable, Subtarget, DAG))
16526 lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16554 lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget,
16566 Zeroable, Subtarget, DAG))
16570 Zeroable, Subtarget, DAG))
16616 Zeroable, Subtarget, DAG);
16624 const APInt &Zeroable, SDValue V1, SDValue V2,
16636 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16645 Zeroable, Subtarget, DAG))
16658 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16664 lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16712 Zeroable, Subtarget, DAG))
16738 return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16747 const APInt &Zeroable, SDValue V1, SDValue V2,
16759 Zeroable, Subtarget, DAG))
16768 Zeroable, Subtarget, DAG))
16781 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16787 lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget,
16825 Zeroable, Subtarget, DAG))
16848 Mask, Zeroable, DAG))
16859 return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16869 SDValue V1, SDValue V2, const APInt &Zeroable,
16879 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16898 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16922 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16924 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16926 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16928 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16930 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16932 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16941 const APInt &Zeroable, SDValue V1, SDValue V2,
16951 // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
16958 if (Widened128Mask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
16959 (Widened128Mask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
16960 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
17047 const APInt &Zeroable, SDValue V1, SDValue V2,
17076 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
17085 Zeroable, Subtarget, DAG))
17088 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8f64, V1, V2, Mask, Zeroable,
17093 Zeroable, Subtarget, DAG))
17101 const APInt &Zeroable, SDValue V1, SDValue V2,
17129 Zeroable, Subtarget, DAG))
17137 Zeroable, Subtarget, DAG))
17141 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
17160 Zeroable, Subtarget, DAG))
17168 const APInt &Zeroable, SDValue V1, SDValue V2,
17178 lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
17203 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
17209 lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable, Subtarget,
17215 Zeroable, Subtarget, DAG))
17228 if (SDValue V = lowerShuffleWithEXPAND(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
17233 Zeroable, Subtarget, DAG))
17241 const APInt &Zeroable, SDValue V1, SDValue V2,
17254 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
17260 lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
17288 lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
17299 Zeroable, Subtarget, DAG))
17326 Zeroable, Subtarget, DAG))
17330 Zeroable, Subtarget, DAG))
17338 const APInt &Zeroable, SDValue V1, SDValue V2,
17350 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
17364 lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask, Zeroable,
17390 Zeroable, Subtarget, DAG))
17394 Zeroable, Subtarget, DAG))
17409 const APInt &Zeroable, SDValue V1, SDValue V2,
17421 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
17435 lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget,
17452 Zeroable, Subtarget, DAG))
17456 Zeroable, Subtarget, DAG))
17470 Zeroable, Subtarget, DAG))
17483 return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v64i8, V1, V2, Mask, Zeroable,
17508 const APInt &Zeroable,
17521 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17537 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17563 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17565 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17567 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17569 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17571 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17573 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17622 int MaskOffset, const APInt &Zeroable) {
17627 if (!Zeroable[j + (Left ? 0 : (Size - Shift))])
17657 const APInt &Zeroable,
17688 if ((int)Zeroable.countl_one() >= (NumElts - SubvecElts)) {
17708 int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable);
17973 APInt Zeroable = KnownUndef | KnownZero;
17974 if (Zeroable.isAllOnes())
17987 canWidenShuffleElements(OrigMask, Zeroable, V2IsZero, WidenedMask)) {
18050 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18053 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18056 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18059 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
38998 const APInt &Zeroable,
39125 Zeroable, Subtarget);
39343 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
39389 if (matchShuffleAsBlend(MaskVT, V1, V2, TargetMask, Zeroable, ForceV1Zero,
39423 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
39436 PermuteImm, Mask, Zeroable)) {
39495 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
39796 APInt Zeroable = KnownUndef | KnownZero;
39835 if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
39857 if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask,
39900 if (matchBinaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
39921 Zeroable)) {
39946 if (matchShuffleAsVTRUNC(ShuffleSrcVT, ShuffleVT, IntMaskVT, Mask, Zeroable,