| H A D | X86ISelLowering.cpp | 1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation ------- 3880 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) widenSubVector() local 4178 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps); getAVX512Node() local 7485 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps); LowerBUILD_VECTORvXbf16() local 8279 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); lowerBuildVectorToBitOp() local 8493 SDValue Res = DAG.getSelectCC( createVariablePermute() local 8510 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL, createVariablePermute() local 8529 SDValue Res = DAG.getSelectCC( createVariablePermute() local 8568 SDValue Res = Opcode == X86ISD::VPERMV createVariablePermute() local 10318 SDValue Res; lowerShuffleWithPACK() local 11446 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2; lowerShuffleAsByteShiftMask() local 17174 SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL); lower1BitShuffleAsKSHIFTR() local 17273 SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL); lower1BitShuffle() local 17961 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG)) LowerEXTRACT_VECTOR_ELT() local 17974 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, LowerEXTRACT_VECTOR_ELT() local 17986 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, LowerEXTRACT_VECTOR_ELT() local 18073 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT, LowerINSERT_VECTOR_ELT() local 18780 SDValue res; LowerGlobalTLSAddress() local 19044 SDValue Res, Chain; lowerINT_TO_FP_vXi64() local 19434 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other}, lowerUINT_TO_FP_v2i32() local 19496 SDValue Res, Chain; lowerUINT_TO_FP_vXi32() local 19951 SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI); FP_TO_INTHelper() local 20033 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi); SplitAndExtendv16i1() local 20159 SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS); truncateVectorWithPACK() local 20172 if (SDValue Res = truncateVectorWithPACK() local 20185 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); truncateVectorWithPACK() local 20194 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); truncateVectorWithPACK() local 20218 SDValue Res = truncateVectorWithPACK() local 20226 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); truncateVectorWithPACK() local 20361 if (SDValue Res = LowerTruncateVecPackWithSignBits(DstHalfVT, Lo, DL, LowerTruncateVecPackWithSignBits() local 20404 if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG)) LowerTruncateVecPack() local 20680 SDValue Res; LowerFP_TO_INT() local 21300 SDValue Res; LowerFP_EXTEND() local 21312 SDValue Res; LowerFP_EXTEND() local 21354 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In, LowerFP_EXTEND() local 21366 SDValue Res = LowerFP_EXTEND() local 21411 SDValue Res; LowerFP_ROUND() local 21437 SDValue Res; LowerFP_ROUND() local 21473 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, LowerFP16_TO_FP() local 21502 SDValue Res, Chain; LowerFP_TO_FP16() local 21534 SDValue Res; LowerFP_TO_BF16() local 21544 SDValue Res = LowerFP_TO_BF16() local 21796 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0); LowerFGETSIGN() local 23694 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); LowerSETCC() local 23715 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); LowerSETCC() local 24095 SDValue Res = LowerSELECT() local 25875 SDValue Res; LowerINTRINSIC_WO_CHAIN() local 26313 SDValue Res = getAVX2GatherNode() local 26351 SDValue Res = getGatherNode() local 26382 SDValue Res = getScatterNode() local 26406 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); getPrefetchNode() local 26790 SDValue Res = LowerINTRINSIC_W_CHAIN() local 26810 SDValue Res = LowerINTRINSIC_W_CHAIN() local 27771 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi); LowerVectorCTLZInRegLUT() local 28529 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask); LowerMULH() local 28607 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); LowerMULO() local 29029 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); LowerShiftByScalarImmediate() local 29080 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, LowerShiftByScalarVariable() local 29295 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale); LowerShift() local 29318 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale); LowerShift() local 29759 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1); LowerFunnelShift() local 29800 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift); LowerFunnelShift() local 30688 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, emitLockedStackOp() local 30703 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, emitLockedStackOp() local 31037 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In); LowerBITREVERSE_XOP() local 31067 SDValue Res = DAG.getBitcast(MVT::v16i8, In); LowerBITREVERSE_XOP() local 32029 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); ReplaceNodeResults() local 32045 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); ReplaceNodeResults() local 32082 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1); ReplaceNodeResults() local 32100 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1); ReplaceNodeResults() local 32153 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1); ReplaceNodeResults() local 32191 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1); ReplaceNodeResults() local 32222 if (SDValue Res = truncateVectorWithPACK(PackOpcode, VT, Src, ReplaceNodeResults() local 32273 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi, ReplaceNodeResults() local 32330 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); ReplaceNodeResults() local 32368 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); ReplaceNodeResults() local 32385 SDValue Res; ReplaceNodeResults() local 32457 SDValue Res; ReplaceNodeResults() local 32505 SDValue Res = ReplaceNodeResults() local 32530 SDValue Res; ReplaceNodeResults() local 32550 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other}, ReplaceNodeResults() local 32583 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT, ReplaceNodeResults() local 32644 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other}, ReplaceNodeResults() local 32661 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other}, ReplaceNodeResults() local 32723 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, ReplaceNodeResults() local 32740 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, ReplaceNodeResults() local 32925 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld, ReplaceNodeResults() local 32934 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld, ReplaceNodeResults() local 33006 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); ReplaceNodeResults() local 33017 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, ReplaceNodeResults() local 33051 SDValue Res = DAG.getMemIntrinsicNode( ReplaceNodeResults() local 33073 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(), ReplaceNodeResults() local 33088 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, ReplaceNodeResults() local 36682 std::optional<bool> Res = Opc == X86ISD::PCMPEQ computeKnownBitsForTargetNode() local 37785 SDValue Res; combineX86ShuffleChain() local 38714 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS); canonicalizeShuffleMaskWithHorizOp() local 38814 SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1); canonicalizeShuffleMaskWithHorizOp() local 39249 if (SDValue Res = combineX86ShufflesRecursively( combineX86ShufflesRecursively() local 39769 SDValue Res; canonicalizeShuffleWithOp() local 39810 SDValue Res = canonicalizeLaneShuffleWithRepeatedOps() local 39830 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, canonicalizeLaneShuffleWithRepeatedOps() local 39887 if (SDValue Res = combineX86ShufflesRecursively( combineTargetShuffle() local 40254 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1); combineTargetShuffle() local 40303 if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL)) combineTargetShuffle() local 40352 SDValue Res = DAG.getNode(Opcode, DL, VT, combineTargetShuffle() local 40882 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineShuffle() local 41058 __anon973982078c02(SDNode *Use) SimplifyDemandedVectorEltsForTargetNode() argument 43039 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, combineBitcast() local 44262 __anon973982079902(SDNode *Use) combineExtractVectorElt() argument 44284 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask); combineExtractVectorElt() local 44300 __anon973982079a02(SDNode *Use) combineExtractVectorElt() argument 44787 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2); combineLogicBlendIntoConditionalNegate() local 45117 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS); combineSelect() local 45798 if (SDValue Res = combinePTESTCC() local 47183 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); combineHorizOpWithShuffle() local 47241 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); combineHorizOpWithShuffle() local 47277 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), combineHorizOpWithShuffle() local 47434 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineVectorPack() local 47464 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(), combineVectorHADDSUB() local 47589 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineVectorShiftImm() local 47699 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineVectorInsert() local 48711 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineAnd() local 48858 SDValue Res = getAVX512Node(X86ISD::VPTERNLOG, DL, OpVT, {A, B, C, Imm}, canonicalizeBitSelect() local 48938 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL, combineLogicBlendIntoPBLENDV() local 49469 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineOr() local 49792 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal); combineTruncateWithSat() local 49887 SDValue Res = SplitOpsAndApply(DAG, Subtarget, DL, Pow2VT, Ops, AVGBuilder); detectAVGPattern() local 50980 SDValue Res; combineFMulcFCMulc() local 51226 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS), combinePMULH() local 52248 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineAndnp() local 52571 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1, combineToExtendCMOV() local 52619 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); combineExtSetcc() local 52690 __anon97398207cf02(SDNode *Use) getInvertedVectorForFMA() argument 53230 SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS); combineMOVMSK() local 53514 SDValue Res = DAG.getBitcast(VT, NewAnd); combineVectorCompareAndMaskUnaryOp() local 53627 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG)) combineSIntToFP() local 53958 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS); combineX86AddSub() local 54397 __anon97398207d702(SDNode *Use) pushAddIntoCmovOfConsts() argument 54595 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0, combineSub() local 54822 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0)); combineConcatVectorOps() local 54921 SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, combineConcatVectorOps() local 54967 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0)); combineConcatVectorOps() local 55215 SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512); combineConcatVectorOps() local 55375 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineINSERT_SUBVECTOR() local 55880 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) combineEXTEND_VECTOR_INREG() local 55918 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, combineFP16_TO_FP() local 57413 std::pair<Register, const TargetRegisterClass*> Res; getRegForInlineAsmConstraint() local [all...] |