Lines Matching defs:X86
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
9 // This file defines the interfaces that X86 uses to lower LLVM code into a
16 #include "X86.h"
137 // X86 is weird. It always uses i8 for shift amounts and setcc results.
139 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
142 // X86 instruction cache is coherent with its data cache so we can use the
198 addRegisterClass(MVT::i8, &X86::GR8RegClass);
199 addRegisterClass(MVT::i16, &X86::GR16RegClass);
200 addRegisterClass(MVT::i32, &X86::GR32RegClass);
202 addRegisterClass(MVT::i64, &X86::GR64RegClass);
260 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
275 // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
291 // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
508 // X86 wants to expand cmov itself.
665 addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
666 : &X86::FR16RegClass);
667 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
668 : &X86::FR32RegClass);
669 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
670 : &X86::FR64RegClass);
750 addRegisterClass(MVT::f32, &X86::FR32RegClass);
752 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
782 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
783 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
798 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
808 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
840 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
905 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
906 : &X86::VR128RegClass);
1069 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
1074 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1075 : &X86::VR128RegClass);
1104 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1105 : &X86::VR128RegClass);
1109 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1110 : &X86::VR128RegClass);
1111 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1112 : &X86::VR128RegClass);
1113 addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1114 : &X86::VR128RegClass);
1115 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1116 : &X86::VR128RegClass);
1117 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1118 : &X86::VR128RegClass);
1453 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1454 : &X86::VR256RegClass);
1455 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1456 : &X86::VR256RegClass);
1457 addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1458 : &X86::VR256RegClass);
1459 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1460 : &X86::VR256RegClass);
1461 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1462 : &X86::VR256RegClass);
1463 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1464 : &X86::VR256RegClass);
1465 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1466 : &X86::VR256RegClass);
1738 addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1739 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1740 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1741 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1742 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1816 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1817 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1818 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1819 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1820 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1821 addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1822 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
2210 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
2211 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
2420 addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2421 : &X86::VR128RegClass);
2422 addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2423 : &X86::VR256RegClass);
2450 addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2546 addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2748 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2773 return X86::createFastISel(funcInfo, libInfo);
2780 bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2799 bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2803 if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2813 bool X86::mayFoldIntoStore(SDValue Op) {
2826 bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2917 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
2951 case X86::COND_E:
2952 case X86::COND_NE:
2953 case X86::COND_B:
2954 case X86::COND_A:
2955 case X86::COND_BE:
2956 case X86::COND_AE:
2958 case X86::COND_G:
2959 case X86::COND_GE:
2960 case X86::COND_L:
2961 case X86::COND_LE:
2966 static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2970 case ISD::SETEQ: return X86::COND_E;
2971 case ISD::SETGT: return X86::COND_G;
2972 case ISD::SETGE: return X86::COND_GE;
2973 case ISD::SETLT: return X86::COND_L;
2974 case ISD::SETLE: return X86::COND_LE;
2975 case ISD::SETNE: return X86::COND_NE;
2976 case ISD::SETULT: return X86::COND_B;
2977 case ISD::SETUGT: return X86::COND_A;
2978 case ISD::SETULE: return X86::COND_BE;
2979 case ISD::SETUGE: return X86::COND_AE;
2984 /// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2987 static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2995 return X86::COND_NS;
2999 return X86::COND_S;
3003 return X86::COND_NS;
3008 return X86::COND_LE;
3044 case ISD::SETEQ: return X86::COND_E;
3047 case ISD::SETGT: return X86::COND_A;
3050 case ISD::SETGE: return X86::COND_AE;
3053 case ISD::SETLT: return X86::COND_B;
3056 case ISD::SETLE: return X86::COND_BE;
3058 case ISD::SETNE: return X86::COND_NE;
3059 case ISD::SETUO: return X86::COND_P;
3060 case ISD::SETO: return X86::COND_NP;
3062 case ISD::SETUNE: return X86::COND_INVALID;
3067 /// Is there a floating point cmov for the specific X86 condition code?
3074 case X86::COND_B:
3075 case X86::COND_BE:
3076 case X86::COND_E:
3077 case X86::COND_P:
3078 case X86::COND_A:
3079 case X86::COND_AE:
3080 case X86::COND_NE:
3081 case X86::COND_NP:
3932 bool X86::isZeroNode(SDValue Elt) {
4169 return V.isUndef() || (ZeroNewElements && X86::isZeroNode(V));
5226 namespace X86 {
5251 } // namespace X86
5790 if (X86::isZeroNode(Op))
5815 AllZero &= X86::isZeroNode(Op);
5898 else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
6217 if (X86::isZeroNode(Scl)) {
6886 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
7147 if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) {
7382 !X86::mayFoldLoadIntoBroadcastFromMem(
8314 // There are no known X86 targets with 512-bit ADDSUB instructions!
9041 if (X86::isZeroNode(Elt)) {
9220 X86::isZeroNode(Op.getOperand(0)) &&
9221 !X86::isZeroNode(Op.getOperand(1))) {
9612 /// used by the X86 shuffle instructions (not a fully general
9627 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
10153 // X86 has dedicated shuffle that can be lowered to VEXPAND
10264 // X86 has dedicated unpack instructions that can handle specific blend
10618 // X86 has dedicated pack instructions that can handle specific truncation
11413 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
11453 !X86::mayFoldLoad(Input, Subtarget)))
15244 X86::mayFoldLoad(peekThroughOneUseBitcasts(V1), Subtarget)) {
18272 if (llvm::isNullConstant(Idx) && !X86::mayFoldIntoZeroExtend(Op) &&
18273 !X86::mayFoldIntoStore(Op))
18461 if (IdxVal == 0 && !X86::mayFoldIntoZeroExtend(Op) &&
18462 !(Subtarget.hasSSE41() && X86::mayFoldIntoStore(Op))) {
18627 bool IsZeroElt = X86::isZeroNode(N1);
18683 X86::mayFoldLoad(N1, Subtarget)))) {
18756 if (IdxVal == 0 && (!MinSize || !X86::mayFoldLoad(N1, Subtarget))) {
18789 if (X86::isZeroNode(Op.getOperand(0)))
18989 X86::isOffsetSuitableForCodeModel(Offset, M, true)) {
19074 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX,
19108 return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD,
19116 return GetTLSADDR(DAG, GA, PtrVT, X86::RAX, X86II::MO_TLSGD);
19123 return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD);
19138 unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX;
19143 Base = GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSLDM,
19293 unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
21664 // but making use of X86 specifics to produce better instruction sequences.
21994 SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
22188 /// This mode isn't supported in hardware on X86. But as long as we aren't
22404 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
22581 X86::CondCode X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
22674 SelectionDAG &DAG, X86::CondCode &X86CC) {
22691 X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
22823 X86::CondCode &X86CC) {
22986 case X86::COND_A: case X86::COND_AE:
22987 case X86::COND_B: case X86::COND_BE:
22990 case X86::COND_G: case X86::COND_GE:
22991 case X86::COND_L: case X86::COND_LE:
22992 case X86::COND_O: case X86::COND_NO: {
23104 !X86::mayFoldLoad(Op0, Subtarget) && !X86::mayFoldLoad(Op1, Subtarget) &&
23113 if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
23145 Op0.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
23154 Op1.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
23355 SelectionDAG &DAG, X86::CondCode &X86CC) {
23414 X86CC = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
24118 X86::CondCode X86Cond;
24120 X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
24123 X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
24155 /// corresponding X86 condition code constant in X86CC.
24162 X86::CondCode X86CondCode;
24196 X86CondCode = (X86::CondCode)Op0.getConstantOperandVal(0);
24197 X86CondCode = X86::GetOppositeBranchCondition(X86CondCode);
24211 X86::CondCode CondCode = CC == ISD::SETEQ ? X86::COND_O : X86::COND_NO;
24229 X86CondCode = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
24236 X86::CondCode CondCode =
24238 assert(CondCode != X86::COND_INVALID && "Unexpected condition code!");
24316 auto NewCC = (CC == ISD::SETOEQ) ? X86::COND_E : (X86::COND_NE);
24324 X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
24325 if (CondCode == X86::COND_INVALID)
24352 X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
24369 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
24380 Cond = X86::COND_O;
24384 Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
24388 Cond = X86::COND_O;
24392 Cond = X86::COND_B;
24396 Cond = X86::COND_O;
24400 Cond = X86::COND_O;
24420 X86::CondCode Cond;
24429 /// Return true if opcode is a X86 logical comparison.
24464 if (X86CC == X86::COND_E && CmpVal.getOpcode() == ISD::AND &&
24558 if ((X86CC == X86::COND_E || X86CC == X86::COND_NE) &&
24571 if (isAllOnesConstant(LHS) == (X86CC == X86::COND_NE)) {
24579 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
24704 ((CondCode == X86::COND_NE && MatchFFSMinus1(Op1, Op2)) ||
24705 (CondCode == X86::COND_E && MatchFFSMinus1(Op2, Op1)))) {
24712 ((CondCode == X86::COND_S) || // smin(x, 0)
24713 (CondCode == X86::COND_G && hasAndNot(Op1)))) { // smax(x, 0)
24723 if (CondCode == X86::COND_G)
24738 if (SDValue R = LowerSELECTWithCmpZero(Cond, Op2, Op1, X86::COND_E, DL, DAG,
24764 X86::CondCode X86Cond;
24779 X86::CondCode X86CondCode;
24789 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
24790 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
24800 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
24805 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
24806 if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
24812 // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
24834 (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) &&
24835 !X86::mayFoldLoad(Op2, Subtarget))) {
25319 X86::CondCode X86Cond;
25323 X86Cond = X86::GetOppositeBranchCondition(X86Cond);
25358 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25361 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
25371 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
25374 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
25378 X86::CondCode X86Cond =
25389 X86::CondCode X86Cond;
25627 // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows,
25826 if (X86::isZeroNode(Mask))
25970 return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION;
25977 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25979 RC ^= X86::STATIC_ROUNDING::NO_EXC;
25982 return RC == 0 || RC == X86::STATIC_ROUNDING::CUR_DIRECTION;
25991 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25993 RC ^= X86::STATIC_ROUNDING::NO_EXC;
25994 return RC == X86::STATIC_ROUNDING::TO_NEAREST_INT ||
25995 RC == X86::STATIC_ROUNDING::TO_NEG_INF ||
25996 RC == X86::STATIC_ROUNDING::TO_POS_INF ||
25997 RC == X86::STATIC_ROUNDING::TO_ZERO;
26454 SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
26458 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
26463 SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
26467 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
26473 SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
26478 SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
26609 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
26725 X86::CondCode X86CC;
26734 X86CC = X86::COND_B;
26741 X86CC = X86::COND_E;
26752 X86CC = X86::COND_E;
26763 X86CC = X86::COND_B;
26774 X86CC = X86::COND_A;
26796 X86::CondCode X86CC;
26801 X86CC = X86::COND_A;
26805 X86CC = X86::COND_A;
26809 X86CC = X86::COND_B;
26813 X86CC = X86::COND_B;
26817 X86CC = X86::COND_O;
26821 X86CC = X86::COND_O;
26825 X86CC = X86::COND_S;
26829 X86CC = X86::COND_S;
26833 X86CC = X86::COND_E;
26837 X86CC = X86::COND_E;
26941 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26943 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
27179 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
27180 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
27183 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
27184 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
27220 if (Opcode != X86::RDTSCP)
27226 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
27235 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
27299 bool X86::isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
27319 if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF)) {
27323 SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
27325 SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
27411 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27432 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
27467 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
27508 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
27522 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27542 Opc = X86::PT2RPNTLVWZ0V;
27545 Opc = X86::PT2RPNTLVWZ0T1V;
27548 Opc = X86::PT2RPNTLVWZ1V;
27551 Opc = X86::PT2RPNTLVWZ1T1V;
27554 Opc = X86::PT2RPNTLVWZ0RSV;
27557 Opc = X86::PT2RPNTLVWZ0RST1V;
27560 Opc = X86::PT2RPNTLVWZ1RSV;
27563 Opc = X86::PT2RPNTLVWZ1RST1V;
27581 SDValue Res0 = DAG.getTargetExtractSubreg(X86::sub_t0, DL, MVT::x86amx,
27583 SDValue Res1 = DAG.getTargetExtractSubreg(X86::sub_t1, DL, MVT::x86amx,
27603 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27623 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27691 X86::CondCode CC = (X86::CondCode)Op.getConstantOperandVal(4);
27737 DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
27809 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
27818 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
27934 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
27935 (FrameReg == X86::EBP && VT == MVT::i32)) &&
27951 .Case("esp", X86::ESP)
27952 .Case("rsp", X86::RSP)
27953 .Case("ebp", X86::EBP)
27954 .Case("rbp", X86::RBP)
27955 .Case("r14", X86::R14)
27956 .Case("r15", X86::R15)
27959 if (Reg == X86::EBP || Reg == X86::RBP) {
27967 assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
27988 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27990 return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX;
27997 return X86::NoRegister;
27998 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
28014 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
28015 (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
28018 Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
28083 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
28085 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
28086 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
28141 NestReg = X86::ECX;
28173 NestReg = X86::EAX;
28185 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
28310 case RoundingMode::NearestTiesToEven: FieldVal = X86::rmToNearest; break;
28311 case RoundingMode::TowardNegative: FieldVal = X86::rmDownward; break;
28312 case RoundingMode::TowardPositive: FieldVal = X86::rmUpward; break;
28313 case RoundingMode::TowardZero: FieldVal = X86::rmTowardZero; break;
28315 llvm_unreachable("rounding mode is not supported by X86 hardware");
28328 // (0xc9 << 4) & 0xc00 = X86::rmTowardZero
28329 // (0xc9 << 6) & 0xc00 = X86::rmToNearest
28715 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28755 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28853 // Since X86 does not have CMOV for 8-bit integer, we don't convert
28858 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
29097 X86::CondCode CC = IsSigned ? X86::COND_L : X86::COND_B;
29873 if (!X86::isConstantSplat(Amt, APIntShiftAmt))
30282 bool IsConstantSplat = X86::isConstantSplat(Amt, APIntShiftAmt);
30793 bool IsCstSplat = X86::isConstantSplat(Amt, APIntShiftAmt);
30995 bool IsCstSplat = X86::isConstantSplat(Amt, CstSplatValue);
31650 X86::CondCode CC = X86::COND_INVALID;
31656 CC = X86::COND_E;
31659 CC = X86::COND_NE;
31662 CC = X86::COND_S;
31665 CC = X86::COND_NS;
31854 DAG.getRegister(X86::RSP, MVT::i64), // Base
31861 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31868 DAG.getRegister(X86::ESP, MVT::i32), // Base
31876 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31912 case MVT::i8: Reg = X86::AL; size = 1; break;
31913 case MVT::i16: Reg = X86::AX; size = 2; break;
31914 case MVT::i32: Reg = X86::EAX; size = 4; break;
31917 Reg = X86::RAX; size = 8;
31934 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
31936 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
32429 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32467 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
32540 // On X86, the only ordering which actually requires an instruction is
32678 SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
33048 return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
33174 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
33192 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
34221 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
34224 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
34227 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
34231 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
34235 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
34241 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
34254 Regs64bit ? X86::RAX : X86::EAX, cpInL, SDValue());
34256 DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX,
34262 DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
34280 swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL,
34289 Regs64bit ? X86::RAX : X86::EAX,
34292 Regs64bit ? X86::RDX : X86::EDX,
34296 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
34298 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
35015 // X86 supports extremely general addressing modes.
35018 // X86 allows a sign-extended 32-bit immediate field as a displacement.
35019 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
35170 // X86 has 8, 16, and 32-bit zero-extending loads.
35295 // X86 Scheduler Hooks
35304 if (mi.readsRegister(X86::EFLAGS, /*TRI=*/nullptr))
35307 if (mi.definesRegister(X86::EFLAGS, /*TRI=*/nullptr))
35314 if (Succ->isLiveIn(X86::EFLAGS))
35353 mainMBB->addLiveIn(X86::EFLAGS);
35354 fallMBB->addLiveIn(X86::EFLAGS);
35355 sinkMBB->addLiveIn(X86::EFLAGS);
35373 BuildMI(thisMBB, MIMD, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
35379 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
35380 BuildMI(mainMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
35387 BuildMI(fallMBB, MIMD, TII->get(X86::XABORT_DEF));
35389 .addReg(X86::EAX);
35394 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
35405 // Emit va_arg instruction on X86-64.
35416 static_assert(X86::AddrNumOperands == 5, "VAARG assumes 5 address operands");
35531 BuildMI(thisMBB, MIMD, TII->get(X86::MOV32rm), OffsetReg)
35540 BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
35546 BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
35547 .addMBB(overflowMBB).addImm(X86::COND_AE);
35558 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
35570 BuildMI(offsetMBB, MIMD, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
35573 .addImm(X86::sub_32bit);
35576 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD64rr), OffsetDestReg)
35581 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32rr), OffsetDestReg)
35588 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
35593 BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
35603 BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
35614 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
35632 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
35639 TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri),
35653 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
35660 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr))
35672 TII->get(X86::PHI), DestReg)
35696 SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
35705 case X86::CMOV_FR16:
35706 case X86::CMOV_FR16X:
35707 case X86::CMOV_FR32:
35708 case X86::CMOV_FR32X:
35709 case X86::CMOV_FR64:
35710 case X86::CMOV_FR64X:
35711 case X86::CMOV_GR8:
35712 case X86::CMOV_GR16:
35713 case X86::CMOV_GR32:
35714 case X86::CMOV_RFP32:
35715 case X86::CMOV_RFP64:
35716 case X86::CMOV_RFP80:
35717 case X86::CMOV_VR64:
35718 case X86::CMOV_VR128:
35719 case X86::CMOV_VR128X:
35720 case X86::CMOV_VR256:
35721 case X86::CMOV_VR256X:
35722 case X86::CMOV_VR512:
35723 case X86::CMOV_VK1:
35724 case X86::CMOV_VK2:
35725 case X86::CMOV_VK4:
35726 case X86::CMOV_VK8:
35727 case X86::CMOV_VK16:
35728 case X86::CMOV_VK32:
35729 case X86::CMOV_VK64:
35750 X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm());
35751 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
35782 BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg)
35888 FirstInsertedMBB->addLiveIn(X86::EFLAGS);
35893 if (!SecondCascadedCMOV.killsRegister(X86::EFLAGS, /*TRI=*/nullptr) &&
35895 SecondInsertedMBB->addLiveIn(X86::EFLAGS);
35896 SinkMBB->addLiveIn(X86::EFLAGS);
35917 X86::CondCode FirstCC = X86::CondCode(FirstCMOV.getOperand(3).getImm());
35918 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
35920 X86::CondCode SecondCC =
35921 X86::CondCode(SecondCascadedCMOV.getOperand(3).getImm());
35922 BuildMI(FirstInsertedMBB, MIMD, TII->get(X86::JCC_1))
35932 BuildMI(*SinkMBB, SinkMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
36002 X86::CondCode CC = X86::CondCode(MI.getOperand(3).getImm());
36003 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
36049 if (!LastCMOV->killsRegister(X86::EFLAGS, /*TRI=*/nullptr) &&
36051 FalseMBB->addLiveIn(X86::EFLAGS);
36052 SinkMBB->addLiveIn(X86::EFLAGS);
36076 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
36094 return X86::SUB64ri32;
36096 return X86::SUB32ri;
36122 Register physSPReg = TFI.Uses64BitFramePtr ? X86::RSP : X86::ESP;
36125 TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
36127 TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
36132 const unsigned Opc = TFI.Uses64BitFramePtr ? X86::SUB64rr : X86::SUB32rr;
36141 TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
36145 BuildMI(testMBB, MIMD, TII->get(X86::JCC_1))
36147 .addImm(X86::COND_GE);
36164 TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
36173 BuildMI(blockMBB, MIMD, TII->get(X86::JMP_1)).addMBB(testMBB);
36206 const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
36239 IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP;
36254 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
36256 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
36259 BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
36267 BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
36273 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
36275 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
36278 .addReg(X86::RDI, RegState::Implicit)
36279 .addReg(X86::RAX, RegState::ImplicitDefine);
36281 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
36283 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
36286 .addReg(X86::EDI, RegState::Implicit)
36287 .addReg(X86::EAX, RegState::ImplicitDefine);
36289 BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
36291 BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
36292 BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
36295 .addReg(X86::EAX, RegState::ImplicitDefine);
36299 BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
36303 .addReg(IsLP64 ? X86::RAX : X86::EAX);
36304 BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
36313 BuildMI(*continueMBB, continueMBB->begin(), MIMD, TII->get(X86::PHI),
36358 BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
36385 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
36386 .addReg(X86::RIP)
36392 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL64m));
36393 addDirectMem(MIB, X86::RDI);
36394 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
36397 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
36404 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
36405 addDirectMem(MIB, X86::EAX);
36406 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
36409 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
36416 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
36417 addDirectMem(MIB, X86::EAX);
36418 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
36427 case X86::INDIRECT_THUNK_CALL32:
36428 return X86::CALLpcrel32;
36429 case X86::INDIRECT_THUNK_CALL64:
36430 return X86::CALL64pcrel32;
36431 case X86::INDIRECT_THUNK_TCRETURN32:
36432 return X86::TCRETURNdi;
36433 case X86::INDIRECT_THUNK_TCRETURN64:
36434 return X86::TCRETURNdi64;
36456 case X86::EAX:
36459 case X86::ECX:
36462 case X86::EDX:
36465 case X86::EDI:
36468 case X86::R11:
36479 case X86::EAX:
36482 case X86::ECX:
36485 case X86::EDX:
36488 case X86::EDI:
36491 case X86::R11:
36523 AvailableRegs.push_back(X86::R11);
36525 AvailableRegs.append({X86::EAX, X86::ECX, X86::EDX, X86::EDI});
36583 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
36591 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
36595 unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
36599 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36600 if (i == X86::AddrDisp)
36683 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
36687 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
36688 .addReg(X86::RIP)
36695 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
36703 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
36706 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36707 if (i == X86::AddrDisp)
36723 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
36733 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32r0), mainDstReg);
36737 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
36751 unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
36756 BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
36757 BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
36831 Register ZReg = MRI.createVirtualRegister(&X86::GR32RegClass);
36832 BuildMI(checkSspMBB, MIMD, TII->get(X86::MOV32r0), ZReg);
36836 BuildMI(checkSspMBB, MIMD, TII->get(X86::SUBREG_TO_REG), TmpZReg)
36839 .addImm(X86::sub_32bit);
36845 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
36850 unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;
36854 BuildMI(checkSspMBB, MIMD, TII->get(X86::JCC_1))
36856 .addImm(X86::COND_E);
36862 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
36866 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36868 if (i == X86::AddrDisp)
36880 unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;
36886 BuildMI(fallMBB, MIMD, TII->get(X86::JCC_1))
36888 .addImm(X86::COND_BE);
36893 unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;
36901 unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;
36911 BuildMI(fixShadowMBB, MIMD, TII->get(X86::JCC_1))
36913 .addImm(X86::COND_E);
36918 unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64ri : X86::SHL32ri;
36926 unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;
36935 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::PHI), CounterReg)
36945 unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;
36949 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::JCC_1))
36951 .addImm(X86::COND_NE);
36974 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
36978 Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
36986 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
36987 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
36998 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
37011 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
37013 if (i == X86::AddrDisp)
37025 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
37026 if (i == X86::AddrDisp)
37061 Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
37064 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
37066 Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
37069 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA64r), VR)
37070 .addReg(X86::RIP)
37076 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA32r), VR)
37150 BuildMI(TrapBB, MIMD, TII->get(X86::TRAP));
37181 unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
37186 BuildMI(DispatchBB, MIMD, TII->get(X86::NOOP))
37191 Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass);
37192 addFrameReference(BuildMI(DispatchBB, MIMD, TII->get(X86::MOV32rm), IReg), FI,
37194 BuildMI(DispatchBB, MIMD, TII->get(X86::CMP32ri))
37197 BuildMI(DispatchBB, MIMD, TII->get(X86::JCC_1))
37199 .addImm(X86::COND_AE);
37202 Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass);
37203 Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
37206 BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), BReg)
37207 .addReg(X86::RIP)
37216 .addImm(X86::sub_32bit);
37221 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64m))
37229 Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass);
37230 Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
37231 Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
37234 BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg)
37241 BuildMI(DispContBB, MIMD, TII->get(X86::MOVSX64rr32), OReg64)
37244 BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg)
37248 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
37256 BuildMI(DispContBB, MIMD, TII->get(X86::JMP32m))
37357 return X86::TMM0 + Imm;
37361 return X86::TMM0_TMM1 + Imm / 2;
37366 case X86::INDIRECT_THUNK_CALL32:
37367 case X86::INDIRECT_THUNK_CALL64:
37368 case X86::INDIRECT_THUNK_TCRETURN32:
37369 case X86::INDIRECT_THUNK_TCRETURN64:
37371 case X86::CATCHRET:
37373 case X86::SEG_ALLOCA_32:
37374 case X86::SEG_ALLOCA_64:
37376 case X86::PROBED_ALLOCA_32:
37377 case X86::PROBED_ALLOCA_64:
37379 case X86::TLSCall_32:
37380 case X86::TLSCall_64:
37382 case X86::CMOV_FR16:
37383 case X86::CMOV_FR16X:
37384 case X86::CMOV_FR32:
37385 case X86::CMOV_FR32X:
37386 case X86::CMOV_FR64:
37387 case X86::CMOV_FR64X:
37388 case X86::CMOV_GR8:
37389 case X86::CMOV_GR16:
37390 case X86::CMOV_GR32:
37391 case X86::CMOV_RFP32:
37392 case X86::CMOV_RFP64:
37393 case X86::CMOV_RFP80:
37394 case X86::CMOV_VR64:
37395 case X86::CMOV_VR128:
37396 case X86::CMOV_VR128X:
37397 case X86::CMOV_VR256:
37398 case X86::CMOV_VR256X:
37399 case X86::CMOV_VR512:
37400 case X86::CMOV_VK1:
37401 case X86::CMOV_VK2:
37402 case X86::CMOV_VK4:
37403 case X86::CMOV_VK8:
37404 case X86::CMOV_VK16:
37405 case X86::CMOV_VK32:
37406 case X86::CMOV_VK64:
37409 case X86::FP80_ADDr:
37410 case X86::FP80_ADDm32: {
37415 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
37419 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
37420 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
37425 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
37426 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
37432 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
37434 .addReg(NewCW, RegState::Kill, X86::sub_16bit);
37439 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
37444 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
37448 if (MI.getOpcode() == X86::FP80_ADDr) {
37449 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80))
37454 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80m32))
37465 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
37472 case X86::FP32_TO_INT16_IN_MEM:
37473 case X86::FP32_TO_INT32_IN_MEM:
37474 case X86::FP32_TO_INT64_IN_MEM:
37475 case X86::FP64_TO_INT16_IN_MEM:
37476 case X86::FP64_TO_INT32_IN_MEM:
37477 case X86::FP64_TO_INT64_IN_MEM:
37478 case X86::FP80_TO_INT16_IN_MEM:
37479 case X86::FP80_TO_INT32_IN_MEM:
37480 case X86::FP80_TO_INT64_IN_MEM: {
37485 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
37489 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
37490 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
37494 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
37495 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
37500 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
37502 .addReg(NewCW, RegState::Kill, X86::sub_16bit);
37507 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
37513 TII->get(X86::FLDCW16m)), NewCWFrameIdx);
37515 // Get the X86 opcode to use.
37520 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
37521 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
37522 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
37523 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
37524 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
37525 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
37526 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
37527 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
37528 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
37534 .addReg(MI.getOperand(X86::AddrNumOperands).getReg());
37537 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
37545 case X86::XBEGIN:
37548 case X86::VAARG_64:
37549 case X86::VAARG_X32:
37552 case X86::EH_SjLj_SetJmp32:
37553 case X86::EH_SjLj_SetJmp64:
37556 case X86::EH_SjLj_LongJmp32:
37557 case X86::EH_SjLj_LongJmp64:
37560 case X86::Int_eh_sjlj_setup_dispatch:
37576 case X86::LCMPXCHG8B: {
37596 assert(TRI->getBaseRegister() == X86::ESI &&
37597 "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
37608 if (AM.IndexReg == X86::NoRegister)
37616 (RMBBI->definesRegister(X86::EAX, /*TRI=*/nullptr) ||
37617 RMBBI->definesRegister(X86::EBX, /*TRI=*/nullptr) ||
37618 RMBBI->definesRegister(X86::ECX, /*TRI=*/nullptr) ||
37619 RMBBI->definesRegister(X86::EDX, /*TRI=*/nullptr))) {
37624 BuildMI(*BB, *MBBI, MIMD, TII->get(X86::LEA32r), computedAddrVReg), AM);
37630 case X86::LCMPXCHG16B_NO_RBX: {
37634 (BasePtr == X86::RBX || BasePtr == X86::EBX)) {
37639 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
37641 .addReg(X86::RBX);
37642 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
37644 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst);
37645 for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
37647 MIB.add(MI.getOperand(X86::AddrNumOperands));
37651 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::RBX)
37652 .add(MI.getOperand(X86::AddrNumOperands));
37654 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B));
37655 for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
37661 case X86::MWAITX: {
37664 bool IsRBX = (BasePtr == X86::RBX || BasePtr == X86::EBX);
37668 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
37670 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
37672 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EBX)
37674 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITXrrr));
37681 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
37683 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
37688 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
37690 .addReg(X86::RBX);
37692 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
37693 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITX_SAVE_RBX))
37710 BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP)
37711 .addReg(X86::ESP)
37725 addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r),
37727 X86::ESP, false, ArgOffset);
37731 case X86::PTDPBSSD:
37732 case X86::PTDPBSUD:
37733 case X86::PTDPBUSD:
37734 case X86::PTDPBUUD:
37735 case X86::PTDPBF16PS:
37736 case X86::PTDPFP16PS:
37737 case X86::PTCMMIMFP16PS:
37738 case X86::PTCMMRLFP16PS:
37739 case X86::PTDPBF8PS:
37740 case X86::PTDPBHF8PS:
37741 case X86::PTDPHBF8PS:
37742 case X86::PTDPHF8PS:
37743 case X86::PTTDPBF16PS:
37744 case X86::PTTDPFP16PS:
37745 case X86::PTTCMMIMFP16PS:
37746 case X86::PTTCMMRLFP16PS:
37747 case X86::PTCONJTCMMIMFP16PS:
37748 case X86::PTMMULTF32PS:
37749 case X86::PTTMMULTF32PS: {
37753 case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
37754 case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
37755 case X86::PTDPBUSD: Opc = X86::TDPBUSD; break;
37756 case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
37757 case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
37758 case X86::PTDPFP16PS: Opc = X86::TDPFP16PS; break;
37759 case X86::PTCMMIMFP16PS:
37760 Opc = X86::TCMMIMFP16PS;
37762 case X86::PTCMMRLFP16PS:
37763 Opc = X86::TCMMRLFP16PS;
37765 case X86::PTDPBF8PS: Opc = X86::TDPBF8PS; break;
37766 case X86::PTDPBHF8PS: Opc = X86::TDPBHF8PS; break;
37767 case X86::PTDPHBF8PS: Opc = X86::TDPHBF8PS; break;
37768 case X86::PTDPHF8PS: Opc = X86::TDPHF8PS; break;
37769 case X86::PTTDPBF16PS:
37770 Opc = X86::TTDPBF16PS;
37772 case X86::PTTDPFP16PS:
37773 Opc = X86::TTDPFP16PS;
37775 case X86::PTTCMMIMFP16PS:
37776 Opc = X86::TTCMMIMFP16PS;
37778 case X86::PTTCMMRLFP16PS:
37779 Opc = X86::TTCMMRLFP16PS;
37781 case X86::PTCONJTCMMIMFP16PS:
37782 Opc = X86::TCONJTCMMIMFP16PS;
37784 case X86::PTMMULTF32PS:
37785 Opc = X86::TMMULTF32PS;
37787 case X86::PTTMMULTF32PS:
37788 Opc = X86::TTMMULTF32PS;
37801 case X86::PTILEZERO: {
37803 BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
37809 case X86::PTILEZEROV: {
37814 case X86::PTILELOADDRS:
37815 case X86::PTILELOADDRST1:
37816 case X86::PTILELOADD:
37817 case X86::PTILELOADDT1:
37818 case X86::PTILESTORED: {
37823 case X86::PTILELOADD:
37824 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
37826 case X86::PTILELOADDT1:
37827 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
37829 case X86::PTILESTORED:
37830 Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
37832 case X86::PTILELOADDRS:
37833 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRS);
37835 case X86::PTILELOADDRST1:
37836 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRST1);
37843 if (Opc != X86::TILESTORED && Opc != X86::TILESTORED_EVEX)
37853 if (Opc == X86::TILESTORED || Opc == X86::TILESTORED_EVEX)
37860 case X86::PT2RPNTLVWZ0:
37861 case X86::PT2RPNTLVWZ0T1:
37862 case X86::PT2RPNTLVWZ1:
37863 case X86::PT2RPNTLVWZ1T1:
37864 case X86::PT2RPNTLVWZ0RS:
37865 case X86::PT2RPNTLVWZ0RST1:
37866 case X86::PT2RPNTLVWZ1RS:
37867 case X86::PT2RPNTLVWZ1RST1: {
37874 case X86::PT2RPNTLVWZ0:
37875 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ0);
37877 case X86::PT2RPNTLVWZ0T1:
37878 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ0T1);
37880 case X86::PT2RPNTLVWZ1:
37881 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ1);
37883 case X86::PT2RPNTLVWZ1T1:
37884 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ1T1);
37886 case X86::PT2RPNTLVWZ0RS:
37887 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ0RS);
37889 case X86::PT2RPNTLVWZ0RST1:
37890 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ0RST1);
37892 case X86::PT2RPNTLVWZ1RS:
37893 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ1RS);
37895 case X86::PT2RPNTLVWZ1RST1:
37896 Opc = GET_EGPR_IF_ENABLED(X86::T2RPNTLVWZ1RST1);
37911 case X86::PTTRANSPOSED:
37912 case X86::PTCONJTFP16: {
37914 unsigned Opc = MI.getOpcode() == X86::PTTRANSPOSED ? X86::TTRANSPOSED
37915 : X86::TCONJTFP16;
37924 case X86::PTCVTROWPS2BF16Hrri:
37925 case X86::PTCVTROWPS2BF16Lrri:
37926 case X86::PTCVTROWPS2PHHrri:
37927 case X86::PTCVTROWPS2PHLrri:
37928 case X86::PTCVTROWD2PSrri:
37929 case X86::PTILEMOVROWrri: {
37935 case X86::PTCVTROWD2PSrri:
37936 Opc = X86::TCVTROWD2PSrri;
37938 case X86::PTCVTROWPS2BF16Hrri:
37939 Opc = X86::TCVTROWPS2BF16Hrri;
37941 case X86::PTCVTROWPS2PHHrri:
37942 Opc = X86::TCVTROWPS2PHHrri;
37944 case X86::PTCVTROWPS2BF16Lrri:
37945 Opc = X86::TCVTROWPS2BF16Lrri;
37947 case X86::PTCVTROWPS2PHLrri:
37948 Opc = X86::TCVTROWPS2PHLrri;
37950 case X86::PTILEMOVROWrri:
37951 Opc = X86::TILEMOVROWrri;
37962 case X86::PTCVTROWPS2BF16Hrre:
37963 case X86::PTCVTROWPS2BF16Lrre:
37964 case X86::PTCVTROWPS2PHHrre:
37965 case X86::PTCVTROWPS2PHLrre:
37966 case X86::PTCVTROWD2PSrre:
37967 case X86::PTILEMOVROWrre: {
37973 case X86::PTCVTROWD2PSrre:
37974 Opc = X86::TCVTROWD2PSrre;
37976 case X86::PTCVTROWPS2BF16Hrre:
37977 Opc = X86::TCVTROWPS2BF16Hrre;
37979 case X86::PTCVTROWPS2BF16Lrre:
37980 Opc = X86::TCVTROWPS2BF16Lrre;
37982 case X86::PTCVTROWPS2PHHrre:
37983 Opc = X86::TCVTROWPS2PHHrre;
37985 case X86::PTCVTROWPS2PHLrre:
37986 Opc = X86::TCVTROWPS2PHLrre;
37988 case X86::PTILEMOVROWrre:
37989 Opc = X86::TILEMOVROWrre;
38004 // X86 Optimization Hooks
39807 X86::mayFoldLoad(V1.getOperand(0), Subtarget)) {
40708 namespace X86 {
40712 } // namespace X86
41184 {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
41380 if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
41381 X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
41818 X86::MaxShuffleCombineDepth,
42772 // X86 targets with 512-bit ADDSUB instructions!
42778 // X86 targets with FP16 ADDSUB instructions!
43767 assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range");
43775 {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
45582 X86::CondCode X86CC;
47623 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
47681 if (CC == X86::COND_A && !Comparison.isMaxValue()) {
47683 CC = X86::COND_AE;
47684 } else if (CC == X86::COND_LE && !Comparison.isMaxSignedValue()) {
47686 CC = X86::COND_L;
47691 if (CC == X86::COND_AE && !Comparison.isMinValue()) {
47693 CC = X86::COND_A;
47694 } else if (CC == X86::COND_L && !Comparison.isMinSignedValue()) {
47696 CC = X86::COND_LE;
47723 if (CC == X86::COND_S && Addend == 1)
47724 CC = X86::COND_LE;
47725 else if (CC == X86::COND_NS && Addend == 1)
47726 CC = X86::COND_G;
47727 else if (CC == X86::COND_G && Addend == -1)
47728 CC = X86::COND_GE;
47729 else if (CC == X86::COND_LE && Addend == -1)
47730 CC = X86::COND_L;
47742 static SDValue checkSignTestSetCCCombine(SDValue Cmp, X86::CondCode &CC,
47744 if (CC != X86::COND_S && CC != X86::COND_NS)
47791 CC = CC == X86::COND_S ? X86::COND_NE : X86::COND_E;
47811 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
47818 if (CC != X86::COND_E && CC != X86::COND_NE)
47828 bool needOppositeCond = (CC == X86::COND_E);
47872 assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
47877 CC = X86::CondCode(SetCC.getConstantOperandVal(0));
47879 CC = X86::GetOppositeBranchCondition(CC);
47915 CC = X86::CondCode(SetCC.getConstantOperandVal(2));
47917 CC = X86::GetOppositeBranchCondition(CC);
47929 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
47930 X86::CondCode &CC1, SDValue &Flags,
47961 CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
47962 CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
47987 if (CarryCC == X86::COND_B)
47989 if (CarryCC == X86::COND_A) {
48008 if (CarryCC == X86::COND_E &&
48029 static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
48049 X86::CondCode InvCC;
48051 case X86::COND_B:
48053 InvCC = X86::COND_E;
48055 case X86::COND_AE:
48057 InvCC = X86::COND_NE;
48059 case X86::COND_E:
48061 InvCC = X86::COND_B;
48063 case X86::COND_NE:
48065 InvCC = X86::COND_AE;
48067 case X86::COND_A:
48068 case X86::COND_BE:
48073 InvCC = X86::COND_INVALID;
48077 if (InvCC != X86::COND_INVALID) {
48084 if (CC == X86::COND_B || CC == X86::COND_AE) {
48102 CC = (CC == X86::COND_B ? X86::COND_E : X86::COND_NE);
48109 if (CC == X86::COND_E || CC == X86::COND_NE) {
48112 CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
48130 CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
48183 if (CC == X86::COND_E && OpVT.is128BitVector() && Subtarget.hasAVX()) {
48205 static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
48210 if (!(CC == X86::COND_E || CC == X86::COND_NE))
48428 CC = IsAnyOf ? CC : (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
48440 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
48443 if (CC == X86::COND_B)
48462 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
48470 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
48478 // We can't always do this as FCMOV only supports a subset of X86 cond.
48498 CC = X86::GetOppositeBranchCondition(CC);
48597 if (CC == X86::COND_NE &&
48599 CC = X86::GetOppositeBranchCondition(CC);
48603 if (CC == X86::COND_E && CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
48618 if (CC == X86::COND_AE && isOneConstant(FalseOp) &&
48653 if (CC == X86::COND_NE) {
48655 X86::CondCode CC0, CC1;
48660 CC0 = X86::GetOppositeBranchCondition(CC0);
48661 CC1 = X86::GetOppositeBranchCondition(CC1);
48678 if ((CC == X86::COND_NE || CC == X86::COND_E) &&
48683 if (CC == X86::COND_E)
48701 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
49259 // TODO: This is X86 specific because we want to be able to handle wide types
49409 // sexts in X86 are MOVs. The MOVs have the same code size
50139 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
50140 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
50142 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
50143 X86::CondCode tmp = cc0;
50148 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
50149 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
50152 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
50626 if (!X86::isConstantSplat(Op1, SplatVal, false) || !SplatVal.isMask())
50695 // This function recognizes cases where X86 bzhi instruction can replace and
50952 if (static_cast<X86::CondCode>(BrCond->getConstantOperandVal(CondNo)) !=
50953 X86::COND_NE)
50963 X86::CondCode CC =
50964 static_cast<X86::CondCode>(CCN->getAsAPIntVal().getSExtValue());
50965 X86::CondCode OppositeCC = X86::GetOppositeBranchCondition(CC);
51028 X86::CondCode CC0 =
51029 static_cast<X86::CondCode>(SetCC0.getConstantOperandVal(0));
51031 if (CC0 == X86::COND_P || CC0 == X86::COND_NP)
51040 IsOR ? DAG.getTargetConstant(X86::GetOppositeBranchCondition(CC0),
51044 X86::CondCode CC1 =
51045 static_cast<X86::CondCode>(CC1N->getAsAPIntVal().getSExtValue());
51046 X86::CondCode OppositeCC1 = X86::GetOppositeBranchCondition(CC1);
51047 X86::CondCode CFlagsCC = IsOR ? CC1 : OppositeCC1;
51050 X86::getCCMPCondFlagsFromCondCode(CFlagsCC), DL, MVT::i8);
51212 X86::CondCode X86CC = X86::COND_B;
51216 X86CC = X86::COND_AE;
51226 X86CC = X86CC == X86::COND_AE ? X86::COND_B : X86::COND_AE;
51321 X86::MaxShuffleCombineDepth,
51499 // The result of the shift is true or false, and on X86, the 32-bit
51534 X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E &&
51657 X86::CondCode CC;
51660 CC = (X86::CondCode)Y.getConstantOperandVal(0);
51674 if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnes()) ||
51675 (IsSub && CC == X86::COND_B && ConstantX->isZero())) {
51680 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51684 if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnes()) ||
51685 (IsSub && CC == X86::COND_A && ConstantX->isZero())) {
51697 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51703 if (CC == X86::COND_B) {
51714 if (CC == X86::COND_A) {
51734 if (CC == X86::COND_AE) {
51742 if (CC == X86::COND_BE) {
51764 if (CC != X86::COND_E && CC != X86::COND_NE)
51768 !X86::isZeroNode(EFLAGS.getOperand(1)) ||
51782 if ((IsSub && CC == X86::COND_NE && ConstantX->isZero()) ||
51783 (!IsSub && CC == X86::COND_E && ConstantX->isAllOnes())) {
51788 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51796 if ((IsSub && CC == X86::COND_E && ConstantX->isZero()) ||
51797 (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnes())) {
51802 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
51817 if (CC == X86::COND_NE)
51966 X86::CondCode CCode = (X86::CondCode)Cond.getConstantOperandVal(0);
51967 CCode = X86::GetOppositeBranchCondition(CCode);
53054 (X86::CondCode)Cmov.getConstantOperandVal(2)),
53620 // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its
53646 // TODO: This is X86 specific because we want to be able to handle wide types
54227 X86::CondCode NewCC = X86::GetOppositeBranchCondition(
54228 X86::CondCode(LHS->getConstantOperandVal(0)));
55502 X86::CondCode X86CC;
56063 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
56078 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
56399 X86::CondCode CC;
56406 CC = (X86::CondCode)User->getConstantOperandVal(0);
56410 CC = (X86::CondCode)User->getConstantOperandVal(2);
56417 case X86::COND_A: case X86::COND_AE:
56418 case X86::COND_B: case X86::COND_BE:
56419 case X86::COND_O: case X86::COND_NO:
56420 case X86::COND_G: case X86::COND_GE:
56421 case X86::COND_L: case X86::COND_LE:
56449 X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
56450 if (CC != X86::COND_E && CC != X86::COND_NE)
56591 // Use a X86 specific opcode to avoid DAG combine messing with it.
56700 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),
57030 if (X86::mayFoldLoad(OtherOp, Subtarget))
57143 X86::isZeroNode(Op0.getOperand(1))) {
57168 X86::CondCode CC = (X86::CondCode)N1.getConstantOperandVal(2);
57171 if ((CC == X86::COND_S || CC == X86::COND_NS) &&
57191 if ((CC == X86::COND_L || CC == X86::COND_B) && isNullConstant(N0) &&
57219 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
57220 X86::CondCode NewCC = X86::GetOppositeBranchCondition(CC);
57237 if (N->getConstantOperandVal(3) != X86::COND_NE)
57246 if (!X86::isZeroNode(Sub.getOperand(0)) || SetCC.getOpcode() != X86ISD::SETCC)
57271 // X86 can't encode an immediate LHS of a sub. See if we can push the
57295 X86::isZeroNode(Op1.getOperand(1))) {
57304 !(X86::isZeroNode(Op0) && X86::isZeroNode(Op1.getOperand(1)))) {
57420 X86::mayFoldLoadIntoBroadcastFromMem(Op0.getOperand(0),
57431 X86::mayFoldLoad(Op0.getOperand(0), Subtarget))) &&
57452 !X86::mayFoldLoad(Op0.getOperand(0), Subtarget))
59697 if (X86::mayFoldLoad(N0, Subtarget) && IsFoldableRMW(N0, Op))
59719 if (X86::mayFoldLoad(N1, Subtarget) &&
59723 if (X86::mayFoldLoad(N0, Subtarget) &&
59738 // X86 Inline Assembly Support
59852 static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
59853 X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
59854 .Case("{@cca}", X86::COND_A)
59855 .Case("{@ccae}", X86::COND_AE)
59856 .Case("{@ccb}", X86::COND_B)
59857 .Case("{@ccbe}", X86::COND_BE)
59858 .Case("{@ccc}", X86::COND_B)
59859 .Case("{@cce}", X86::COND_E)
59860 .Case("{@ccz}", X86::COND_E)
59861 .Case("{@ccg}", X86::COND_G)
59862 .Case("{@ccge}", X86::COND_GE)
59863 .Case("{@ccl}", X86::COND_L)
59864 .Case("{@ccle}", X86::COND_LE)
59865 .Case("{@ccna}", X86::COND_BE)
59866 .Case("{@ccnae}", X86::COND_B)
59867 .Case("{@ccnb}", X86::COND_AE)
59868 .Case("{@ccnbe}", X86::COND_A)
59869 .Case("{@ccnc}", X86::COND_AE)
59870 .Case("{@ccne}", X86::COND_NE)
59871 .Case("{@ccnz}", X86::COND_NE)
59872 .Case("{@ccng}", X86::COND_LE)
59873 .Case("{@ccnge}", X86::COND_L)
59874 .Case("{@ccnl}", X86::COND_GE)
59875 .Case("{@ccnle}", X86::COND_G)
59876 .Case("{@ccno}", X86::COND_NO)
59877 .Case("{@ccnp}", X86::COND_NP)
59878 .Case("{@ccns}", X86::COND_NS)
59879 .Case("{@cco}", X86::COND_O)
59880 .Case("{@ccp}", X86::COND_P)
59881 .Case("{@ccs}", X86::COND_S)
59882 .Default(X86::COND_INVALID);
59958 } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
60132 X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
60133 if (Cond == X86::COND_INVALID)
60142 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue);
60145 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
60318 return RC.hasSuperClassEq(&X86::GR8RegClass) ||
60319 RC.hasSuperClassEq(&X86::GR16RegClass) ||
60320 RC.hasSuperClassEq(&X86::GR32RegClass) ||
60321 RC.hasSuperClassEq(&X86::GR64RegClass) ||
60322 RC.hasSuperClassEq(&X86::LOW32_ADDR_ACCESS_RBPRegClass);
60328 return RC.hasSuperClassEq(&X86::FR16XRegClass) ||
60329 RC.hasSuperClassEq(&X86::FR32XRegClass) ||
60330 RC.hasSuperClassEq(&X86::FR64XRegClass) ||
60331 RC.hasSuperClassEq(&X86::VR128XRegClass) ||
60332 RC.hasSuperClassEq(&X86::VR256XRegClass) ||
60333 RC.hasSuperClassEq(&X86::VR512RegClass);
60339 return RC.hasSuperClassEq(&X86::VK1RegClass) ||
60340 RC.hasSuperClassEq(&X86::VK2RegClass) ||
60341 RC.hasSuperClassEq(&X86::VK4RegClass) ||
60342 RC.hasSuperClassEq(&X86::VK8RegClass) ||
60343 RC.hasSuperClassEq(&X86::VK16RegClass) ||
60344 RC.hasSuperClassEq(&X86::VK32RegClass) ||
60345 RC.hasSuperClassEq(&X86::VK64RegClass);
60365 return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
60368 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
60376 return std::make_pair(0U, &X86::VK1RegClass);
60378 return std::make_pair(0U, &X86::VK8RegClass);
60380 return std::make_pair(0U, &X86::VK16RegClass);
60384 return std::make_pair(0U, &X86::VK32RegClass);
60386 return std::make_pair(0U, &X86::VK64RegClass);
60393 ? &X86::GR8RegClass
60394 : &X86::GR8_NOREX2RegClass);
60397 ? &X86::GR16RegClass
60398 : &X86::GR16_NOREX2RegClass);
60401 ? &X86::GR32RegClass
60402 : &X86::GR32_NOREX2RegClass);
60405 ? &X86::GR64RegClass
60406 : &X86::GR64_NOREX2RegClass);
60413 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
60415 return std::make_pair(0U, &X86::GR16_ABCDRegClass);
60418 return std::make_pair(0U, &X86::GR32_ABCDRegClass);
60420 return std::make_pair(0U, &X86::GR64_ABCDRegClass);
60426 ? &X86::GR8RegClass
60427 : &X86::GR8_NOREX2RegClass);
60430 ? &X86::GR16RegClass
60431 : &X86::GR16_NOREX2RegClass);
60435 ? &X86::GR32RegClass
60436 : &X86::GR32_NOREX2RegClass);
60439 ? &X86::GR64RegClass
60440 : &X86::GR64_NOREX2RegClass);
60444 return std::make_pair(0U, &X86::GR8_NOREXRegClass);
60446 return std::make_pair(0U, &X86::GR16_NOREXRegClass);
60449 return std::make_pair(0U, &X86::GR32_NOREXRegClass);
60451 return std::make_pair(0U, &X86::GR64_NOREXRegClass);
60457 return std::make_pair(0U, &X86::RFP32RegClass);
60459 return std::make_pair(0U, &X86::RFP64RegClass);
60461 return std::make_pair(0U, &X86::RFP80RegClass);
60465 return std::make_pair(0U, &X86::VR64RegClass);
60476 return std::make_pair(0U, &X86::FR16XRegClass);
60481 return std::make_pair(0U, &X86::FR32XRegClass);
60482 return std::make_pair(0U, &X86::FR32RegClass);
60486 return std::make_pair(0U, &X86::FR64XRegClass);
60487 return std::make_pair(0U, &X86::FR64RegClass);
60491 return std::make_pair(0U, &X86::VR128XRegClass);
60492 return std::make_pair(0U, &X86::VR128RegClass);
60500 return std::make_pair(0U, &X86::VR128XRegClass);
60501 return std::make_pair(0U, &X86::VR128RegClass);
60506 return std::make_pair(0U, &X86::VR128XRegClass);
60507 return std::make_pair(0U, &X86::VR128RegClass);
60516 return std::make_pair(0U, &X86::VR128XRegClass);
60517 return std::make_pair(0U, &X86::VR128RegClass);
60523 return std::make_pair(0U, &X86::VR256XRegClass);
60524 return std::make_pair(0U, &X86::VR256RegClass);
60529 return std::make_pair(0U, &X86::VR256XRegClass);
60530 return std::make_pair(0U, &X86::VR256RegClass);
60538 return std::make_pair(0U, &X86::VR256XRegClass);
60540 return std::make_pair(0U, &X86::VR256RegClass);
60546 return std::make_pair(0U, &X86::VR512RegClass);
60547 return std::make_pair(0U, &X86::VR512_0_15RegClass);
60552 return std::make_pair(0U, &X86::VR512RegClass);
60553 return std::make_pair(0U, &X86::VR512_0_15RegClass);
60562 return std::make_pair(0U, &X86::VR512RegClass);
60563 return std::make_pair(0U, &X86::VR512_0_15RegClass);
60577 return std::make_pair(0U, &X86::VR64RegClass);
60586 return std::make_pair(X86::XMM0, &X86::FR16XRegClass);
60589 return std::make_pair(X86::XMM0, &X86::FR32RegClass);
60592 return std::make_pair(X86::XMM0, &X86::FR64RegClass);
60596 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
60600 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
60608 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
60613 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
60617 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
60625 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
60630 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
60634 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
60642 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
60650 return std::make_pair(0U, &X86::VK1WMRegClass);
60652 return std::make_pair(0U, &X86::VK8WMRegClass);
60654 return std::make_pair(0U, &X86::VK16WMRegClass);
60658 return std::make_pair(0U, &X86::VK32WMRegClass);
60660 return std::make_pair(0U, &X86::VK64WMRegClass);
60670 return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
60672 return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
60674 return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
60676 return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
60680 return std::make_pair(0U, &X86::GR8RegClass);
60682 return std::make_pair(0U, &X86::GR16RegClass);
60684 return std::make_pair(0U, &X86::GR32RegClass);
60686 return std::make_pair(0U, &X86::GR64RegClass);
60691 if (parseConstraintCode(Constraint) != X86::COND_INVALID)
60692 return std::make_pair(0U, &X86::GR32RegClass);
60713 return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
60714 return std::make_pair(X86::FP0 + Constraint[4] - '0',
60715 &X86::RFP80RegClass);
60720 return std::make_pair(X86::FP0, &X86::RFP80RegClass);
60725 return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
60731 return std::make_pair(X86::DF, &X86::DFCCRRegClass);
60736 return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
60780 Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
60781 : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
60782 : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
60783 : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
60788 case X86::RAX:
60789 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
60790 case X86::RDX:
60791 return std::make_pair(X86::EDX, &X86::GR32_DCRegClass);
60792 case X86::RCX:
60793 return std::make_pair(X86::ECX, &X86::GR32_CBRegClass);
60794 case X86::RBX:
60795 return std::make_pair(X86::EBX, &X86::GR32_BSIRegClass);
60796 case X86::RSI:
60797 return std::make_pair(X86::ESI, &X86::GR32_SIDIRegClass);
60798 case X86::RDI:
60799 return std::make_pair(X86::EDI, &X86::GR32_DIBPRegClass);
60800 case X86::RBP:
60801 return std::make_pair(X86::EBP, &X86::GR32_BPSPRegClass);
60820 Res.second = &X86::FR16XRegClass;
60822 Res.second = &X86::FR32XRegClass;
60824 Res.second = &X86::FR64XRegClass;
60825 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT))
60826 Res.second = &X86::VR128XRegClass;
60827 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT))
60828 Res.second = &X86::VR256XRegClass;
60829 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))
60830 Res.second = &X86::VR512RegClass;
60838 Res.second = &X86::VK1RegClass;
60840 Res.second = &X86::VK8RegClass;
60842 Res.second = &X86::VK16RegClass;
60844 Res.second = &X86::VK32RegClass;
60846 Res.second = &X86::VK64RegClass;
60892 if (X86::GR64RegClass.contains(*I))
60893 RC = &X86::GR64RegClass;
60933 case X86::CALL64m:
60934 case X86::CALL64m_NT:
60935 case X86::TAILJMPm64:
60936 case X86::TAILJMPm64_REX: {
60939 if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true,
60959 case X86::CALL64r:
60960 case X86::CALL64r_NT:
60961 case X86::TAILJMPr64:
60962 case X86::TAILJMPr64_REX:
60967 case X86::CALL64pcrel32:
60968 case X86::TAILJMPd64:
60974 TargetReg = X86::R11;
60981 return BuildMI(MBB, MBBI, MIMetadata(*MBBI), TII->get(X86::KCFI_CHECK))