Lines Matching defs:DAG
1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
1694 SelectionDAG &DAG) const {
1701 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1708 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1715 Ops.push_back(DAG.getSignedTargetConstant(
1722 Ops.push_back(DAG.getSignedTargetConstant(
1729 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1734 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1762 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1768 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
1769 DAG.getValueType(VA.getValVT()));
1771 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
1772 DAG.getValueType(VA.getValVT()));
1775 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1781 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1782 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1791 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1795 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
1797 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
1799 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
1807 Value = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, Value);
1811 Value = DAG.getNode(ISD::BITCAST, DL, BitCastToType, Value);
1815 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
1816 DAG.getConstant(0, DL, MVT::i32));
1826 static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1829 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
1830 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, In);
1831 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
1832 DAG.getNode(ISD::SRL, DL, MVT::i128, In,
1833 DAG.getConstant(64, DL, MVT::i32)));
1835 std::tie(Lo, Hi) = DAG.SplitScalar(In, DL, MVT::i64, MVT::i64);
1840 SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
1845 static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1847 SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
1849 SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
1852 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
1853 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Lo);
1854 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Hi);
1855 Hi = DAG.getNode(ISD::SHL, DL, MVT::i128, Hi,
1856 DAG.getConstant(64, DL, MVT::i32));
1857 return DAG.getNode(ISD::OR, DL, MVT::i128, Lo, Hi);
1859 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
1864 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1869 Parts[0] = lowerI128ToGR128(DAG, DAG.getBitcast(MVT::i128, Val));
1877 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
1881 SDValue Res = lowerGR128ToI128(DAG, Parts[0]);
1882 return DAG.getBitcast(ValueVT, Res);
1891 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1892 MachineFunction &MF = DAG.getMachineFunction();
1898 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1902 SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1951 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1970 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1972 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
1973 DAG.getIntPtrConstant(4, DL));
1974 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
1981 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1990 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1991 DAG.getIntPtrConstant(PartOffset, DL));
1992 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1997 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
2043 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2046 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
2047 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
2051 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2090 static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2092 MachineFunction &MF = DAG.getMachineFunction();
2095 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2097 SDValue Reg = DAG.getRegister(ADAvReg, PtrVT);
2098 SDValue Ofs = DAG.getTargetConstant(Offset, DL, PtrVT);
2100 SDValue Result = DAG.getNode(SystemZISD::ADA_ENTRY, DL, PtrVT, Val, Reg, Ofs);
2102 Result = DAG.getLoad(
2103 PtrVT, DL, DAG.getEntryNode(), Result, MachinePointerInfo(), Align(8),
2111 static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2129 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, ADAtype);
2131 return getADAEntry(DAG, Val, DL, 0, LoadAddr);
2134 static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2138 MachineFunction &MF = DAG.getMachineFunction();
2139 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2149 ADA = DAG.getCopyFromReg(Chain, DL, ADAvReg, PtrVT);
2150 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
2151 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2154 SDValue GA = DAG.getTargetGlobalAddress(
2156 ADA = getADAEntry(DAG, GA, DL, ADADelta);
2157 Callee = getADAEntry(DAG, GA, DL, EPADelta);
2160 SDValue ES = DAG.getTargetExternalSymbol(
2162 ADA = getADAEntry(DAG, ES, DL, ADADelta);
2163 Callee = getADAEntry(DAG, ES, DL, EPADelta);
2166 ADA = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
2167 DAG.getConstant(ADADelta, DL, PtrVT));
2168 ADA = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), ADA,
2169 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2170 Callee = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
2171 DAG.getConstant(EPADelta, DL, PtrVT));
2172 Callee = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Callee,
2173 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2181 SelectionDAG &DAG = CLI.DAG;
2191 MachineFunction &MF = DAG.getMachineFunction();
2193 LLVMContext &Ctx = *DAG.getContext();
2218 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
2242 SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT);
2245 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
2253 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2254 DAG.getIntPtrConstant(PartOffset, DL));
2256 DAG.getStore(Chain, DL, PartValue, Address,
2264 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
2271 ArgValue = lowerI128ToGR128(DAG, ArgValue);
2280 StackPtr = DAG.getCopyFromReg(Chain, DL,
2286 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2287 DAG.getIntPtrConstant(Offset, DL));
2291 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2299 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, ArgValue,
2300 DAG.getIntPtrConstant(1, DL));
2308 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2317 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2321 Chain = DAG.getCopyToReg(Chain, DL, CalleeReg, Callee, Glue);
2323 Callee = DAG.getRegister(CalleeReg, Callee.getValueType());
2329 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
2330 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2332 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
2333 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2335 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
2337 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
2343 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
2356 Ops.push_back(DAG.getRegister(RegsToPass[I].first,
2363 Ops.push_back(DAG.getRegisterMask(Mask));
2370 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2372 SDValue Ret = DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
2373 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
2376 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
2377 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2381 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
2392 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
2399 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
2408 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2417 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2424 DAG.getExternalSymbol(CalleeName, getPointerTy(DAG.getDataLayout()));
2426 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2427 TargetLowering::CallLoweringInfo CLI(DAG);
2461 const SDLoc &DL, SelectionDAG &DAG) const {
2462 MachineFunction &MF = DAG.getMachineFunction();
2469 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2474 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, Chain);
2491 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
2495 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
2497 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
2505 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, RetOps);
2699 static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2710 SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other);
2711 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
2714 DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
2719 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2728 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
2762 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2776 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
2782 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2842 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
2847 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
2853 C.Op1 = DAG.getConstant((uint32_t)Value, DL, MVT::i32);
2941 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
2953 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
3009 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3023 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
3141 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3152 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(1));
3153 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(0));
3245 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
3251 static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3257 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3264 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op0);
3265 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op1);
3300 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3307 KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
3318 static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3351 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3361 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3366 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3391 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
3398 adjustForRedundantAnd(DAG, DL, C);
3399 adjustZeroCmp(DAG, DL, C);
3400 adjustSubwordCmp(DAG, DL, C);
3401 adjustForSubtraction(DAG, DL, C);
3403 adjustICmpTruncate(DAG, DL, C);
3411 adjustForTestUnderMask(DAG, DL, C);
3412 adjustICmp128(DAG, DL, C);
3417 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3422 Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode);
3425 Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode);
3432 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
3433 DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
3437 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
3438 DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
3441 SDVTList VTs = DAG.getVTList(C.Op0.getValueType(), MVT::i32);
3442 SDValue Val = DAG.getNode(C.Opcode, DL, VTs, C.Op0, C.Op1);
3446 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
3447 return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
3449 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
3455 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3458 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
3459 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
3460 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
3461 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
3462 DAG.getConstant(32, DL, MVT::i64));
3463 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
3464 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
3471 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3474 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
3476 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
3477 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
3483 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3485 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
3486 DAG.getConstant(0, DL, MVT::i32),
3487 DAG.getTargetConstant(CCValid, DL, MVT::i32),
3488 DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
3489 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
3565 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3568 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
3570 SDVTList VTs = DAG.getVTList(MVT::v2f64, MVT::Other);
3571 return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
3573 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
3578 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3588 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
3589 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
3590 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
3591 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
3593 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::Other);
3594 SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
3595 SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
3596 SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3600 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
3602 return DAG.getMergeValues(Ops, DL);
3604 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
3605 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
3606 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3609 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
3610 return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
3612 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
3619 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3640 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3642 SDValue GE = getVectorCmp(DAG, getVectorComparison(ISD::SETOGE, Mode),
3644 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
3646 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3657 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3659 SDValue GT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3661 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
3663 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3685 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3689 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
3699 DAG.getSplatBuildVector(VT, DL, DAG.getAllOnesConstant(DL, MVT::i64));
3700 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
3704 Cmp = DAG.getMergeValues(Ops, DL);
3710 SelectionDAG &DAG) const {
3717 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3719 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3720 SDValue CCReg = emitCmp(DAG, DL, C);
3721 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
3725 SelectionDAG &DAG,
3734 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3739 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
3740 SDValue CCReg = emitCmp(DAG, DL, C);
3742 SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
3744 return DAG.getMergeValues(Ops, DL);
3747 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3754 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3755 SDValue CCReg = emitCmp(DAG, DL, C);
3756 return DAG.getNode(
3758 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
3759 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
3773 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
3775 Op = DAG.getNode(ISD::ABS, DL, Op.getValueType(), Op);
3777 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
3778 DAG.getConstant(0, DL, Op.getValueType()), Op);
3782 static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
3812 SDValue Mask = DAG.getNode(Op, DL, VT, C.Op0, C.Op1);
3813 TrueOp = DAG.getNode(ISD::AND, DL, VT, TrueOp, Mask);
3814 FalseOp = DAG.getNode(ISD::AND, DL, VT, FalseOp, DAG.getNOT(DL, Mask, VT));
3815 return DAG.getNode(ISD::OR, DL, VT, TrueOp, FalseOp);
3819 SelectionDAG &DAG) const {
3827 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3838 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
3840 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
3847 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
3850 SDValue CCReg = emitCmp(DAG, DL, C);
3852 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
3853 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
3855 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
3859 SelectionDAG &DAG) const {
3863 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3864 CodeModel::Model CM = DAG.getTarget().getCodeModel();
3871 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
3872 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3879 DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
3880 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
3886 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT);
3887 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3890 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
3891 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3892 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
3893 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3895 Result = getADAEntry(DAG, GV, DL, PtrVT);
3902 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
3903 DAG.getSignedConstant(Offset, DL, PtrVT));
3909 SelectionDAG &DAG,
3913 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3914 SDValue Chain = DAG.getEntryNode();
3917 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3922 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
3923 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
3925 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
3931 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
3937 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
3938 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
3943 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
3945 Ops.push_back(DAG.getRegisterMask(Mask));
3951 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3952 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
3956 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
3960 SelectionDAG &DAG) const {
3961 SDValue Chain = DAG.getEntryNode();
3962 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3965 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
3966 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
3969 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
3970 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
3973 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
3974 DAG.getConstant(32, DL, PtrVT));
3975 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
3979 SelectionDAG &DAG) const {
3980 if (DAG.getTarget().useEmulatedTLS())
3981 return LowerToTLSEmulatedModel(Node, DAG);
3984 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3985 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
3987 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3991 SDValue TP = lowerThreadPointer(DL, DAG);
4001 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4002 Offset = DAG.getLoad(
4003 PtrVT, DL, DAG.getEntryNode(), Offset,
4004 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4007 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
4016 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4017 Offset = DAG.getLoad(
4018 PtrVT, DL, DAG.getEntryNode(), Offset,
4019 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4022 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
4028 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4034 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4035 DTPOffset = DAG.getLoad(
4036 PtrVT, DL, DAG.getEntryNode(), DTPOffset,
4037 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4039 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
4045 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
4047 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
4049 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
4050 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4059 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4060 Offset = DAG.getLoad(
4061 PtrVT, DL, DAG.getEntryNode(), Offset,
4062 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4068 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
4072 SelectionDAG &DAG) const {
4076 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4078 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
4079 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4084 SelectionDAG &DAG) const {
4086 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4087 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
4090 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4094 SelectionDAG &DAG) const {
4096 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4101 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());
4103 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign(),
4107 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4111 SelectionDAG &DAG) const {
4113 MachineFunction &MF = DAG.getMachineFunction();
4119 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4126 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
4133 SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT);
4135 BackChain = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), BackChain,
4137 BackChain = DAG.getNode(ISD::ADD, DL, PtrVT, BackChain, Offset);
4145 SelectionDAG &DAG) const {
4146 MachineFunction &MF = DAG.getMachineFunction();
4150 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
4155 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4162 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4165 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, FrameAddr,
4166 DAG.getSignedConstant(Offset, DL, PtrVT));
4167 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr,
4176 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
4180 SelectionDAG &DAG) const {
4191 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
4194 DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
4201 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
4203 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
4206 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
4207 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
4208 DAG.getConstant(32, DL, MVT::i64));
4210 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
4211 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
4215 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
4216 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
4218 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
4220 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
4222 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
4223 DAG.getConstant(32, DL, MVT::i64));
4224 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
4230 SelectionDAG &DAG) const {
4233 return lowerVASTART_XPLINK(Op, DAG);
4235 return lowerVASTART_ELF(Op, DAG);
4239 SelectionDAG &DAG) const {
4240 MachineFunction &MF = DAG.getMachineFunction();
4248 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4249 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4251 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
4256 SelectionDAG &DAG) const {
4257 MachineFunction &MF = DAG.getMachineFunction();
4260 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4270 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
4271 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
4272 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
4273 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
4282 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
4283 DAG.getIntPtrConstant(Offset, DL));
4284 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
4288 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
4292 SelectionDAG &DAG) const {
4302 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
4310 SelectionDAG &DAG) const {
4312 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4314 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4319 SelectionDAG &DAG) const {
4321 MachineFunction &MF = DAG.getMachineFunction();
4341 NeededSpace = DAG.getNode(ISD::ADD, DL, PtrVT, NeededSpace,
4342 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
4349 makeExternalCall(Chain, DAG, "@@ALCAXP", VT, ArrayRef(NeededSpace),
4361 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, DL, SPReg, PtrVT, Glue);
4365 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, PtrMVT);
4366 SDValue Result = DAG.getNode(ISD::ADD, DL, PtrMVT, NewSPRegNode, ArgAdjust);
4370 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
4371 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
4372 Result = DAG.getNode(ISD::AND, DL, PtrVT, Result,
4373 DAG.getConstant(~(RequiredAlign - 1), DL, PtrVT));
4377 return DAG.getMergeValues(Ops, DL);
4382 SelectionDAG &DAG) const {
4384 MachineFunction &MF = DAG.getMachineFunction();
4405 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
4410 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
4415 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
4416 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4421 NewSP = DAG.getNode(SystemZISD::PROBED_ALLOCA, DL,
4422 DAG.getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
4426 NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
4428 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
4434 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4435 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
4440 DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
4441 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4443 DAG.getNode(ISD::AND, DL, MVT::i64, Result,
4444 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
4448 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
4452 return DAG.getMergeValues(Ops, DL);
4456 SDValue Op, SelectionDAG &DAG) const {
4459 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4463 SelectionDAG &DAG,
4475 lowerGR128Binary(DAG, DL, VT, Opcode,
4481 SelectionDAG &DAG) const {
4488 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
4494 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
4510 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
4513 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
4514 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
4518 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4520 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
4521 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
4522 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
4523 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
4525 return DAG.getMergeValues(Ops, DL);
4529 SelectionDAG &DAG) const {
4536 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
4542 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4544 return DAG.getMergeValues(Ops, DL);
4548 SelectionDAG &DAG) const {
4558 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
4559 else if (DAG.ComputeNumSignBits(Op1) > 32)
4560 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
4565 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
4566 return DAG.getMergeValues(Ops, DL);
4570 SelectionDAG &DAG) const {
4577 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
4579 return DAG.getMergeValues(Ops, DL);
4582 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4587 KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
4588 DAG.computeKnownBits(Ops[1])};
4623 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
4632 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
4633 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
4639 SelectionDAG &DAG) const {
4661 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS);
4662 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS);
4663 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
4664 DAG.getValueType(MVT::i1));
4665 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
4667 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
4668 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
4669 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
4700 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
4701 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
4703 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
4705 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
4707 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
4728 SelectionDAG &DAG) const {
4734 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4759 Carry = DAG.getNode(ISD::XOR, DL, Carry.getValueType(),
4760 Carry, DAG.getConstant(1, DL, Carry.getValueType()));
4761 Carry = DAG.getZExtOrTrunc(Carry, DL, MVT::i128);
4762 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS, Carry);
4763 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS, Carry);
4764 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
4765 DAG.getValueType(MVT::i1));
4766 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
4768 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
4769 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
4770 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
4798 Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
4799 DAG.getConstant(CCValid, DL, MVT::i32),
4800 DAG.getConstant(CCMask, DL, MVT::i32));
4802 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
4803 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
4805 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
4807 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
4809 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
4813 SelectionDAG &DAG) const {
4819 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op);
4820 Op = DAG.getNode(ISD::CTPOP, DL, MVT::v2i64, Op);
4821 SDValue Tmp = DAG.getSplatBuildVector(MVT::v2i64, DL,
4822 DAG.getConstant(0, DL, MVT::i64));
4823 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4829 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
4830 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
4835 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
4836 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
4837 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
4838 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
4839 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
4843 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
4844 DAG.getConstant(0, DL, MVT::i32));
4845 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4849 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
4850 DAG.getConstant(0, DL, MVT::i32));
4851 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
4852 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4862 KnownBits Known = DAG.computeKnownBits(Op);
4865 return DAG.getConstant(0, DL, VT);
4873 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
4874 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
4875 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
4880 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
4882 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
4883 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
4884 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
4889 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
4890 DAG.getConstant(BitSize - 8, DL, VT));
4896 SelectionDAG &DAG) const {
4907 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
4913 return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
4917 SelectionDAG &DAG) const {
4924 LowerOperationWrapper(Node, Results, DAG);
4925 return DAG.getMergeValues(Results, SDLoc(Op));
4930 static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
4937 AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
4938 DAG.getSignedConstant(-4, DL, PtrVT));
4942 BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
4943 DAG.getConstant(3, DL, PtrVT));
4944 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
4948 NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
4949 DAG.getConstant(0, DL, WideVT), BitShift);
4956 SelectionDAG &DAG,
4977 Src2 = DAG.getSignedConstant(-Const->getSExtValue(), DL,
4982 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
4990 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
4991 DAG.getConstant(32 - BitSize, DL, WideVT));
4994 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
4995 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
4998 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
5000 DAG.getConstant(BitSize, DL, WideVT) };
5001 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
5006 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
5007 DAG.getConstant(BitSize, DL, WideVT));
5008 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
5011 return DAG.getMergeValues(RetOps, DL);
5017 SelectionDAG &DAG) const {
5028 DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2);
5029 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
5034 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
5039 SelectionDAG &DAG) const {
5051 LowerOperationWrapper(Node, Results, DAG);
5052 return DAG.getMergeValues(Results, DL);
5060 SDVTList Tys = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
5062 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
5064 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
5067 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
5068 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
5069 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
5078 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5081 SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
5083 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
5084 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
5086 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
5090 SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
5091 DAG.getValueType(NarrowVT));
5092 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
5093 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
5094 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
5101 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5120 SelectionDAG &DAG) const {
5121 MachineFunction &MF = DAG.getMachineFunction();
5126 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
5131 SelectionDAG &DAG) const {
5132 MachineFunction &MF = DAG.getMachineFunction();
5146 SDValue OldSP = DAG.getCopyFromReg(
5148 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
5152 Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP);
5155 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
5162 SelectionDAG &DAG) const {
5172 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
5174 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
5180 static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
5182 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
5183 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
5184 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
5189 SelectionDAG &DAG) const {
5193 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5194 SDValue CC = getCCResult(DAG, SDValue(Node, 0));
5195 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
5204 SelectionDAG &DAG) const {
5207 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5209 return getCCResult(DAG, SDValue(Node, 0));
5211 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
5212 SDValue(Node, 0), getCCResult(DAG, SDValue(Node, 1)));
5218 return lowerThreadPointer(SDLoc(Op), DAG);
5221 return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
5225 return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
5232 return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
5239 return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
5246 return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
5253 return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
5262 return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
5266 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5273 return DAG.getNode(SystemZISD::VACC, SDLoc(Op), Op.getValueType(),
5276 return DAG.getNode(SystemZISD::VAC, SDLoc(Op), Op.getValueType(),
5279 return DAG.getNode(SystemZISD::VACCC, SDLoc(Op), Op.getValueType(),
5283 return DAG.getNode(ISD::SUB, SDLoc(Op), Op.getValueType(),
5290 return DAG.getNode(SystemZISD::VSCBI, SDLoc(Op), Op.getValueType(),
5293 return DAG.getNode(SystemZISD::VSBI, SDLoc(Op), Op.getValueType(),
5296 return DAG.getNode(SystemZISD::VSBCBI, SDLoc(Op), Op.getValueType(),
5539 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5549 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
5550 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
5553 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
5554 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
5558 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
5560 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
5585 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5589 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
5594 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
5596 DAG.getTargetConstant(StartIndex, DL, MVT::i32));
5626 IndexNodes[I] = DAG.getConstant(ZeroIdx, DL, MVT::i32);
5629 IndexNodes[I] = DAG.getConstant(BIdx, DL, MVT::i32);
5632 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
5634 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
5637 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Mask, Src,
5640 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Src, Mask,
5648 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
5650 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
5651 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
5652 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0],
5665 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
5756 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
5760 return DAG.getUNDEF(VT);
5767 Ops.push_back(DAG.getUNDEF(MVT::v16i8));
5798 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
5810 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
5833 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
5835 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
5837 Op = insertUnpackIfPrepared(DAG, DL, Op);
5839 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
5926 SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
5934 SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op);
5938 return DAG.getNode(SystemZISD::UNPACKL_HIGH, DL, OutVT, PackedOp);
5951 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5958 return DAG.getBuildVector(VT, DL, Ops);
5961 return DAG.getUNDEF(VT);
5962 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
5967 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5971 return DAG.getUNDEF(VT);
5972 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
5975 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
5976 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
5977 buildScalarToVector(DAG, DL, VT, Op0),
5978 buildScalarToVector(DAG, DL, VT, Op1));
5983 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
5986 return DAG.getUNDEF(MVT::v2i64);
5990 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
5992 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
5994 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
5995 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
5997 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
6004 static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6042 ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
6045 Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
6050 return GS.getNode(DAG, SDLoc(BVN));
6066 SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6097 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
6109 return joinDwords(DAG, DL, Elems[0], Elems[1]);
6113 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
6123 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
6124 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
6133 Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
6134 Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
6135 SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
6137 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
6162 Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
6163 Result = DAG.getBuildVector(VT, DL, Constants);
6182 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal);
6192 Result = DAG.getNode(ISD::BITCAST, DL, VT,
6193 joinDwords(DAG, DL, Elem1, Elem2));
6197 Result = DAG.getUNDEF(VT);
6204 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
6205 DAG.getConstant(I, DL, MVT::i32));
6210 SelectionDAG &DAG) const {
6224 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6229 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
6236 return buildVector(DAG, DL, VT, Ops);
6240 SelectionDAG &DAG) const {
6254 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
6256 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
6257 DAG.getTargetConstant(Index, DL, MVT::i32));
6269 return GS.getNode(DAG, SDLoc(VSN));
6273 SelectionDAG &DAG) const {
6276 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
6277 Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
6278 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
6282 SelectionDAG &DAG) const {
6306 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
6307 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
6308 DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
6309 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
6314 SelectionDAG &DAG) const {
6333 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
6334 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
6335 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
6339 lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6350 DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(PackedOp), OutVT, PackedOp);
6357 lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6367 DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType()));
6378 SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask);
6379 return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf);
6382 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6401 SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
6403 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6411 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
6412 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6428 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
6430 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6439 static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
6453 Op = DAG.getNode(ISD::AND, dl, MVT::i32, Src,
6454 DAG.getConstant(0x7fffffff, dl, MVT::i32));
6455 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op);
6457 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
6458 Op = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
6459 DAG.getConstant(0x7fffffff, dl, MVT::i32));
6460 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op);
6468 SelectionDAG &DAG) const {
6495 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, MVT::i64);
6497 SDValue Intr = DAG.getNode(SystemZISD::TDC, DL, ResultVT, Arg, TDCMaskV);
6498 return getCCResult(DAG, Intr);
6502 SelectionDAG &DAG) const {
6507 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
6510 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
6514 Chain = DAG.getMemIntrinsicNode(
6515 SystemZISD::STCKF, DL, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
6519 return DAG.getLoad(MVT::i64, DL, Chain, StackPtr, MPI);
6523 SelectionDAG &DAG) const {
6526 return lowerFRAMEADDR(Op, DAG);
6528 return lowerRETURNADDR(Op, DAG);
6530 return lowerBR_CC(Op, DAG);
6532 return lowerSELECT_CC(Op, DAG);
6534 return lowerSETCC(Op, DAG);
6536 return lowerSTRICT_FSETCC(Op, DAG, false);
6538 return lowerSTRICT_FSETCC(Op, DAG, true);
6540 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
6542 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG);
6544 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG);
6546 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG);
6548 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG);
6550 return lowerBITCAST(Op, DAG);
6552 return lowerVASTART(Op, DAG);
6554 return lowerVACOPY(Op, DAG);
6556 return lowerDYNAMIC_STACKALLOC(Op, DAG);
6558 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
6560 return lowerMULH(Op, DAG, SystemZISD::SMUL_LOHI);
6562 return lowerMULH(Op, DAG, SystemZISD::UMUL_LOHI);
6564 return lowerSMUL_LOHI(Op, DAG);
6566 return lowerUMUL_LOHI(Op, DAG);
6568 return lowerSDIVREM(Op, DAG);
6570 return lowerUDIVREM(Op, DAG);
6575 return lowerXALUO(Op, DAG);
6578 return lowerUADDSUBO_CARRY(Op, DAG);
6580 return lowerOR(Op, DAG);
6582 return lowerCTPOP(Op, DAG);
6584 return lowerVECREDUCE_ADD(Op, DAG);
6586 return lowerATOMIC_FENCE(Op, DAG);
6588 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
6591 return lowerATOMIC_LDST_I128(Op, DAG);
6593 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
6595 return lowerATOMIC_LOAD_SUB(Op, DAG);
6597 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
6599 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
6601 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
6603 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
6605 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
6607 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
6609 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
6611 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
6613 return lowerATOMIC_CMP_SWAP(Op, DAG);
6615 return lowerSTACKSAVE(Op, DAG);
6617 return lowerSTACKRESTORE(Op, DAG);
6619 return lowerPREFETCH(Op, DAG);
6621 return lowerINTRINSIC_W_CHAIN(Op, DAG);
6623 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
6625 return lowerBUILD_VECTOR(Op, DAG);
6627 return lowerVECTOR_SHUFFLE(Op, DAG);
6629 return lowerSCALAR_TO_VECTOR(Op, DAG);
6631 return lowerINSERT_VECTOR_ELT(Op, DAG);
6633 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
6635 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
6637 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
6639 return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
6641 return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
6643 return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
6645 return lowerAddrSpaceCast(Op, DAG);
6647 return lowerShift(Op, DAG, SystemZISD::VROTL_BY_SCALAR);
6649 return lowerIS_FPCLASS(Op, DAG);
6651 return lowerGET_ROUNDING(Op, DAG);
6653 return lowerREADCYCLECOUNTER(Op, DAG);
6667 static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
6670 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128))
6671 return DAG.getBitcast(MVT::f128, Src);
6674 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
6678 std::tie(Lo, Hi) = DAG.SplitScalar(Src, SL, MVT::i64, MVT::i64);
6680 Hi = DAG.getBitcast(MVT::f64, Hi);
6681 Lo = DAG.getBitcast(MVT::f64, Lo);
6683 SDNode *Pair = DAG.getMachineNode(
6685 {DAG.getTargetConstant(SystemZ::FP128BitRegClassID, SL, MVT::i32), Lo,
6686 DAG.getTargetConstant(SystemZ::subreg_l64, SL, MVT::i32), Hi,
6687 DAG.getTargetConstant(SystemZ::subreg_h64, SL, MVT::i32)});
6691 static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
6694 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128))
6695 return DAG.getBitcast(MVT::i128, Src);
6698 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
6702 DAG.getTargetExtractSubreg(SystemZ::subreg_l64, SL, MVT::f64, Src);
6704 DAG.getTargetExtractSubreg(SystemZ::subreg_h64, SL, MVT::f64, Src);
6705 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i64, LoFP);
6706 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i64, HiFP);
6708 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i128, Lo, Hi);
6716 SelectionDAG &DAG) const {
6720 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other);
6723 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
6726 SDValue Lowered = lowerGR128ToI128(DAG, Res);
6728 Lowered = expandBitCastI128ToF128(DAG, Lowered, DL);
6735 SDVTList Tys = DAG.getVTList(MVT::Other);
6738 Val = expandBitCastF128ToI128(DAG, Val, DL);
6739 Val = lowerI128ToGR128(DAG, Val);
6743 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
6749 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
6756 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other);
6758 lowerI128ToGR128(DAG, N->getOperand(2)),
6759 lowerI128ToGR128(DAG, N->getOperand(3)) };
6761 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
6763 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
6765 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
6766 Results.push_back(lowerGR128ToI128(DAG, Res));
6776 Results.push_back(expandBitCastF128ToI128(DAG, Src, DL));
6788 SelectionDAG &DAG) const {
6789 return LowerOperationWrapper(N, Results, DAG);
6950 SelectionDAG &DAG = DCI.DAG;
6973 return DAG.getUNDEF(ResVT);
7000 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
7004 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
7007 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
7040 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
7043 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
7044 DAG.getConstant(Index, DL, MVT::i32));
7075 VecVT = EVT::getVectorVT(*DCI.DAG.getContext(),
7090 SelectionDAG &DAG = DCI.DAG;
7098 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
7099 DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
7101 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7105 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
7120 KnownBits Known = DAG.computeKnownBits(X);
7125 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
7127 return DAG.getNode(ISD::XOR, SDLoc(N0), VT,
7128 X, DAG.getConstant(Mask, SDLoc(N0), VT));
7141 SelectionDAG &DAG = DCI.DAG;
7150 DAG.getAllOnesConstant(DL, VT),
7151 DAG.getConstant(0, DL, VT), N0.getOperand(2) };
7152 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
7162 SelectionDAG &DAG = DCI.DAG;
7174 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
7176 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
7177 DAG.getConstant(NewShlAmt, SDLoc(Inner),
7179 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
7180 DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
7190 SelectionDAG &DAG = DCI.DAG;
7211 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
7214 SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
7216 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
7293 SelectionDAG &DAG = DCI.DAG;
7297 MVT PtrVT = getPointerTy(DAG.getDataLayout());
7301 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
7303 return DAG.getExtLoad(LN->getExtensionType(), DL, LN->getValueType(0),
7321 SDValue EltLoad = DAG.getLoad(
7330 SDValue EltLoad = DAG.getLoad(
7332 DAG.getObjectPtrOffset(DL, LD->getBasePtr(), TypeSize::getFixed(8)),
7341 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, ArgChains);
7342 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
7369 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
7370 Replicate, DAG.getConstant(0, DL, MVT::i32));
7376 DAG.UpdateNodeOperands(U, Ops);
7406 static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
7414 if (DAG.isSplatValue(BuildVector, true/*AllowUndefs*/) &&
7415 isOnlyUsedByStores(BuildVector, DAG))
7470 SelectionDAG &DAG = DCI.DAG;
7476 MVT PtrVT = getPointerTy(DAG.getDataLayout());
7480 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(DL, PtrVT, SN->getBasePtr(),
7482 return DAG.getStore(SN->getChain(), DL, SN->getValue(), AddrSpaceCast,
7498 return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
7512 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp);
7519 DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other),
7534 return DAG.getMemIntrinsicNode(SystemZISD::VSTER, SDLoc(N),
7535 DAG.getVTList(MVT::Other),
7546 return DAG.getMemIntrinsicNode(SystemZISD::STCKF, SDLoc(N),
7547 DAG.getVTList(MVT::Other),
7558 DAG.getStore(SN->getChain(), DL, HiPart, SN->getBasePtr(),
7562 DAG.getStore(SN->getChain(), DL, LoPart,
7563 DAG.getObjectPtrOffset(DL, SN->getBasePtr(),
7569 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain0, Chain1);
7576 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
7579 isOnlyUsedByStores(Op1, DAG)) {
7602 Word = DAG.getConstant(VCI.OpVals[0], SDLoc(SN), MVT::i32);
7628 Word = DAG.getZExtOrTrunc(LHS->getOperand(0), SDLoc(SN), WordVT);
7634 DAG.isSplatValue(Op1, true/*AllowUndefs*/)) {
7651 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), WordVT, NumElts);
7652 SDValue SplatVal = DAG.getSplatVector(SplatVT, SDLoc(SN), Word);
7653 return DAG.getStore(SN->getChain(), SDLoc(SN), SplatVal,
7663 SelectionDAG &DAG = DCI.DAG;
7680 DAG.getMemIntrinsicNode(SystemZISD::VLER, SDLoc(N),
7681 DAG.getVTList(LD->getValueType(0), MVT::Other),
7702 SelectionDAG &DAG = DCI.DAG;
7720 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), EltVT,
7723 Op = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Op);
7726 Op = DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op);
7744 SelectionDAG &DAG = DCI.DAG;
7747 return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
7778 SelectionDAG &DAG = DCI.DAG;
7801 VRound = DAG.getNode(SystemZISD::STRICT_VROUND, SDLoc(N),
7805 VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
7809 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
7810 VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
7812 DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
7814 DAG.ReplaceAllUsesOfValueWith(OtherRound.getValue(1), Chain);
7816 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
7817 VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
7819 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
7842 SelectionDAG &DAG = DCI.DAG;
7865 VExtend = DAG.getNode(SystemZISD::STRICT_VEXTEND, SDLoc(N),
7869 VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
7873 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f64,
7874 VExtend, DAG.getConstant(1, SDLoc(U), MVT::i32));
7876 DAG.ReplaceAllUsesOfValueWith(OtherExtend, Extract1);
7878 DAG.ReplaceAllUsesOfValueWith(OtherExtend.getValue(1), Chain);
7880 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f64,
7881 VExtend, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
7883 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
7897 SelectionDAG &DAG = DCI.DAG;
7898 LLVMContext &Ctx = *DAG.getContext();
7917 SDValue ExtOp = DAG.getNode(ExtOpcode, SDLoc(N), ExtVT, Op);
7918 return DAG.getNode(Opcode, SDLoc(N), OutVT, ExtOp);
7925 SelectionDAG &DAG = DCI.DAG;
7942 DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N),
7943 DAG.getVTList(LoadVT, MVT::Other),
7949 ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad);
7978 if (DAG.isConstantIntBuildVectorOrConstantInt(Vec) ||
7980 DAG.isConstantIntBuildVectorOrConstantInt(Elt) ||
7987 Vec = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Vec);
7991 Elt = DAG.getNode(ISD::BITCAST, SDLoc(N), EltVT, Elt);
7994 Vec = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Vec);
7996 Elt = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Elt);
7998 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VecVT,
8009 if (DAG.isConstantIntBuildVectorOrConstantInt(Op0) ||
8011 DAG.isConstantIntBuildVectorOrConstantInt(Op1) ||
8015 Op0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op0);
8019 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op1);
8022 Op0 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op0);
8024 Op1 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op1);
8026 return DAG.getVectorShuffle(VecVT, SDLoc(N), Op0, Op1, SV->getMask());
8122 SelectionDAG &DAG = DCI.DAG;
8136 return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
8138 DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
8139 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
8146 SelectionDAG &DAG = DCI.DAG;
8159 return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
8161 DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
8162 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
8213 SelectionDAG &DAG = DCI.DAG;
8217 // do this during DAG combining, but it only succeeds when it can build a
8223 DAG.isConstantIntBuildVectorOrConstantInt(N->getOperand(1)))
8224 return DAG.UnrollVectorOp(N);
8230 SelectionDAG &DAG = DCI.DAG;
8240 return DAG.getLoad(N->getValueType(0), SDLoc(N), N->getOperand(0),
8248 return DAG.getStore(N->getOperand(0), SDLoc(N), N->getOperand(2),
8398 const SelectionDAG &DAG, unsigned Depth,
8403 DAG.computeKnownBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
8405 DAG.computeKnownBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
8413 const SelectionDAG &DAG,
8452 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 1);
8470 Known = DAG.computeKnownBits(SrcOp, SrcDemE, Depth + 1);
8484 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 0);
8488 Known = DAG.computeKnownBits(SrcOp, Depth + 1);
8505 const SelectionDAG &DAG, unsigned Depth,
8508 unsigned LHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
8511 unsigned RHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
8529 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8552 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 1);
8561 unsigned Tmp = DAG.ComputeNumSignBits(PackedOp, SrcDemE, Depth + 1);
8573 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 0);
8584 const APInt &DemandedElts, const SelectionDAG &DAG,
9943 getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
9944 MachineFunction &MF = DAG.getMachineFunction();
9947 return DAG.getNode(ISD::ADD, DL, MVT::i64, SP,
9948 DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL));
10123 SelectionDAG &DAG) const {
10144 DAG.getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0);
10148 SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, EFPC,
10149 DAG.getConstant(3, dl, MVT::i32));
10151 SDValue CWD2 = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1,
10152 DAG.getNode(ISD::SRL, dl, MVT::i32, CWD1,
10153 DAG.getConstant(1, dl, MVT::i32)));
10155 SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD2,
10156 DAG.getConstant(1, dl, MVT::i32));
10157 RetVal = DAG.getZExtOrTrunc(RetVal, dl, Op.getValueType());
10159 return DAG.getMergeValues({RetVal, Chain}, dl);
10163 SelectionDAG &DAG) const {
10173 SDValue Zero = DAG.getSplatBuildVector(OpVT, DL, DAG.getConstant(0, DL, VT));
10179 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Zero);
10183 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::i128, Op,
10184 DAG.getBitcast(Op.getValueType(), Zero));
10192 return DAG.getNode(
10193 ISD::EXTRACT_VECTOR_ELT, DL, VT, DAG.getBitcast(OpVT, Op),
10194 DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));