197   bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
202   unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
205   unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
208   unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
211   unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
215   unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
223   bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt);
224   bool emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
225   bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS);
234   unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
238   unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
241   unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
243   unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
246   unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
248   unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
250   unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
252   unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
253   unsigned emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1);
254   unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
255   unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
256   unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
257   unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
259   unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
260   unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
262   unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
263   unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
275   bool finishCall(CallLoweringInfo &CLI, MVT RetVT, unsigned NumBytes);
1161 unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,  in emitAddSub()  argument
1166   switch (RetVT.SimpleTy) {  in emitAddSub()
1184   MVT SrcVT = RetVT;  in emitAddSub()
1185   RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32);  in emitAddSub()
1210     LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);  in emitAddSub()
1216       ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,  in emitAddSub()
1219       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,  in emitAddSub()
1223       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);  in emitAddSub()
1237           return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType,  in emitAddSub()
1243     return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,  in emitAddSub()
1262       ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, AArch64_AM::LSL,  in emitAddSub()
1285           ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,  in emitAddSub()
1299     RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);  in emitAddSub()
1301   return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);  in emitAddSub()
1304 unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rr()  argument
1313   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rr()
1322   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rr()
1341 unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_ri()  argument
1346   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_ri()
1364   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_ri()
1386 unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rs()  argument
1395   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rs()
1399   if (ShiftImm >= RetVT.getSizeInBits())  in emitAddSub_rs()
1408   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rs()
1428 unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,  in emitAddSub_rx()  argument
1437   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in emitAddSub_rx()
1449   bool Is64Bit = RetVT == MVT::i64;  in emitAddSub_rx()
1494 bool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS,  in emitICmp()  argument
1496   return emitSub(RetVT, LHS, RHS, /*SetFlags=*/true, /*WantResult=*/false,  in emitICmp()
1500 bool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm) {  in emitICmp_ri()  argument
1501   return emitAddSub_ri(/*UseAdd=*/false, RetVT, LHSReg, Imm,  in emitICmp_ri()
1505 bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {  in emitFCmp()  argument
1506   if (RetVT != MVT::f32 && RetVT != MVT::f64)  in emitFCmp()
1521     unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;  in emitFCmp()
1531   unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;  in emitFCmp()
1538 unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,  in emitAdd()  argument
1540   return emitAddSub(/*UseAdd=*/true, RetVT, LHS, RHS, SetFlags, WantResult,  in emitAdd()
1567 unsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,  in emitSub()  argument
1569   return emitAddSub(/*UseAdd=*/false, RetVT, LHS, RHS, SetFlags, WantResult,  in emitSub()
1573 unsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg,  in emitSubs_rr()  argument
1575   return emitAddSub_rr(/*UseAdd=*/false, RetVT, LHSReg, RHSReg,  in emitSubs_rr()
1579 unsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg,  in emitSubs_rs()  argument
1583   return emitAddSub_rs(/*UseAdd=*/false, RetVT, LHSReg, RHSReg, ShiftType,  in emitSubs_rs()
1587 unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp()  argument
1611     ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);  in emitLogicalOp()
1632       ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1646         ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);  in emitLogicalOp()
1656   MVT VT = std::max(MVT::i32, RetVT.SimpleTy);  in emitLogicalOp()
1658   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp()
1659     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp()
1665 unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_ri()  argument
1677   switch (RetVT.SimpleTy) {  in emitLogicalOp_ri()
1703   if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc != ISD::AND) {  in emitLogicalOp_ri()
1704     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_ri()
1710 unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,  in emitLogicalOp_rs()  argument
1722   if (ShiftImm >= RetVT.getSizeInBits())  in emitLogicalOp_rs()
1727   switch (RetVT.SimpleTy) {  in emitLogicalOp_rs()
1745   if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {  in emitLogicalOp_rs()
1746     uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff;  in emitLogicalOp_rs()
1752 unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg,  in emitAnd_ri()  argument
1754   return emitLogicalOp_ri(ISD::AND, RetVT, LHSReg, Imm);  in emitAnd_ri()
1757 unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,  in emitLoad()  argument
1833   bool IsRet64Bit = RetVT == MVT::i64;  in emitLoad()
1882   if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {  in emitLoad()
1980   MVT RetVT = VT;  in selectLoad()  local
1984       if (isTypeSupported(ZE->getType(), RetVT))  in selectLoad()
1987         RetVT = VT;  in selectLoad()
1989       if (isTypeSupported(SE->getType(), RetVT))  in selectLoad()
1992         RetVT = VT;  in selectLoad()
1998       emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));  in selectLoad()
2021       if (RetVT == MVT::i64 && VT <= MVT::i32) {  in selectLoad()
3105 bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,  in finishCall()  argument
3115   if (RetVT != MVT::isVoid) {  in finishCall()
3118     CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));  in finishCall()
3189   MVT RetVT;  in fastLowerCall()  local
3191     RetVT = MVT::isVoid;  in fastLowerCall()
3192   else if (!isTypeLegal(CLI.RetTy, RetVT))  in fastLowerCall()
3290   return finishCall(CLI, RetVT, NumBytes);  in fastLowerCall()
3366   MVT RetVT;  in foldXALUIntrinsic()  local
3370   if (!isTypeLegal(RetTy, RetVT))  in foldXALUIntrinsic()
3373   if (RetVT != MVT::i32 && RetVT != MVT::i64)  in foldXALUIntrinsic()
3546     MVT RetVT;  in fastLowerIntrinsicCall()  local
3547     if (!isTypeLegal(II->getType(), RetVT))  in fastLowerIntrinsicCall()
3550     if (RetVT != MVT::f32 && RetVT != MVT::f64)  in fastLowerIntrinsicCall()
3559     bool Is64Bit = RetVT == MVT::f64;  in fastLowerIntrinsicCall()
3997 unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitMul_rr()  argument
3999   switch (RetVT.SimpleTy) {  in emitMul_rr()
4004     RetVT = MVT::i32;  in emitMul_rr()
4011       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitMul_rr()
4015 unsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitSMULL_rr()  argument
4016   if (RetVT != MVT::i64)  in emitSMULL_rr()
4023 unsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {  in emitUMULL_rr()  argument
4024   if (RetVT != MVT::i64)  in emitUMULL_rr()
4031 unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,  in emitLSL_rr()  argument
4036   switch (RetVT.SimpleTy) {  in emitLSL_rr()
4045       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSL_rr()
4055 unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSL_ri()  argument
4057   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSL_ri()
4062   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSL_ri()
4063           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSL_ri()
4065   bool Is64Bit = (RetVT == MVT::i64);  in emitLSL_ri()
4067   unsigned DstBits = RetVT.getSizeInBits();  in emitLSL_ri()
4074     if (RetVT == SrcVT) {  in emitLSL_ri()
4081       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSL_ri()
4121   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSL_ri()
4133 unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,  in emitLSR_rr()  argument
4138   switch (RetVT.SimpleTy) {  in emitLSR_rr()
4147       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitLSR_rr()
4158 unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitLSR_ri()  argument
4160   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitLSR_ri()
4165   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitLSR_ri()
4166           RetVT == MVT::i64) && "Unexpected return value type.");  in emitLSR_ri()
4168   bool Is64Bit = (RetVT == MVT::i64);  in emitLSR_ri()
4170   unsigned DstBits = RetVT.getSizeInBits();  in emitLSR_ri()
4177     if (RetVT == SrcVT) {  in emitLSR_ri()
4184       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4217     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitLSR_ri()
4222     Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitLSR_ri()
4225     SrcVT = RetVT;  in emitLSR_ri()
4237   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitLSR_ri()
4249 unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,  in emitASR_rr()  argument
4254   switch (RetVT.SimpleTy) {  in emitASR_rr()
4263       (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;  in emitASR_rr()
4265     Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false);  in emitASR_rr()
4274 unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,  in emitASR_ri()  argument
4276   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&  in emitASR_ri()
4281   assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||  in emitASR_ri()
4282           RetVT == MVT::i64) && "Unexpected return value type.");  in emitASR_ri()
4284   bool Is64Bit = (RetVT == MVT::i64);  in emitASR_ri()
4286   unsigned DstBits = RetVT.getSizeInBits();  in emitASR_ri()
4293     if (RetVT == SrcVT) {  in emitASR_ri()
4300       return emitIntExt(SrcVT, Op0, RetVT, IsZExt);  in emitASR_ri()
4333     return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT);  in emitASR_ri()
4342   if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) {  in emitASR_ri()
4463 bool AArch64FastISel::optimizeIntExtLoad(const Instruction *I, MVT RetVT,  in optimizeIntExtLoad()  argument
4492   if (RetVT != MVT::i64 || SrcVT > MVT::i32) {  in optimizeIntExtLoad()
4520   MVT RetVT;  in selectIntExt()  local
4522   if (!isTypeSupported(I->getType(), RetVT))  in selectIntExt()
4529   if (optimizeIntExtLoad(I, RetVT, SrcVT))  in selectIntExt()
4540       if (RetVT == MVT::i64 && SrcVT != MVT::i64) {  in selectIntExt()
4555   unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);  in selectIntExt()
4675   MVT RetVT;  in selectShift()  local
4676   if (!isTypeSupported(I->getType(), RetVT, /*IsVectorAllowed=*/true))  in selectShift()
4679   if (RetVT.isVector())  in selectShift()
4685     MVT SrcVT = RetVT;  in selectShift()
4715       ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4718       ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4721       ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);  in selectShift()
4743     ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4746     ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4749     ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);  in selectShift()
4761   MVT RetVT, SrcVT;  in selectBitCast()  local
4765   if (!isTypeLegal(I->getType(), RetVT))  in selectBitCast()
4769   if (RetVT == MVT::f32 && SrcVT == MVT::i32)  in selectBitCast()
4771   else if (RetVT == MVT::f64 && SrcVT == MVT::i64)  in selectBitCast()
4773   else if (RetVT == MVT::i32 && SrcVT == MVT::f32)  in selectBitCast()
4775   else if (RetVT == MVT::i64 && SrcVT == MVT::f64)  in selectBitCast()
4781   switch (RetVT.SimpleTy) {  in selectBitCast()
4801   MVT RetVT;  in selectFRem()  local
4802   if (!isTypeLegal(I->getType(), RetVT))  in selectFRem()
4806   switch (RetVT.SimpleTy) {  in selectFRem()