xref: /llvm-project/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp (revision 2f4328e6979004fbf531d69a40c2e06d43d3128c)
1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
15 #include "llvm/CodeGen/Analysis.h"
16 #include "llvm/CodeGen/CallingConvLower.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/GlobalISel/Utils.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/TargetLowering.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/LLVMContext.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/Target/TargetMachine.h"
27 
28 #define DEBUG_TYPE "call-lowering"
29 
30 using namespace llvm;
31 
32 void CallLowering::anchor() {}
33 
34 /// Helper function which updates \p Flags when \p AttrFn returns true.
35 static void
36 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
37                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
38   if (AttrFn(Attribute::SExt))
39     Flags.setSExt();
40   if (AttrFn(Attribute::ZExt))
41     Flags.setZExt();
42   if (AttrFn(Attribute::InReg))
43     Flags.setInReg();
44   if (AttrFn(Attribute::StructRet))
45     Flags.setSRet();
46   if (AttrFn(Attribute::Nest))
47     Flags.setNest();
48   if (AttrFn(Attribute::ByVal))
49     Flags.setByVal();
50   if (AttrFn(Attribute::Preallocated))
51     Flags.setPreallocated();
52   if (AttrFn(Attribute::InAlloca))
53     Flags.setInAlloca();
54   if (AttrFn(Attribute::Returned))
55     Flags.setReturned();
56   if (AttrFn(Attribute::SwiftSelf))
57     Flags.setSwiftSelf();
58   if (AttrFn(Attribute::SwiftAsync))
59     Flags.setSwiftAsync();
60   if (AttrFn(Attribute::SwiftError))
61     Flags.setSwiftError();
62 }
63 
64 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
65                                                      unsigned ArgIdx) const {
66   ISD::ArgFlagsTy Flags;
67   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
68     return Call.paramHasAttr(ArgIdx, Attr);
69   });
70   return Flags;
71 }
72 
73 ISD::ArgFlagsTy
74 CallLowering::getAttributesForReturn(const CallBase &Call) const {
75   ISD::ArgFlagsTy Flags;
76   addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {
77     return Call.hasRetAttr(Attr);
78   });
79   return Flags;
80 }
81 
82 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
83                                              const AttributeList &Attrs,
84                                              unsigned OpIdx) const {
85   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
86     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
87   });
88 }
89 
90 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
91                              ArrayRef<Register> ResRegs,
92                              ArrayRef<ArrayRef<Register>> ArgRegs,
93                              Register SwiftErrorVReg,
94                              std::function<unsigned()> GetCalleeReg) const {
95   CallLoweringInfo Info;
96   const DataLayout &DL = MIRBuilder.getDataLayout();
97   MachineFunction &MF = MIRBuilder.getMF();
98   MachineRegisterInfo &MRI = MF.getRegInfo();
99   bool CanBeTailCalled = CB.isTailCall() &&
100                          isInTailCallPosition(CB, MF.getTarget()) &&
101                          (MF.getFunction()
102                               .getFnAttribute("disable-tail-calls")
103                               .getValueAsString() != "true");
104 
105   CallingConv::ID CallConv = CB.getCallingConv();
106   Type *RetTy = CB.getType();
107   bool IsVarArg = CB.getFunctionType()->isVarArg();
108 
109   SmallVector<BaseArgInfo, 4> SplitArgs;
110   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
111   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
112 
113   Info.IsConvergent = CB.isConvergent();
114 
115   if (!Info.CanLowerReturn) {
116     // Callee requires sret demotion.
117     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
118 
119     // The sret demotion isn't compatible with tail-calls, since the sret
120     // argument points into the caller's stack frame.
121     CanBeTailCalled = false;
122   }
123 
124 
125   // First step is to marshall all the function's parameters into the correct
126   // physregs and memory locations. Gather the sequence of argument types that
127   // we'll pass to the assigner function.
128   unsigned i = 0;
129   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
130   for (const auto &Arg : CB.args()) {
131     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
132                     i < NumFixedArgs};
133     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
134 
135     // If we have an explicit sret argument that is an Instruction, (i.e., it
136     // might point to function-local memory), we can't meaningfully tail-call.
137     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
138       CanBeTailCalled = false;
139 
140     Info.OrigArgs.push_back(OrigArg);
141     ++i;
142   }
143 
144   // Try looking through a bitcast from one function type to another.
145   // Commonly happens with calls to objc_msgSend().
146   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
147   if (const Function *F = dyn_cast<Function>(CalleeV))
148     Info.Callee = MachineOperand::CreateGA(F, 0);
149   else
150     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
151 
152   Register ReturnHintAlignReg;
153   Align ReturnHintAlign;
154 
155   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};
156 
157   if (!Info.OrigRet.Ty->isVoidTy()) {
158     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
159 
160     if (MaybeAlign Alignment = CB.getRetAlign()) {
161       if (*Alignment > Align(1)) {
162         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
163         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
164         ReturnHintAlign = *Alignment;
165       }
166     }
167   }
168 
169   auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);
170   if (Bundle && CB.isIndirectCall()) {
171     Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
172     assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
173   }
174 
175   Info.CB = &CB;
176   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
177   Info.CallConv = CallConv;
178   Info.SwiftErrorVReg = SwiftErrorVReg;
179   Info.IsMustTailCall = CB.isMustTailCall();
180   Info.IsTailCall = CanBeTailCalled;
181   Info.IsVarArg = IsVarArg;
182   if (!lowerCall(MIRBuilder, Info))
183     return false;
184 
185   if (ReturnHintAlignReg && !Info.IsTailCall) {
186     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
187                                 ReturnHintAlign);
188   }
189 
190   return true;
191 }
192 
193 template <typename FuncInfoTy>
194 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
195                                const DataLayout &DL,
196                                const FuncInfoTy &FuncInfo) const {
197   auto &Flags = Arg.Flags[0];
198   const AttributeList &Attrs = FuncInfo.getAttributes();
199   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
200 
201   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
202   if (PtrTy) {
203     Flags.setPointer();
204     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
205   }
206 
207   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
208   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
209     assert(OpIdx >= AttributeList::FirstArgIndex);
210     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
211 
212     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
213     if (!ElementTy)
214       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
215     if (!ElementTy)
216       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
217     assert(ElementTy && "Must have byval, inalloca or preallocated type");
218     Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
219 
220     // For ByVal, alignment should be passed from FE.  BE will guess if
221     // this info is not there but there are cases it cannot get right.
222     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
223       MemAlign = *ParamAlign;
224     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
225       MemAlign = *ParamAlign;
226     else
227       MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
228   } else if (OpIdx >= AttributeList::FirstArgIndex) {
229     if (auto ParamAlign =
230             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
231       MemAlign = *ParamAlign;
232   }
233   Flags.setMemAlign(MemAlign);
234   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
235 
236   // Don't try to use the returned attribute if the argument is marked as
237   // swiftself, since it won't be passed in x0.
238   if (Flags.isSwiftSelf())
239     Flags.setReturned(false);
240 }
241 
242 template void
243 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
244                                     const DataLayout &DL,
245                                     const Function &FuncInfo) const;
246 
247 template void
248 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
249                                     const DataLayout &DL,
250                                     const CallBase &FuncInfo) const;
251 
252 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
253                                      SmallVectorImpl<ArgInfo> &SplitArgs,
254                                      const DataLayout &DL,
255                                      CallingConv::ID CallConv,
256                                      SmallVectorImpl<uint64_t> *Offsets) const {
257   LLVMContext &Ctx = OrigArg.Ty->getContext();
258 
259   SmallVector<EVT, 4> SplitVTs;
260   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
261 
262   if (SplitVTs.size() == 0)
263     return;
264 
265   if (SplitVTs.size() == 1) {
266     // No splitting to do, but we want to replace the original type (e.g. [1 x
267     // double] -> double).
268     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
269                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
270                            OrigArg.IsFixed, OrigArg.OrigValue);
271     return;
272   }
273 
274   // Create one ArgInfo for each virtual register in the original ArgInfo.
275   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
276 
277   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
278       OrigArg.Ty, CallConv, false, DL);
279   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
280     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
281     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
282                            OrigArg.Flags[0], OrigArg.IsFixed);
283     if (NeedsRegBlock)
284       SplitArgs.back().Flags[0].setInConsecutiveRegs();
285   }
286 
287   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
288 }
289 
290 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
291 static MachineInstrBuilder
292 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
293                             ArrayRef<Register> SrcRegs) {
294   MachineRegisterInfo &MRI = *B.getMRI();
295   LLT LLTy = MRI.getType(DstRegs[0]);
296   LLT PartLLT = MRI.getType(SrcRegs[0]);
297 
298   // Deal with v3s16 split into v2s16
299   LLT LCMTy = getCoverTy(LLTy, PartLLT);
300   if (LCMTy == LLTy) {
301     // Common case where no padding is needed.
302     assert(DstRegs.size() == 1);
303     return B.buildConcatVectors(DstRegs[0], SrcRegs);
304   }
305 
306   // We need to create an unmerge to the result registers, which may require
307   // widening the original value.
308   Register UnmergeSrcReg;
309   if (LCMTy != PartLLT) {
310     assert(DstRegs.size() == 1);
311     return B.buildDeleteTrailingVectorElements(
312         DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));
313   } else {
314     // We don't need to widen anything if we're extracting a scalar which was
315     // promoted to a vector e.g. s8 -> v4s8 -> s8
316     assert(SrcRegs.size() == 1);
317     UnmergeSrcReg = SrcRegs[0];
318   }
319 
320   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
321 
322   SmallVector<Register, 8> PadDstRegs(NumDst);
323   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
324 
325   // Create the excess dead defs for the unmerge.
326   for (int I = DstRegs.size(); I != NumDst; ++I)
327     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
328 
329   if (PadDstRegs.size() == 1)
330     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
331   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
332 }
333 
334 /// Create a sequence of instructions to combine pieces split into register
335 /// typed values to the original IR value. \p OrigRegs contains the destination
336 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
337 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
338 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
339                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
340                               const ISD::ArgFlagsTy Flags) {
341   MachineRegisterInfo &MRI = *B.getMRI();
342 
343   if (PartLLT == LLTy) {
344     // We should have avoided introducing a new virtual register, and just
345     // directly assigned here.
346     assert(OrigRegs[0] == Regs[0]);
347     return;
348   }
349 
350   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
351       Regs.size() == 1) {
352     B.buildBitcast(OrigRegs[0], Regs[0]);
353     return;
354   }
355 
356   // A vector PartLLT needs extending to LLTy's element size.
357   // E.g. <2 x s64> = G_SEXT <2 x s32>.
358   if (PartLLT.isVector() == LLTy.isVector() &&
359       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
360       (!PartLLT.isVector() ||
361        PartLLT.getNumElements() == LLTy.getNumElements()) &&
362       OrigRegs.size() == 1 && Regs.size() == 1) {
363     Register SrcReg = Regs[0];
364 
365     LLT LocTy = MRI.getType(SrcReg);
366 
367     if (Flags.isSExt()) {
368       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
369                    .getReg(0);
370     } else if (Flags.isZExt()) {
371       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
372                    .getReg(0);
373     }
374 
375     // Sometimes pointers are passed zero extended.
376     LLT OrigTy = MRI.getType(OrigRegs[0]);
377     if (OrigTy.isPointer()) {
378       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
379       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
380       return;
381     }
382 
383     B.buildTrunc(OrigRegs[0], SrcReg);
384     return;
385   }
386 
387   if (!LLTy.isVector() && !PartLLT.isVector()) {
388     assert(OrigRegs.size() == 1);
389     LLT OrigTy = MRI.getType(OrigRegs[0]);
390 
391     unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
392     if (SrcSize == OrigTy.getSizeInBits())
393       B.buildMergeValues(OrigRegs[0], Regs);
394     else {
395       auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);
396       B.buildTrunc(OrigRegs[0], Widened);
397     }
398 
399     return;
400   }
401 
402   if (PartLLT.isVector()) {
403     assert(OrigRegs.size() == 1);
404     SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
405 
406     // If PartLLT is a mismatched vector in both number of elements and element
407     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
408     // have the same elt type, i.e. v4s32.
409     if (PartLLT.getSizeInBits() > LLTy.getSizeInBits() &&
410         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
411         Regs.size() == 1) {
412       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
413                       .changeElementCount(PartLLT.getElementCount() * 2);
414       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
415       PartLLT = NewTy;
416     }
417 
418     if (LLTy.getScalarType() == PartLLT.getElementType()) {
419       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
420     } else {
421       unsigned I = 0;
422       LLT GCDTy = getGCDType(LLTy, PartLLT);
423 
424       // We are both splitting a vector, and bitcasting its element types. Cast
425       // the source pieces into the appropriate number of pieces with the result
426       // element type.
427       for (Register SrcReg : CastRegs)
428         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
429       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
430     }
431 
432     return;
433   }
434 
435   assert(LLTy.isVector() && !PartLLT.isVector());
436 
437   LLT DstEltTy = LLTy.getElementType();
438 
439   // Pointer information was discarded. We'll need to coerce some register types
440   // to avoid violating type constraints.
441   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
442 
443   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
444 
445   if (DstEltTy == PartLLT) {
446     // Vector was trivially scalarized.
447 
448     if (RealDstEltTy.isPointer()) {
449       for (Register Reg : Regs)
450         MRI.setType(Reg, RealDstEltTy);
451     }
452 
453     B.buildBuildVector(OrigRegs[0], Regs);
454   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
455     // Deal with vector with 64-bit elements decomposed to 32-bit
456     // registers. Need to create intermediate 64-bit elements.
457     SmallVector<Register, 8> EltMerges;
458     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
459 
460     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
461 
462     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
463       auto Merge =
464           B.buildMergeLikeInstr(RealDstEltTy, Regs.take_front(PartsPerElt));
465       // Fix the type in case this is really a vector of pointers.
466       MRI.setType(Merge.getReg(0), RealDstEltTy);
467       EltMerges.push_back(Merge.getReg(0));
468       Regs = Regs.drop_front(PartsPerElt);
469     }
470 
471     B.buildBuildVector(OrigRegs[0], EltMerges);
472   } else {
473     // Vector was split, and elements promoted to a wider type.
474     // FIXME: Should handle floating point promotions.
475     LLT BVType = LLT::fixed_vector(LLTy.getNumElements(), PartLLT);
476     auto BV = B.buildBuildVector(BVType, Regs);
477     B.buildTrunc(OrigRegs[0], BV);
478   }
479 }
480 
481 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
482 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
483 /// contain the type of scalar value extension if necessary.
484 ///
485 /// This is used for outgoing values (vregs to physregs)
486 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
487                             Register SrcReg, LLT SrcTy, LLT PartTy,
488                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
489   // We could just insert a regular copy, but this is unreachable at the moment.
490   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
491 
492   const unsigned PartSize = PartTy.getSizeInBits();
493 
494   if (PartTy.isVector() == SrcTy.isVector() &&
495       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
496     assert(DstRegs.size() == 1);
497     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
498     return;
499   }
500 
501   if (SrcTy.isVector() && !PartTy.isVector() &&
502       PartSize > SrcTy.getElementType().getSizeInBits()) {
503     // Vector was scalarized, and the elements extended.
504     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
505     for (int i = 0, e = DstRegs.size(); i != e; ++i)
506       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
507     return;
508   }
509 
510   if (SrcTy.isVector() && PartTy.isVector() &&
511       PartTy.getScalarSizeInBits() == SrcTy.getScalarSizeInBits() &&
512       SrcTy.getNumElements() < PartTy.getNumElements()) {
513     // A coercion like: v2f32 -> v4f32.
514     Register DstReg = DstRegs.front();
515     B.buildPadVectorWithUndefElements(DstReg, SrcReg);
516     return;
517   }
518 
519   LLT GCDTy = getGCDType(SrcTy, PartTy);
520   if (GCDTy == PartTy) {
521     // If this already evenly divisible, we can create a simple unmerge.
522     B.buildUnmerge(DstRegs, SrcReg);
523     return;
524   }
525 
526   MachineRegisterInfo &MRI = *B.getMRI();
527   LLT DstTy = MRI.getType(DstRegs[0]);
528   LLT LCMTy = getCoverTy(SrcTy, PartTy);
529 
530   if (PartTy.isVector() && LCMTy == PartTy) {
531     assert(DstRegs.size() == 1);
532     B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);
533     return;
534   }
535 
536   const unsigned DstSize = DstTy.getSizeInBits();
537   const unsigned SrcSize = SrcTy.getSizeInBits();
538   unsigned CoveringSize = LCMTy.getSizeInBits();
539 
540   Register UnmergeSrc = SrcReg;
541 
542   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
543     // For scalars, it's common to be able to use a simple extension.
544     if (SrcTy.isScalar() && DstTy.isScalar()) {
545       CoveringSize = alignTo(SrcSize, DstSize);
546       LLT CoverTy = LLT::scalar(CoveringSize);
547       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
548     } else {
549       // Widen to the common type.
550       // FIXME: This should respect the extend type
551       Register Undef = B.buildUndef(SrcTy).getReg(0);
552       SmallVector<Register, 8> MergeParts(1, SrcReg);
553       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
554         MergeParts.push_back(Undef);
555       UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);
556     }
557   }
558 
559   if (LCMTy.isVector() && CoveringSize != SrcSize)
560     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
561 
562   B.buildUnmerge(DstRegs, UnmergeSrc);
563 }
564 
565 bool CallLowering::determineAndHandleAssignments(
566     ValueHandler &Handler, ValueAssigner &Assigner,
567     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
568     CallingConv::ID CallConv, bool IsVarArg,
569     ArrayRef<Register> ThisReturnRegs) const {
570   MachineFunction &MF = MIRBuilder.getMF();
571   const Function &F = MF.getFunction();
572   SmallVector<CCValAssign, 16> ArgLocs;
573 
574   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
575   if (!determineAssignments(Assigner, Args, CCInfo))
576     return false;
577 
578   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
579                            ThisReturnRegs);
580 }
581 
582 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
583   if (Flags.isSExt())
584     return TargetOpcode::G_SEXT;
585   if (Flags.isZExt())
586     return TargetOpcode::G_ZEXT;
587   return TargetOpcode::G_ANYEXT;
588 }
589 
590 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
591                                         SmallVectorImpl<ArgInfo> &Args,
592                                         CCState &CCInfo) const {
593   LLVMContext &Ctx = CCInfo.getContext();
594   const CallingConv::ID CallConv = CCInfo.getCallingConv();
595 
596   unsigned NumArgs = Args.size();
597   for (unsigned i = 0; i != NumArgs; ++i) {
598     EVT CurVT = EVT::getEVT(Args[i].Ty);
599 
600     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
601 
602     // If we need to split the type over multiple regs, check it's a scenario
603     // we currently support.
604     unsigned NumParts =
605         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
606 
607     if (NumParts == 1) {
608       // Try to use the register type if we couldn't assign the VT.
609       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
610                              Args[i].Flags[0], CCInfo))
611         return false;
612       continue;
613     }
614 
615     // For incoming arguments (physregs to vregs), we could have values in
616     // physregs (or memlocs) which we want to extract and copy to vregs.
617     // During this, we might have to deal with the LLT being split across
618     // multiple regs, so we have to record this information for later.
619     //
620     // If we have outgoing args, then we have the opposite case. We have a
621     // vreg with an LLT which we want to assign to a physical location, and
622     // we might have to record that the value has to be split later.
623 
624     // We're handling an incoming arg which is split over multiple regs.
625     // E.g. passing an s128 on AArch64.
626     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
627     Args[i].Flags.clear();
628 
629     for (unsigned Part = 0; Part < NumParts; ++Part) {
630       ISD::ArgFlagsTy Flags = OrigFlags;
631       if (Part == 0) {
632         Flags.setSplit();
633       } else {
634         Flags.setOrigAlign(Align(1));
635         if (Part == NumParts - 1)
636           Flags.setSplitEnd();
637       }
638 
639       Args[i].Flags.push_back(Flags);
640       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
641                              Args[i].Flags[Part], CCInfo)) {
642         // Still couldn't assign this smaller part type for some reason.
643         return false;
644       }
645     }
646   }
647 
648   return true;
649 }
650 
651 bool CallLowering::handleAssignments(ValueHandler &Handler,
652                                      SmallVectorImpl<ArgInfo> &Args,
653                                      CCState &CCInfo,
654                                      SmallVectorImpl<CCValAssign> &ArgLocs,
655                                      MachineIRBuilder &MIRBuilder,
656                                      ArrayRef<Register> ThisReturnRegs) const {
657   MachineFunction &MF = MIRBuilder.getMF();
658   MachineRegisterInfo &MRI = MF.getRegInfo();
659   const Function &F = MF.getFunction();
660   const DataLayout &DL = F.getParent()->getDataLayout();
661 
662   const unsigned NumArgs = Args.size();
663 
664   // Stores thunks for outgoing register assignments. This is used so we delay
665   // generating register copies until mem loc assignments are done. We do this
666   // so that if the target is using the delayed stack protector feature, we can
667   // find the split point of the block accurately. E.g. if we have:
668   // G_STORE %val, %memloc
669   // $x0 = COPY %foo
670   // $x1 = COPY %bar
671   // CALL func
672   // ... then the split point for the block will correctly be at, and including,
673   // the copy to $x0. If instead the G_STORE instruction immediately precedes
674   // the CALL, then we'd prematurely choose the CALL as the split point, thus
675   // generating a split block with a CALL that uses undefined physregs.
676   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
677 
678   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
679     assert(j < ArgLocs.size() && "Skipped too many arg locs");
680     CCValAssign &VA = ArgLocs[j];
681     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
682 
683     if (VA.needsCustom()) {
684       std::function<void()> Thunk;
685       unsigned NumArgRegs = Handler.assignCustomValue(
686           Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);
687       if (Thunk)
688         DelayedOutgoingRegAssignments.emplace_back(Thunk);
689       if (!NumArgRegs)
690         return false;
691       j += NumArgRegs;
692       continue;
693     }
694 
695     const MVT ValVT = VA.getValVT();
696     const MVT LocVT = VA.getLocVT();
697 
698     const LLT LocTy(LocVT);
699     const LLT ValTy(ValVT);
700     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
701     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
702     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
703 
704     // Expected to be multiple regs for a single incoming arg.
705     // There should be Regs.size() ArgLocs per argument.
706     // This should be the same as getNumRegistersForCallingConv
707     const unsigned NumParts = Args[i].Flags.size();
708 
709     // Now split the registers into the assigned types.
710     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
711 
712     if (NumParts != 1 || NewLLT != OrigTy) {
713       // If we can't directly assign the register, we need one or more
714       // intermediate values.
715       Args[i].Regs.resize(NumParts);
716 
717       // For each split register, create and assign a vreg that will store
718       // the incoming component of the larger value. These will later be
719       // merged to form the final vreg.
720       for (unsigned Part = 0; Part < NumParts; ++Part)
721         Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
722     }
723 
724     assert((j + (NumParts - 1)) < ArgLocs.size() &&
725            "Too many regs for number of args");
726 
727     // Coerce into outgoing value types before register assignment.
728     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
729       assert(Args[i].OrigRegs.size() == 1);
730       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
731                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
732     }
733 
734     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
735     for (unsigned Part = 0; Part < NumParts; ++Part) {
736       Register ArgReg = Args[i].Regs[Part];
737       // There should be Regs.size() ArgLocs per argument.
738       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
739       CCValAssign &VA = ArgLocs[j + Idx];
740       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
741 
742       if (VA.isMemLoc() && !Flags.isByVal()) {
743         // Individual pieces may have been spilled to the stack and others
744         // passed in registers.
745 
746         // TODO: The memory size may be larger than the value we need to
747         // store. We may need to adjust the offset for big endian targets.
748         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
749 
750         MachinePointerInfo MPO;
751         Register StackAddr = Handler.getStackAddress(
752             MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags);
753 
754         Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA);
755         continue;
756       }
757 
758       if (VA.isMemLoc() && Flags.isByVal()) {
759         assert(Args[i].Regs.size() == 1 &&
760                "didn't expect split byval pointer");
761 
762         if (Handler.isIncomingArgumentHandler()) {
763           // We just need to copy the frame index value to the pointer.
764           MachinePointerInfo MPO;
765           Register StackAddr = Handler.getStackAddress(
766               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
767           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
768         } else {
769           // For outgoing byval arguments, insert the implicit copy byval
770           // implies, such that writes in the callee do not modify the caller's
771           // value.
772           uint64_t MemSize = Flags.getByValSize();
773           int64_t Offset = VA.getLocMemOffset();
774 
775           MachinePointerInfo DstMPO;
776           Register StackAddr =
777               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
778 
779           MachinePointerInfo SrcMPO(Args[i].OrigValue);
780           if (!Args[i].OrigValue) {
781             // We still need to accurately track the stack address space if we
782             // don't know the underlying value.
783             const LLT PtrTy = MRI.getType(StackAddr);
784             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
785           }
786 
787           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
788                                     inferAlignFromPtrInfo(MF, DstMPO));
789 
790           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
791                                     inferAlignFromPtrInfo(MF, SrcMPO));
792 
793           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
794                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
795                                      MemSize, VA);
796         }
797         continue;
798       }
799 
800       assert(!VA.needsCustom() && "custom loc should have been handled already");
801 
802       if (i == 0 && !ThisReturnRegs.empty() &&
803           Handler.isIncomingArgumentHandler() &&
804           isTypeIsValidForThisReturn(ValVT)) {
805         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
806         continue;
807       }
808 
809       if (Handler.isIncomingArgumentHandler())
810         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
811       else {
812         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
813           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
814         });
815       }
816     }
817 
818     // Now that all pieces have been assigned, re-pack the register typed values
819     // into the original value typed registers.
820     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
821       // Merge the split registers into the expected larger result vregs of
822       // the original call.
823       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
824                         LocTy, Args[i].Flags[0]);
825     }
826 
827     j += NumParts - 1;
828   }
829   for (auto &Fn : DelayedOutgoingRegAssignments)
830     Fn();
831 
832   return true;
833 }
834 
835 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
836                                    ArrayRef<Register> VRegs, Register DemoteReg,
837                                    int FI) const {
838   MachineFunction &MF = MIRBuilder.getMF();
839   MachineRegisterInfo &MRI = MF.getRegInfo();
840   const DataLayout &DL = MF.getDataLayout();
841 
842   SmallVector<EVT, 4> SplitVTs;
843   SmallVector<uint64_t, 4> Offsets;
844   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
845 
846   assert(VRegs.size() == SplitVTs.size());
847 
848   unsigned NumValues = SplitVTs.size();
849   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
850   Type *RetPtrTy =
851       PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());
852   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);
853 
854   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
855 
856   for (unsigned I = 0; I < NumValues; ++I) {
857     Register Addr;
858     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
859     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
860                                         MRI.getType(VRegs[I]),
861                                         commonAlignment(BaseAlign, Offsets[I]));
862     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
863   }
864 }
865 
866 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
867                                     ArrayRef<Register> VRegs,
868                                     Register DemoteReg) const {
869   MachineFunction &MF = MIRBuilder.getMF();
870   MachineRegisterInfo &MRI = MF.getRegInfo();
871   const DataLayout &DL = MF.getDataLayout();
872 
873   SmallVector<EVT, 4> SplitVTs;
874   SmallVector<uint64_t, 4> Offsets;
875   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
876 
877   assert(VRegs.size() == SplitVTs.size());
878 
879   unsigned NumValues = SplitVTs.size();
880   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
881   unsigned AS = DL.getAllocaAddrSpace();
882   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL);
883 
884   MachinePointerInfo PtrInfo(AS);
885 
886   for (unsigned I = 0; I < NumValues; ++I) {
887     Register Addr;
888     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
889     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
890                                         MRI.getType(VRegs[I]),
891                                         commonAlignment(BaseAlign, Offsets[I]));
892     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
893   }
894 }
895 
896 void CallLowering::insertSRetIncomingArgument(
897     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
898     MachineRegisterInfo &MRI, const DataLayout &DL) const {
899   unsigned AS = DL.getAllocaAddrSpace();
900   DemoteReg = MRI.createGenericVirtualRegister(
901       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
902 
903   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
904 
905   SmallVector<EVT, 1> ValueVTs;
906   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
907 
908   // NOTE: Assume that a pointer won't get split into more than one VT.
909   assert(ValueVTs.size() == 1);
910 
911   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
912                     ArgInfo::NoArgIndex);
913   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
914   DemoteArg.Flags[0].setSRet();
915   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
916 }
917 
918 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
919                                               const CallBase &CB,
920                                               CallLoweringInfo &Info) const {
921   const DataLayout &DL = MIRBuilder.getDataLayout();
922   Type *RetTy = CB.getType();
923   unsigned AS = DL.getAllocaAddrSpace();
924   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
925 
926   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
927       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
928 
929   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
930   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS),
931                     ArgInfo::NoArgIndex);
932   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
933   DemoteArg.Flags[0].setSRet();
934 
935   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
936   Info.DemoteStackIndex = FI;
937   Info.DemoteRegister = DemoteReg;
938 }
939 
940 bool CallLowering::checkReturn(CCState &CCInfo,
941                                SmallVectorImpl<BaseArgInfo> &Outs,
942                                CCAssignFn *Fn) const {
943   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
944     MVT VT = MVT::getVT(Outs[I].Ty);
945     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
946       return false;
947   }
948   return true;
949 }
950 
951 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
952                                  AttributeList Attrs,
953                                  SmallVectorImpl<BaseArgInfo> &Outs,
954                                  const DataLayout &DL) const {
955   LLVMContext &Context = RetTy->getContext();
956   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
957 
958   SmallVector<EVT, 4> SplitVTs;
959   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
960   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
961 
962   for (EVT VT : SplitVTs) {
963     unsigned NumParts =
964         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
965     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
966     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
967 
968     for (unsigned I = 0; I < NumParts; ++I) {
969       Outs.emplace_back(PartTy, Flags);
970     }
971   }
972 }
973 
974 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
975   const auto &F = MF.getFunction();
976   Type *ReturnType = F.getReturnType();
977   CallingConv::ID CallConv = F.getCallingConv();
978 
979   SmallVector<BaseArgInfo, 4> SplitArgs;
980   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
981                 MF.getDataLayout());
982   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
983 }
984 
985 bool CallLowering::parametersInCSRMatch(
986     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
987     const SmallVectorImpl<CCValAssign> &OutLocs,
988     const SmallVectorImpl<ArgInfo> &OutArgs) const {
989   for (unsigned i = 0; i < OutLocs.size(); ++i) {
990     const auto &ArgLoc = OutLocs[i];
991     // If it's not a register, it's fine.
992     if (!ArgLoc.isRegLoc())
993       continue;
994 
995     MCRegister PhysReg = ArgLoc.getLocReg();
996 
997     // Only look at callee-saved registers.
998     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
999       continue;
1000 
1001     LLVM_DEBUG(
1002         dbgs()
1003         << "... Call has an argument passed in a callee-saved register.\n");
1004 
1005     // Check if it was copied from.
1006     const ArgInfo &OutInfo = OutArgs[i];
1007 
1008     if (OutInfo.Regs.size() > 1) {
1009       LLVM_DEBUG(
1010           dbgs() << "... Cannot handle arguments in multiple registers.\n");
1011       return false;
1012     }
1013 
1014     // Check if we copy the register, walking through copies from virtual
1015     // registers. Note that getDefIgnoringCopies does not ignore copies from
1016     // physical registers.
1017     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
1018     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1019       LLVM_DEBUG(
1020           dbgs()
1021           << "... Parameter was not copied into a VReg, cannot tail call.\n");
1022       return false;
1023     }
1024 
1025     // Got a copy. Verify that it's the same as the register we want.
1026     Register CopyRHS = RegDef->getOperand(1).getReg();
1027     if (CopyRHS != PhysReg) {
1028       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1029                            "VReg, cannot tail call.\n");
1030       return false;
1031     }
1032   }
1033 
1034   return true;
1035 }
1036 
1037 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1038                                      MachineFunction &MF,
1039                                      SmallVectorImpl<ArgInfo> &InArgs,
1040                                      ValueAssigner &CalleeAssigner,
1041                                      ValueAssigner &CallerAssigner) const {
1042   const Function &F = MF.getFunction();
1043   CallingConv::ID CalleeCC = Info.CallConv;
1044   CallingConv::ID CallerCC = F.getCallingConv();
1045 
1046   if (CallerCC == CalleeCC)
1047     return true;
1048 
1049   SmallVector<CCValAssign, 16> ArgLocs1;
1050   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1051   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1052     return false;
1053 
1054   SmallVector<CCValAssign, 16> ArgLocs2;
1055   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1056   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1057     return false;
1058 
1059   // We need the argument locations to match up exactly. If there's more in
1060   // one than the other, then we are done.
1061   if (ArgLocs1.size() != ArgLocs2.size())
1062     return false;
1063 
1064   // Make sure that each location is passed in exactly the same way.
1065   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1066     const CCValAssign &Loc1 = ArgLocs1[i];
1067     const CCValAssign &Loc2 = ArgLocs2[i];
1068 
1069     // We need both of them to be the same. So if one is a register and one
1070     // isn't, we're done.
1071     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1072       return false;
1073 
1074     if (Loc1.isRegLoc()) {
1075       // If they don't have the same register location, we're done.
1076       if (Loc1.getLocReg() != Loc2.getLocReg())
1077         return false;
1078 
1079       // They matched, so we can move to the next ArgLoc.
1080       continue;
1081     }
1082 
1083     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1084     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1085       return false;
1086   }
1087 
1088   return true;
1089 }
1090 
1091 LLT CallLowering::ValueHandler::getStackValueStoreType(
1092     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1093   const MVT ValVT = VA.getValVT();
1094   if (ValVT != MVT::iPTR) {
1095     LLT ValTy(ValVT);
1096 
1097     // We lost the pointeriness going through CCValAssign, so try to restore it
1098     // based on the flags.
1099     if (Flags.isPointer()) {
1100       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1101                                ValTy.getScalarSizeInBits());
1102       if (ValVT.isVector())
1103         return LLT::vector(ValTy.getElementCount(), PtrTy);
1104       return PtrTy;
1105     }
1106 
1107     return ValTy;
1108   }
1109 
1110   unsigned AddrSpace = Flags.getPointerAddrSpace();
1111   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1112 }
1113 
1114 void CallLowering::ValueHandler::copyArgumentMemory(
1115     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1116     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1117     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1118     CCValAssign &VA) const {
1119   MachineFunction &MF = MIRBuilder.getMF();
1120   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1121       SrcPtrInfo,
1122       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1123       SrcAlign);
1124 
1125   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1126       DstPtrInfo,
1127       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1128       MemSize, DstAlign);
1129 
1130   const LLT PtrTy = MRI.getType(DstPtr);
1131   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1132 
1133   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1134   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1135 }
1136 
1137 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1138                                                     const CCValAssign &VA,
1139                                                     unsigned MaxSizeBits) {
1140   LLT LocTy{VA.getLocVT()};
1141   LLT ValTy{VA.getValVT()};
1142 
1143   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1144     return ValReg;
1145 
1146   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1147     if (MaxSizeBits <= ValTy.getSizeInBits())
1148       return ValReg;
1149     LocTy = LLT::scalar(MaxSizeBits);
1150   }
1151 
1152   const LLT ValRegTy = MRI.getType(ValReg);
1153   if (ValRegTy.isPointer()) {
1154     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1155     // we have to cast to do the extension.
1156     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1157     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1158   }
1159 
1160   switch (VA.getLocInfo()) {
1161   default: break;
1162   case CCValAssign::Full:
1163   case CCValAssign::BCvt:
1164     // FIXME: bitconverting between vector types may or may not be a
1165     // nop in big-endian situations.
1166     return ValReg;
1167   case CCValAssign::AExt: {
1168     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1169     return MIB.getReg(0);
1170   }
1171   case CCValAssign::SExt: {
1172     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1173     MIRBuilder.buildSExt(NewReg, ValReg);
1174     return NewReg;
1175   }
1176   case CCValAssign::ZExt: {
1177     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1178     MIRBuilder.buildZExt(NewReg, ValReg);
1179     return NewReg;
1180   }
1181   }
1182   llvm_unreachable("unable to extend register");
1183 }
1184 
1185 void CallLowering::ValueAssigner::anchor() {}
1186 
1187 Register CallLowering::IncomingValueHandler::buildExtensionHint(
1188     const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1189   switch (VA.getLocInfo()) {
1190   case CCValAssign::LocInfo::ZExt: {
1191     return MIRBuilder
1192         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1193                          NarrowTy.getScalarSizeInBits())
1194         .getReg(0);
1195   }
1196   case CCValAssign::LocInfo::SExt: {
1197     return MIRBuilder
1198         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1199                          NarrowTy.getScalarSizeInBits())
1200         .getReg(0);
1201     break;
1202   }
1203   default:
1204     return SrcReg;
1205   }
1206 }
1207 
1208 /// Check if we can use a basic COPY instruction between the two types.
1209 ///
1210 /// We're currently building on top of the infrastructure using MVT, which loses
1211 /// pointer information in the CCValAssign. We accept copies from physical
1212 /// registers that have been reported as integers if it's to an equivalent sized
1213 /// pointer LLT.
1214 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1215   if (SrcTy == DstTy)
1216     return true;
1217 
1218   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1219     return false;
1220 
1221   SrcTy = SrcTy.getScalarType();
1222   DstTy = DstTy.getScalarType();
1223 
1224   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1225          (DstTy.isPointer() && SrcTy.isScalar());
1226 }
1227 
1228 void CallLowering::IncomingValueHandler::assignValueToReg(
1229     Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1230   const MVT LocVT = VA.getLocVT();
1231   const LLT LocTy(LocVT);
1232   const LLT RegTy = MRI.getType(ValVReg);
1233 
1234   if (isCopyCompatibleType(RegTy, LocTy)) {
1235     MIRBuilder.buildCopy(ValVReg, PhysReg);
1236     return;
1237   }
1238 
1239   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1240   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1241   MIRBuilder.buildTrunc(ValVReg, Hint);
1242 }
1243