xref: /llvm-project/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp (revision cc548ec47c05971a67ed9ec7086d414aabcf8b05)
1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
15 #include "llvm/CodeGen/Analysis.h"
16 #include "llvm/CodeGen/CallingConvLower.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/GlobalISel/Utils.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/TargetLowering.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Target/TargetMachine.h"
28 
29 #define DEBUG_TYPE "call-lowering"
30 
31 using namespace llvm;
32 
33 void CallLowering::anchor() {}
34 
35 /// Helper function which updates \p Flags when \p AttrFn returns true.
36 static void
37 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
38                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
39   if (AttrFn(Attribute::SExt))
40     Flags.setSExt();
41   if (AttrFn(Attribute::ZExt))
42     Flags.setZExt();
43   if (AttrFn(Attribute::InReg))
44     Flags.setInReg();
45   if (AttrFn(Attribute::StructRet))
46     Flags.setSRet();
47   if (AttrFn(Attribute::Nest))
48     Flags.setNest();
49   if (AttrFn(Attribute::ByVal))
50     Flags.setByVal();
51   if (AttrFn(Attribute::Preallocated))
52     Flags.setPreallocated();
53   if (AttrFn(Attribute::InAlloca))
54     Flags.setInAlloca();
55   if (AttrFn(Attribute::Returned))
56     Flags.setReturned();
57   if (AttrFn(Attribute::SwiftSelf))
58     Flags.setSwiftSelf();
59   if (AttrFn(Attribute::SwiftAsync))
60     Flags.setSwiftAsync();
61   if (AttrFn(Attribute::SwiftError))
62     Flags.setSwiftError();
63 }
64 
65 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
66                                                      unsigned ArgIdx) const {
67   ISD::ArgFlagsTy Flags;
68   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
69     return Call.paramHasAttr(ArgIdx, Attr);
70   });
71   return Flags;
72 }
73 
74 ISD::ArgFlagsTy
75 CallLowering::getAttributesForReturn(const CallBase &Call) const {
76   ISD::ArgFlagsTy Flags;
77   addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {
78     return Call.hasRetAttr(Attr);
79   });
80   return Flags;
81 }
82 
83 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
84                                              const AttributeList &Attrs,
85                                              unsigned OpIdx) const {
86   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
87     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
88   });
89 }
90 
91 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
92                              ArrayRef<Register> ResRegs,
93                              ArrayRef<ArrayRef<Register>> ArgRegs,
94                              Register SwiftErrorVReg,
95                              std::optional<PtrAuthInfo> PAI,
96                              Register ConvergenceCtrlToken,
97                              std::function<unsigned()> GetCalleeReg) const {
98   CallLoweringInfo Info;
99   const DataLayout &DL = MIRBuilder.getDataLayout();
100   MachineFunction &MF = MIRBuilder.getMF();
101   MachineRegisterInfo &MRI = MF.getRegInfo();
102   bool CanBeTailCalled = CB.isTailCall() &&
103                          isInTailCallPosition(CB, MF.getTarget()) &&
104                          (MF.getFunction()
105                               .getFnAttribute("disable-tail-calls")
106                               .getValueAsString() != "true");
107 
108   CallingConv::ID CallConv = CB.getCallingConv();
109   Type *RetTy = CB.getType();
110   bool IsVarArg = CB.getFunctionType()->isVarArg();
111 
112   SmallVector<BaseArgInfo, 4> SplitArgs;
113   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
114   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
115 
116   Info.IsConvergent = CB.isConvergent();
117 
118   if (!Info.CanLowerReturn) {
119     // Callee requires sret demotion.
120     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
121 
122     // The sret demotion isn't compatible with tail-calls, since the sret
123     // argument points into the caller's stack frame.
124     CanBeTailCalled = false;
125   }
126 
127   // First step is to marshall all the function's parameters into the correct
128   // physregs and memory locations. Gather the sequence of argument types that
129   // we'll pass to the assigner function.
130   unsigned i = 0;
131   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
132   for (const auto &Arg : CB.args()) {
133     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
134                     i < NumFixedArgs};
135     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
136 
137     // If we have an explicit sret argument that is an Instruction, (i.e., it
138     // might point to function-local memory), we can't meaningfully tail-call.
139     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
140       CanBeTailCalled = false;
141 
142     Info.OrigArgs.push_back(OrigArg);
143     ++i;
144   }
145 
146   // Try looking through a bitcast from one function type to another.
147   // Commonly happens with calls to objc_msgSend().
148   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
149   if (const Function *F = dyn_cast<Function>(CalleeV)) {
150     if (F->hasFnAttribute(Attribute::NonLazyBind)) {
151       LLT Ty = getLLTForType(*F->getType(), DL);
152       Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0);
153       Info.Callee = MachineOperand::CreateReg(Reg, false);
154     } else {
155       Info.Callee = MachineOperand::CreateGA(F, 0);
156     }
157   } else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) {
158     // IR IFuncs and Aliases can't be forward declared (only defined), so the
159     // callee must be in the same TU and therefore we can direct-call it without
160     // worrying about it being out of range.
161     Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0);
162   } else
163     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
164 
165   Register ReturnHintAlignReg;
166   Align ReturnHintAlign;
167 
168   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};
169 
170   if (!Info.OrigRet.Ty->isVoidTy()) {
171     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
172 
173     if (MaybeAlign Alignment = CB.getRetAlign()) {
174       if (*Alignment > Align(1)) {
175         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
176         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
177         ReturnHintAlign = *Alignment;
178       }
179     }
180   }
181 
182   auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);
183   if (Bundle && CB.isIndirectCall()) {
184     Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
185     assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
186   }
187 
188   Info.CB = &CB;
189   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
190   Info.CallConv = CallConv;
191   Info.SwiftErrorVReg = SwiftErrorVReg;
192   Info.PAI = PAI;
193   Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
194   Info.IsMustTailCall = CB.isMustTailCall();
195   Info.IsTailCall = CanBeTailCalled;
196   Info.IsVarArg = IsVarArg;
197   if (!lowerCall(MIRBuilder, Info))
198     return false;
199 
200   if (ReturnHintAlignReg && !Info.LoweredTailCall) {
201     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
202                                 ReturnHintAlign);
203   }
204 
205   return true;
206 }
207 
208 template <typename FuncInfoTy>
209 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
210                                const DataLayout &DL,
211                                const FuncInfoTy &FuncInfo) const {
212   auto &Flags = Arg.Flags[0];
213   const AttributeList &Attrs = FuncInfo.getAttributes();
214   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
215 
216   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
217   if (PtrTy) {
218     Flags.setPointer();
219     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
220   }
221 
222   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
223   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
224     assert(OpIdx >= AttributeList::FirstArgIndex);
225     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
226 
227     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
228     if (!ElementTy)
229       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
230     if (!ElementTy)
231       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
232     assert(ElementTy && "Must have byval, inalloca or preallocated type");
233     Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
234 
235     // For ByVal, alignment should be passed from FE.  BE will guess if
236     // this info is not there but there are cases it cannot get right.
237     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
238       MemAlign = *ParamAlign;
239     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
240       MemAlign = *ParamAlign;
241     else
242       MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
243   } else if (OpIdx >= AttributeList::FirstArgIndex) {
244     if (auto ParamAlign =
245             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
246       MemAlign = *ParamAlign;
247   }
248   Flags.setMemAlign(MemAlign);
249   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
250 
251   // Don't try to use the returned attribute if the argument is marked as
252   // swiftself, since it won't be passed in x0.
253   if (Flags.isSwiftSelf())
254     Flags.setReturned(false);
255 }
256 
257 template void
258 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
259                                     const DataLayout &DL,
260                                     const Function &FuncInfo) const;
261 
262 template void
263 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
264                                     const DataLayout &DL,
265                                     const CallBase &FuncInfo) const;
266 
267 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
268                                      SmallVectorImpl<ArgInfo> &SplitArgs,
269                                      const DataLayout &DL,
270                                      CallingConv::ID CallConv,
271                                      SmallVectorImpl<uint64_t> *Offsets) const {
272   LLVMContext &Ctx = OrigArg.Ty->getContext();
273 
274   SmallVector<EVT, 4> SplitVTs;
275   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
276 
277   if (SplitVTs.size() == 0)
278     return;
279 
280   if (SplitVTs.size() == 1) {
281     // No splitting to do, but we want to replace the original type (e.g. [1 x
282     // double] -> double).
283     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
284                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
285                            OrigArg.IsFixed, OrigArg.OrigValue);
286     return;
287   }
288 
289   // Create one ArgInfo for each virtual register in the original ArgInfo.
290   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
291 
292   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
293       OrigArg.Ty, CallConv, false, DL);
294   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
295     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
296     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
297                            OrigArg.Flags[0], OrigArg.IsFixed);
298     if (NeedsRegBlock)
299       SplitArgs.back().Flags[0].setInConsecutiveRegs();
300   }
301 
302   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
303 }
304 
305 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
306 static MachineInstrBuilder
307 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
308                             ArrayRef<Register> SrcRegs) {
309   MachineRegisterInfo &MRI = *B.getMRI();
310   LLT LLTy = MRI.getType(DstRegs[0]);
311   LLT PartLLT = MRI.getType(SrcRegs[0]);
312 
313   // Deal with v3s16 split into v2s16
314   LLT LCMTy = getCoverTy(LLTy, PartLLT);
315   if (LCMTy == LLTy) {
316     // Common case where no padding is needed.
317     assert(DstRegs.size() == 1);
318     return B.buildConcatVectors(DstRegs[0], SrcRegs);
319   }
320 
321   // We need to create an unmerge to the result registers, which may require
322   // widening the original value.
323   Register UnmergeSrcReg;
324   if (LCMTy != PartLLT) {
325     assert(DstRegs.size() == 1);
326     return B.buildDeleteTrailingVectorElements(
327         DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));
328   } else {
329     // We don't need to widen anything if we're extracting a scalar which was
330     // promoted to a vector e.g. s8 -> v4s8 -> s8
331     assert(SrcRegs.size() == 1);
332     UnmergeSrcReg = SrcRegs[0];
333   }
334 
335   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
336 
337   SmallVector<Register, 8> PadDstRegs(NumDst);
338   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
339 
340   // Create the excess dead defs for the unmerge.
341   for (int I = DstRegs.size(); I != NumDst; ++I)
342     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
343 
344   if (PadDstRegs.size() == 1)
345     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
346   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
347 }
348 
349 /// Create a sequence of instructions to combine pieces split into register
350 /// typed values to the original IR value. \p OrigRegs contains the destination
351 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
352 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
353 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
354                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
355                               const ISD::ArgFlagsTy Flags) {
356   MachineRegisterInfo &MRI = *B.getMRI();
357 
358   if (PartLLT == LLTy) {
359     // We should have avoided introducing a new virtual register, and just
360     // directly assigned here.
361     assert(OrigRegs[0] == Regs[0]);
362     return;
363   }
364 
365   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
366       Regs.size() == 1) {
367     B.buildBitcast(OrigRegs[0], Regs[0]);
368     return;
369   }
370 
371   // A vector PartLLT needs extending to LLTy's element size.
372   // E.g. <2 x s64> = G_SEXT <2 x s32>.
373   if (PartLLT.isVector() == LLTy.isVector() &&
374       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
375       (!PartLLT.isVector() ||
376        PartLLT.getElementCount() == LLTy.getElementCount()) &&
377       OrigRegs.size() == 1 && Regs.size() == 1) {
378     Register SrcReg = Regs[0];
379 
380     LLT LocTy = MRI.getType(SrcReg);
381 
382     if (Flags.isSExt()) {
383       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
384                    .getReg(0);
385     } else if (Flags.isZExt()) {
386       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
387                    .getReg(0);
388     }
389 
390     // Sometimes pointers are passed zero extended.
391     LLT OrigTy = MRI.getType(OrigRegs[0]);
392     if (OrigTy.isPointer()) {
393       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
394       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
395       return;
396     }
397 
398     B.buildTrunc(OrigRegs[0], SrcReg);
399     return;
400   }
401 
402   if (!LLTy.isVector() && !PartLLT.isVector()) {
403     assert(OrigRegs.size() == 1);
404     LLT OrigTy = MRI.getType(OrigRegs[0]);
405 
406     unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
407     if (SrcSize == OrigTy.getSizeInBits())
408       B.buildMergeValues(OrigRegs[0], Regs);
409     else {
410       auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);
411       B.buildTrunc(OrigRegs[0], Widened);
412     }
413 
414     return;
415   }
416 
417   if (PartLLT.isVector()) {
418     assert(OrigRegs.size() == 1);
419     SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
420 
421     // If PartLLT is a mismatched vector in both number of elements and element
422     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
423     // have the same elt type, i.e. v4s32.
424     // TODO: Extend this coersion to element multiples other than just 2.
425     if (TypeSize::isKnownGT(PartLLT.getSizeInBits(), LLTy.getSizeInBits()) &&
426         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
427         Regs.size() == 1) {
428       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
429                       .changeElementCount(PartLLT.getElementCount() * 2);
430       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
431       PartLLT = NewTy;
432     }
433 
434     if (LLTy.getScalarType() == PartLLT.getElementType()) {
435       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
436     } else {
437       unsigned I = 0;
438       LLT GCDTy = getGCDType(LLTy, PartLLT);
439 
440       // We are both splitting a vector, and bitcasting its element types. Cast
441       // the source pieces into the appropriate number of pieces with the result
442       // element type.
443       for (Register SrcReg : CastRegs)
444         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
445       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
446     }
447 
448     return;
449   }
450 
451   assert(LLTy.isVector() && !PartLLT.isVector());
452 
453   LLT DstEltTy = LLTy.getElementType();
454 
455   // Pointer information was discarded. We'll need to coerce some register types
456   // to avoid violating type constraints.
457   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
458 
459   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
460 
461   if (DstEltTy == PartLLT) {
462     // Vector was trivially scalarized.
463 
464     if (RealDstEltTy.isPointer()) {
465       for (Register Reg : Regs)
466         MRI.setType(Reg, RealDstEltTy);
467     }
468 
469     B.buildBuildVector(OrigRegs[0], Regs);
470   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
471     // Deal with vector with 64-bit elements decomposed to 32-bit
472     // registers. Need to create intermediate 64-bit elements.
473     SmallVector<Register, 8> EltMerges;
474     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
475 
476     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
477 
478     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
479       auto Merge =
480           B.buildMergeLikeInstr(RealDstEltTy, Regs.take_front(PartsPerElt));
481       // Fix the type in case this is really a vector of pointers.
482       MRI.setType(Merge.getReg(0), RealDstEltTy);
483       EltMerges.push_back(Merge.getReg(0));
484       Regs = Regs.drop_front(PartsPerElt);
485     }
486 
487     B.buildBuildVector(OrigRegs[0], EltMerges);
488   } else {
489     // Vector was split, and elements promoted to a wider type.
490     // FIXME: Should handle floating point promotions.
491     unsigned NumElts = LLTy.getNumElements();
492     LLT BVType = LLT::fixed_vector(NumElts, PartLLT);
493 
494     Register BuildVec;
495     if (NumElts == Regs.size())
496       BuildVec = B.buildBuildVector(BVType, Regs).getReg(0);
497     else {
498       // Vector elements are packed in the inputs.
499       // e.g. we have a <4 x s16> but 2 x s32 in regs.
500       assert(NumElts > Regs.size());
501       LLT SrcEltTy = MRI.getType(Regs[0]);
502 
503       LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType();
504 
505       // Input registers contain packed elements.
506       // Determine how many elements per reg.
507       assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
508       unsigned EltPerReg =
509           (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
510 
511       SmallVector<Register, 0> BVRegs;
512       BVRegs.reserve(Regs.size() * EltPerReg);
513       for (Register R : Regs) {
514         auto Unmerge = B.buildUnmerge(OriginalEltTy, R);
515         for (unsigned K = 0; K < EltPerReg; ++K)
516           BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0));
517       }
518 
519       // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
520       // for a <3 x s16> vector. We should have less than EltPerReg extra items.
521       if (BVRegs.size() > NumElts) {
522         assert((BVRegs.size() - NumElts) < EltPerReg);
523         BVRegs.truncate(NumElts);
524       }
525       BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0);
526     }
527     B.buildTrunc(OrigRegs[0], BuildVec);
528   }
529 }
530 
531 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
532 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
533 /// contain the type of scalar value extension if necessary.
534 ///
535 /// This is used for outgoing values (vregs to physregs)
536 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
537                             Register SrcReg, LLT SrcTy, LLT PartTy,
538                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
539   // We could just insert a regular copy, but this is unreachable at the moment.
540   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
541 
542   const TypeSize PartSize = PartTy.getSizeInBits();
543 
544   if (PartTy.isVector() == SrcTy.isVector() &&
545       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
546     assert(DstRegs.size() == 1);
547     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
548     return;
549   }
550 
551   if (SrcTy.isVector() && !PartTy.isVector() &&
552       TypeSize::isKnownGT(PartSize, SrcTy.getElementType().getSizeInBits())) {
553     // Vector was scalarized, and the elements extended.
554     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
555     for (int i = 0, e = DstRegs.size(); i != e; ++i)
556       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
557     return;
558   }
559 
560   if (SrcTy.isVector() && PartTy.isVector() &&
561       PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
562       ElementCount::isKnownLT(SrcTy.getElementCount(),
563                               PartTy.getElementCount())) {
564     // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
565     Register DstReg = DstRegs.front();
566     B.buildPadVectorWithUndefElements(DstReg, SrcReg);
567     return;
568   }
569 
570   LLT GCDTy = getGCDType(SrcTy, PartTy);
571   if (GCDTy == PartTy) {
572     // If this already evenly divisible, we can create a simple unmerge.
573     B.buildUnmerge(DstRegs, SrcReg);
574     return;
575   }
576 
577   MachineRegisterInfo &MRI = *B.getMRI();
578   LLT DstTy = MRI.getType(DstRegs[0]);
579   LLT LCMTy = getCoverTy(SrcTy, PartTy);
580 
581   if (PartTy.isVector() && LCMTy == PartTy) {
582     assert(DstRegs.size() == 1);
583     B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);
584     return;
585   }
586 
587   const unsigned DstSize = DstTy.getSizeInBits();
588   const unsigned SrcSize = SrcTy.getSizeInBits();
589   unsigned CoveringSize = LCMTy.getSizeInBits();
590 
591   Register UnmergeSrc = SrcReg;
592 
593   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
594     // For scalars, it's common to be able to use a simple extension.
595     if (SrcTy.isScalar() && DstTy.isScalar()) {
596       CoveringSize = alignTo(SrcSize, DstSize);
597       LLT CoverTy = LLT::scalar(CoveringSize);
598       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
599     } else {
600       // Widen to the common type.
601       // FIXME: This should respect the extend type
602       Register Undef = B.buildUndef(SrcTy).getReg(0);
603       SmallVector<Register, 8> MergeParts(1, SrcReg);
604       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
605         MergeParts.push_back(Undef);
606       UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);
607     }
608   }
609 
610   if (LCMTy.isVector() && CoveringSize != SrcSize)
611     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
612 
613   B.buildUnmerge(DstRegs, UnmergeSrc);
614 }
615 
616 bool CallLowering::determineAndHandleAssignments(
617     ValueHandler &Handler, ValueAssigner &Assigner,
618     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
619     CallingConv::ID CallConv, bool IsVarArg,
620     ArrayRef<Register> ThisReturnRegs) const {
621   MachineFunction &MF = MIRBuilder.getMF();
622   const Function &F = MF.getFunction();
623   SmallVector<CCValAssign, 16> ArgLocs;
624 
625   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
626   if (!determineAssignments(Assigner, Args, CCInfo))
627     return false;
628 
629   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
630                            ThisReturnRegs);
631 }
632 
633 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
634   if (Flags.isSExt())
635     return TargetOpcode::G_SEXT;
636   if (Flags.isZExt())
637     return TargetOpcode::G_ZEXT;
638   return TargetOpcode::G_ANYEXT;
639 }
640 
641 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
642                                         SmallVectorImpl<ArgInfo> &Args,
643                                         CCState &CCInfo) const {
644   LLVMContext &Ctx = CCInfo.getContext();
645   const CallingConv::ID CallConv = CCInfo.getCallingConv();
646 
647   unsigned NumArgs = Args.size();
648   for (unsigned i = 0; i != NumArgs; ++i) {
649     EVT CurVT = EVT::getEVT(Args[i].Ty);
650 
651     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
652 
653     // If we need to split the type over multiple regs, check it's a scenario
654     // we currently support.
655     unsigned NumParts =
656         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
657 
658     if (NumParts == 1) {
659       // Try to use the register type if we couldn't assign the VT.
660       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
661                              Args[i].Flags[0], CCInfo))
662         return false;
663       continue;
664     }
665 
666     // For incoming arguments (physregs to vregs), we could have values in
667     // physregs (or memlocs) which we want to extract and copy to vregs.
668     // During this, we might have to deal with the LLT being split across
669     // multiple regs, so we have to record this information for later.
670     //
671     // If we have outgoing args, then we have the opposite case. We have a
672     // vreg with an LLT which we want to assign to a physical location, and
673     // we might have to record that the value has to be split later.
674 
675     // We're handling an incoming arg which is split over multiple regs.
676     // E.g. passing an s128 on AArch64.
677     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
678     Args[i].Flags.clear();
679 
680     for (unsigned Part = 0; Part < NumParts; ++Part) {
681       ISD::ArgFlagsTy Flags = OrigFlags;
682       if (Part == 0) {
683         Flags.setSplit();
684       } else {
685         Flags.setOrigAlign(Align(1));
686         if (Part == NumParts - 1)
687           Flags.setSplitEnd();
688       }
689 
690       Args[i].Flags.push_back(Flags);
691       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
692                              Args[i].Flags[Part], CCInfo)) {
693         // Still couldn't assign this smaller part type for some reason.
694         return false;
695       }
696     }
697   }
698 
699   return true;
700 }
701 
702 bool CallLowering::handleAssignments(ValueHandler &Handler,
703                                      SmallVectorImpl<ArgInfo> &Args,
704                                      CCState &CCInfo,
705                                      SmallVectorImpl<CCValAssign> &ArgLocs,
706                                      MachineIRBuilder &MIRBuilder,
707                                      ArrayRef<Register> ThisReturnRegs) const {
708   MachineFunction &MF = MIRBuilder.getMF();
709   MachineRegisterInfo &MRI = MF.getRegInfo();
710   const Function &F = MF.getFunction();
711   const DataLayout &DL = F.getParent()->getDataLayout();
712 
713   const unsigned NumArgs = Args.size();
714 
715   // Stores thunks for outgoing register assignments. This is used so we delay
716   // generating register copies until mem loc assignments are done. We do this
717   // so that if the target is using the delayed stack protector feature, we can
718   // find the split point of the block accurately. E.g. if we have:
719   // G_STORE %val, %memloc
720   // $x0 = COPY %foo
721   // $x1 = COPY %bar
722   // CALL func
723   // ... then the split point for the block will correctly be at, and including,
724   // the copy to $x0. If instead the G_STORE instruction immediately precedes
725   // the CALL, then we'd prematurely choose the CALL as the split point, thus
726   // generating a split block with a CALL that uses undefined physregs.
727   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
728 
729   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
730     assert(j < ArgLocs.size() && "Skipped too many arg locs");
731     CCValAssign &VA = ArgLocs[j];
732     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
733 
734     if (VA.needsCustom()) {
735       std::function<void()> Thunk;
736       unsigned NumArgRegs = Handler.assignCustomValue(
737           Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);
738       if (Thunk)
739         DelayedOutgoingRegAssignments.emplace_back(Thunk);
740       if (!NumArgRegs)
741         return false;
742       j += (NumArgRegs - 1);
743       continue;
744     }
745 
746     const MVT ValVT = VA.getValVT();
747     const MVT LocVT = VA.getLocVT();
748 
749     const LLT LocTy(LocVT);
750     const LLT ValTy(ValVT);
751     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
752     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
753     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
754 
755     // Expected to be multiple regs for a single incoming arg.
756     // There should be Regs.size() ArgLocs per argument.
757     // This should be the same as getNumRegistersForCallingConv
758     const unsigned NumParts = Args[i].Flags.size();
759 
760     // Now split the registers into the assigned types.
761     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
762 
763     if (NumParts != 1 || NewLLT != OrigTy) {
764       // If we can't directly assign the register, we need one or more
765       // intermediate values.
766       Args[i].Regs.resize(NumParts);
767 
768       // For each split register, create and assign a vreg that will store
769       // the incoming component of the larger value. These will later be
770       // merged to form the final vreg.
771       for (unsigned Part = 0; Part < NumParts; ++Part)
772         Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
773     }
774 
775     assert((j + (NumParts - 1)) < ArgLocs.size() &&
776            "Too many regs for number of args");
777 
778     // Coerce into outgoing value types before register assignment.
779     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
780       assert(Args[i].OrigRegs.size() == 1);
781       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
782                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
783     }
784 
785     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
786     for (unsigned Part = 0; Part < NumParts; ++Part) {
787       Register ArgReg = Args[i].Regs[Part];
788       // There should be Regs.size() ArgLocs per argument.
789       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
790       CCValAssign &VA = ArgLocs[j + Idx];
791       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
792 
793       if (VA.isMemLoc() && !Flags.isByVal()) {
794         // Individual pieces may have been spilled to the stack and others
795         // passed in registers.
796 
797         // TODO: The memory size may be larger than the value we need to
798         // store. We may need to adjust the offset for big endian targets.
799         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
800 
801         MachinePointerInfo MPO;
802         Register StackAddr = Handler.getStackAddress(
803             MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags);
804 
805         Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA);
806         continue;
807       }
808 
809       if (VA.isMemLoc() && Flags.isByVal()) {
810         assert(Args[i].Regs.size() == 1 &&
811                "didn't expect split byval pointer");
812 
813         if (Handler.isIncomingArgumentHandler()) {
814           // We just need to copy the frame index value to the pointer.
815           MachinePointerInfo MPO;
816           Register StackAddr = Handler.getStackAddress(
817               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
818           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
819         } else {
820           // For outgoing byval arguments, insert the implicit copy byval
821           // implies, such that writes in the callee do not modify the caller's
822           // value.
823           uint64_t MemSize = Flags.getByValSize();
824           int64_t Offset = VA.getLocMemOffset();
825 
826           MachinePointerInfo DstMPO;
827           Register StackAddr =
828               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
829 
830           MachinePointerInfo SrcMPO(Args[i].OrigValue);
831           if (!Args[i].OrigValue) {
832             // We still need to accurately track the stack address space if we
833             // don't know the underlying value.
834             const LLT PtrTy = MRI.getType(StackAddr);
835             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
836           }
837 
838           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
839                                     inferAlignFromPtrInfo(MF, DstMPO));
840 
841           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
842                                     inferAlignFromPtrInfo(MF, SrcMPO));
843 
844           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
845                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
846                                      MemSize, VA);
847         }
848         continue;
849       }
850 
851       assert(!VA.needsCustom() && "custom loc should have been handled already");
852 
853       if (i == 0 && !ThisReturnRegs.empty() &&
854           Handler.isIncomingArgumentHandler() &&
855           isTypeIsValidForThisReturn(ValVT)) {
856         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
857         continue;
858       }
859 
860       if (Handler.isIncomingArgumentHandler())
861         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
862       else {
863         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
864           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
865         });
866       }
867     }
868 
869     // Now that all pieces have been assigned, re-pack the register typed values
870     // into the original value typed registers.
871     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
872       // Merge the split registers into the expected larger result vregs of
873       // the original call.
874       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
875                         LocTy, Args[i].Flags[0]);
876     }
877 
878     j += NumParts - 1;
879   }
880   for (auto &Fn : DelayedOutgoingRegAssignments)
881     Fn();
882 
883   return true;
884 }
885 
886 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
887                                    ArrayRef<Register> VRegs, Register DemoteReg,
888                                    int FI) const {
889   MachineFunction &MF = MIRBuilder.getMF();
890   MachineRegisterInfo &MRI = MF.getRegInfo();
891   const DataLayout &DL = MF.getDataLayout();
892 
893   SmallVector<EVT, 4> SplitVTs;
894   SmallVector<uint64_t, 4> Offsets;
895   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
896 
897   assert(VRegs.size() == SplitVTs.size());
898 
899   unsigned NumValues = SplitVTs.size();
900   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
901   Type *RetPtrTy =
902       PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());
903   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);
904 
905   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
906 
907   for (unsigned I = 0; I < NumValues; ++I) {
908     Register Addr;
909     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
910     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
911                                         MRI.getType(VRegs[I]),
912                                         commonAlignment(BaseAlign, Offsets[I]));
913     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
914   }
915 }
916 
917 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
918                                     ArrayRef<Register> VRegs,
919                                     Register DemoteReg) const {
920   MachineFunction &MF = MIRBuilder.getMF();
921   MachineRegisterInfo &MRI = MF.getRegInfo();
922   const DataLayout &DL = MF.getDataLayout();
923 
924   SmallVector<EVT, 4> SplitVTs;
925   SmallVector<uint64_t, 4> Offsets;
926   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
927 
928   assert(VRegs.size() == SplitVTs.size());
929 
930   unsigned NumValues = SplitVTs.size();
931   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
932   unsigned AS = DL.getAllocaAddrSpace();
933   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL);
934 
935   MachinePointerInfo PtrInfo(AS);
936 
937   for (unsigned I = 0; I < NumValues; ++I) {
938     Register Addr;
939     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
940     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
941                                         MRI.getType(VRegs[I]),
942                                         commonAlignment(BaseAlign, Offsets[I]));
943     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
944   }
945 }
946 
947 void CallLowering::insertSRetIncomingArgument(
948     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
949     MachineRegisterInfo &MRI, const DataLayout &DL) const {
950   unsigned AS = DL.getAllocaAddrSpace();
951   DemoteReg = MRI.createGenericVirtualRegister(
952       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
953 
954   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
955 
956   SmallVector<EVT, 1> ValueVTs;
957   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
958 
959   // NOTE: Assume that a pointer won't get split into more than one VT.
960   assert(ValueVTs.size() == 1);
961 
962   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
963                     ArgInfo::NoArgIndex);
964   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
965   DemoteArg.Flags[0].setSRet();
966   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
967 }
968 
969 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
970                                               const CallBase &CB,
971                                               CallLoweringInfo &Info) const {
972   const DataLayout &DL = MIRBuilder.getDataLayout();
973   Type *RetTy = CB.getType();
974   unsigned AS = DL.getAllocaAddrSpace();
975   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
976 
977   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
978       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
979 
980   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
981   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS),
982                     ArgInfo::NoArgIndex);
983   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
984   DemoteArg.Flags[0].setSRet();
985 
986   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
987   Info.DemoteStackIndex = FI;
988   Info.DemoteRegister = DemoteReg;
989 }
990 
991 bool CallLowering::checkReturn(CCState &CCInfo,
992                                SmallVectorImpl<BaseArgInfo> &Outs,
993                                CCAssignFn *Fn) const {
994   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
995     MVT VT = MVT::getVT(Outs[I].Ty);
996     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
997       return false;
998   }
999   return true;
1000 }
1001 
1002 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1003                                  AttributeList Attrs,
1004                                  SmallVectorImpl<BaseArgInfo> &Outs,
1005                                  const DataLayout &DL) const {
1006   LLVMContext &Context = RetTy->getContext();
1007   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1008 
1009   SmallVector<EVT, 4> SplitVTs;
1010   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
1011   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
1012 
1013   for (EVT VT : SplitVTs) {
1014     unsigned NumParts =
1015         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
1016     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
1017     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1018 
1019     for (unsigned I = 0; I < NumParts; ++I) {
1020       Outs.emplace_back(PartTy, Flags);
1021     }
1022   }
1023 }
1024 
1025 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1026   const auto &F = MF.getFunction();
1027   Type *ReturnType = F.getReturnType();
1028   CallingConv::ID CallConv = F.getCallingConv();
1029 
1030   SmallVector<BaseArgInfo, 4> SplitArgs;
1031   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
1032                 MF.getDataLayout());
1033   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
1034 }
1035 
1036 bool CallLowering::parametersInCSRMatch(
1037     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1038     const SmallVectorImpl<CCValAssign> &OutLocs,
1039     const SmallVectorImpl<ArgInfo> &OutArgs) const {
1040   for (unsigned i = 0; i < OutLocs.size(); ++i) {
1041     const auto &ArgLoc = OutLocs[i];
1042     // If it's not a register, it's fine.
1043     if (!ArgLoc.isRegLoc())
1044       continue;
1045 
1046     MCRegister PhysReg = ArgLoc.getLocReg();
1047 
1048     // Only look at callee-saved registers.
1049     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
1050       continue;
1051 
1052     LLVM_DEBUG(
1053         dbgs()
1054         << "... Call has an argument passed in a callee-saved register.\n");
1055 
1056     // Check if it was copied from.
1057     const ArgInfo &OutInfo = OutArgs[i];
1058 
1059     if (OutInfo.Regs.size() > 1) {
1060       LLVM_DEBUG(
1061           dbgs() << "... Cannot handle arguments in multiple registers.\n");
1062       return false;
1063     }
1064 
1065     // Check if we copy the register, walking through copies from virtual
1066     // registers. Note that getDefIgnoringCopies does not ignore copies from
1067     // physical registers.
1068     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
1069     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1070       LLVM_DEBUG(
1071           dbgs()
1072           << "... Parameter was not copied into a VReg, cannot tail call.\n");
1073       return false;
1074     }
1075 
1076     // Got a copy. Verify that it's the same as the register we want.
1077     Register CopyRHS = RegDef->getOperand(1).getReg();
1078     if (CopyRHS != PhysReg) {
1079       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1080                            "VReg, cannot tail call.\n");
1081       return false;
1082     }
1083   }
1084 
1085   return true;
1086 }
1087 
1088 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1089                                      MachineFunction &MF,
1090                                      SmallVectorImpl<ArgInfo> &InArgs,
1091                                      ValueAssigner &CalleeAssigner,
1092                                      ValueAssigner &CallerAssigner) const {
1093   const Function &F = MF.getFunction();
1094   CallingConv::ID CalleeCC = Info.CallConv;
1095   CallingConv::ID CallerCC = F.getCallingConv();
1096 
1097   if (CallerCC == CalleeCC)
1098     return true;
1099 
1100   SmallVector<CCValAssign, 16> ArgLocs1;
1101   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1102   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1103     return false;
1104 
1105   SmallVector<CCValAssign, 16> ArgLocs2;
1106   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1107   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1108     return false;
1109 
1110   // We need the argument locations to match up exactly. If there's more in
1111   // one than the other, then we are done.
1112   if (ArgLocs1.size() != ArgLocs2.size())
1113     return false;
1114 
1115   // Make sure that each location is passed in exactly the same way.
1116   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1117     const CCValAssign &Loc1 = ArgLocs1[i];
1118     const CCValAssign &Loc2 = ArgLocs2[i];
1119 
1120     // We need both of them to be the same. So if one is a register and one
1121     // isn't, we're done.
1122     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1123       return false;
1124 
1125     if (Loc1.isRegLoc()) {
1126       // If they don't have the same register location, we're done.
1127       if (Loc1.getLocReg() != Loc2.getLocReg())
1128         return false;
1129 
1130       // They matched, so we can move to the next ArgLoc.
1131       continue;
1132     }
1133 
1134     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1135     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1136       return false;
1137   }
1138 
1139   return true;
1140 }
1141 
1142 LLT CallLowering::ValueHandler::getStackValueStoreType(
1143     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1144   const MVT ValVT = VA.getValVT();
1145   if (ValVT != MVT::iPTR) {
1146     LLT ValTy(ValVT);
1147 
1148     // We lost the pointeriness going through CCValAssign, so try to restore it
1149     // based on the flags.
1150     if (Flags.isPointer()) {
1151       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1152                                ValTy.getScalarSizeInBits());
1153       if (ValVT.isVector())
1154         return LLT::vector(ValTy.getElementCount(), PtrTy);
1155       return PtrTy;
1156     }
1157 
1158     return ValTy;
1159   }
1160 
1161   unsigned AddrSpace = Flags.getPointerAddrSpace();
1162   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1163 }
1164 
1165 void CallLowering::ValueHandler::copyArgumentMemory(
1166     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1167     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1168     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1169     CCValAssign &VA) const {
1170   MachineFunction &MF = MIRBuilder.getMF();
1171   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1172       SrcPtrInfo,
1173       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1174       SrcAlign);
1175 
1176   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1177       DstPtrInfo,
1178       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1179       MemSize, DstAlign);
1180 
1181   const LLT PtrTy = MRI.getType(DstPtr);
1182   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1183 
1184   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1185   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1186 }
1187 
1188 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1189                                                     const CCValAssign &VA,
1190                                                     unsigned MaxSizeBits) {
1191   LLT LocTy{VA.getLocVT()};
1192   LLT ValTy{VA.getValVT()};
1193 
1194   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1195     return ValReg;
1196 
1197   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1198     if (MaxSizeBits <= ValTy.getSizeInBits())
1199       return ValReg;
1200     LocTy = LLT::scalar(MaxSizeBits);
1201   }
1202 
1203   const LLT ValRegTy = MRI.getType(ValReg);
1204   if (ValRegTy.isPointer()) {
1205     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1206     // we have to cast to do the extension.
1207     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1208     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1209   }
1210 
1211   switch (VA.getLocInfo()) {
1212   default: break;
1213   case CCValAssign::Full:
1214   case CCValAssign::BCvt:
1215     // FIXME: bitconverting between vector types may or may not be a
1216     // nop in big-endian situations.
1217     return ValReg;
1218   case CCValAssign::AExt: {
1219     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1220     return MIB.getReg(0);
1221   }
1222   case CCValAssign::SExt: {
1223     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1224     MIRBuilder.buildSExt(NewReg, ValReg);
1225     return NewReg;
1226   }
1227   case CCValAssign::ZExt: {
1228     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1229     MIRBuilder.buildZExt(NewReg, ValReg);
1230     return NewReg;
1231   }
1232   }
1233   llvm_unreachable("unable to extend register");
1234 }
1235 
1236 void CallLowering::ValueAssigner::anchor() {}
1237 
1238 Register CallLowering::IncomingValueHandler::buildExtensionHint(
1239     const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1240   switch (VA.getLocInfo()) {
1241   case CCValAssign::LocInfo::ZExt: {
1242     return MIRBuilder
1243         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1244                          NarrowTy.getScalarSizeInBits())
1245         .getReg(0);
1246   }
1247   case CCValAssign::LocInfo::SExt: {
1248     return MIRBuilder
1249         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1250                          NarrowTy.getScalarSizeInBits())
1251         .getReg(0);
1252     break;
1253   }
1254   default:
1255     return SrcReg;
1256   }
1257 }
1258 
1259 /// Check if we can use a basic COPY instruction between the two types.
1260 ///
1261 /// We're currently building on top of the infrastructure using MVT, which loses
1262 /// pointer information in the CCValAssign. We accept copies from physical
1263 /// registers that have been reported as integers if it's to an equivalent sized
1264 /// pointer LLT.
1265 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1266   if (SrcTy == DstTy)
1267     return true;
1268 
1269   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1270     return false;
1271 
1272   SrcTy = SrcTy.getScalarType();
1273   DstTy = DstTy.getScalarType();
1274 
1275   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1276          (DstTy.isPointer() && SrcTy.isScalar());
1277 }
1278 
1279 void CallLowering::IncomingValueHandler::assignValueToReg(
1280     Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1281   const MVT LocVT = VA.getLocVT();
1282   const LLT LocTy(LocVT);
1283   const LLT RegTy = MRI.getType(ValVReg);
1284 
1285   if (isCopyCompatibleType(RegTy, LocTy)) {
1286     MIRBuilder.buildCopy(ValVReg, PhysReg);
1287     return;
1288   }
1289 
1290   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1291   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1292   MIRBuilder.buildTrunc(ValVReg, Hint);
1293 }
1294