xref: /llvm-project/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp (revision cf5ecd5644ce2f35aad5510d6ce245a9cfe2505f)
1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/Analysis.h"
15 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
16 #include "llvm/CodeGen/GlobalISel/Utils.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/MachineOperand.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/Target/TargetMachine.h"
26 
27 #define DEBUG_TYPE "call-lowering"
28 
29 using namespace llvm;
30 
31 void CallLowering::anchor() {}
32 
33 /// Helper function which updates \p Flags when \p AttrFn returns true.
34 static void
35 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
36                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
37   if (AttrFn(Attribute::SExt))
38     Flags.setSExt();
39   if (AttrFn(Attribute::ZExt))
40     Flags.setZExt();
41   if (AttrFn(Attribute::InReg))
42     Flags.setInReg();
43   if (AttrFn(Attribute::StructRet))
44     Flags.setSRet();
45   if (AttrFn(Attribute::Nest))
46     Flags.setNest();
47   if (AttrFn(Attribute::ByVal))
48     Flags.setByVal();
49   if (AttrFn(Attribute::Preallocated))
50     Flags.setPreallocated();
51   if (AttrFn(Attribute::InAlloca))
52     Flags.setInAlloca();
53   if (AttrFn(Attribute::Returned))
54     Flags.setReturned();
55   if (AttrFn(Attribute::SwiftSelf))
56     Flags.setSwiftSelf();
57   if (AttrFn(Attribute::SwiftError))
58     Flags.setSwiftError();
59 }
60 
61 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
62                                                      unsigned ArgIdx) const {
63   ISD::ArgFlagsTy Flags;
64   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
65     return Call.paramHasAttr(ArgIdx, Attr);
66   });
67   return Flags;
68 }
69 
70 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
71                                              const AttributeList &Attrs,
72                                              unsigned OpIdx) const {
73   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
74     return Attrs.hasAttribute(OpIdx, Attr);
75   });
76 }
77 
78 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
79                              ArrayRef<Register> ResRegs,
80                              ArrayRef<ArrayRef<Register>> ArgRegs,
81                              Register SwiftErrorVReg,
82                              std::function<unsigned()> GetCalleeReg) const {
83   CallLoweringInfo Info;
84   const DataLayout &DL = MIRBuilder.getDataLayout();
85   MachineFunction &MF = MIRBuilder.getMF();
86   bool CanBeTailCalled = CB.isTailCall() &&
87                          isInTailCallPosition(CB, MF.getTarget()) &&
88                          (MF.getFunction()
89                               .getFnAttribute("disable-tail-calls")
90                               .getValueAsString() != "true");
91 
92   CallingConv::ID CallConv = CB.getCallingConv();
93   Type *RetTy = CB.getType();
94   bool IsVarArg = CB.getFunctionType()->isVarArg();
95 
96   SmallVector<BaseArgInfo, 4> SplitArgs;
97   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
98   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
99 
100   if (!Info.CanLowerReturn) {
101     // Callee requires sret demotion.
102     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
103 
104     // The sret demotion isn't compatible with tail-calls, since the sret
105     // argument points into the caller's stack frame.
106     CanBeTailCalled = false;
107   }
108 
109   // First step is to marshall all the function's parameters into the correct
110   // physregs and memory locations. Gather the sequence of argument types that
111   // we'll pass to the assigner function.
112   unsigned i = 0;
113   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
114   for (auto &Arg : CB.args()) {
115     ArgInfo OrigArg{ArgRegs[i], Arg->getType(), getAttributesForArgIdx(CB, i),
116                     i < NumFixedArgs};
117     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
118 
119     // If we have an explicit sret argument that is an Instruction, (i.e., it
120     // might point to function-local memory), we can't meaningfully tail-call.
121     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
122       CanBeTailCalled = false;
123 
124     Info.OrigArgs.push_back(OrigArg);
125     ++i;
126   }
127 
128   // Try looking through a bitcast from one function type to another.
129   // Commonly happens with calls to objc_msgSend().
130   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
131   if (const Function *F = dyn_cast<Function>(CalleeV))
132     Info.Callee = MachineOperand::CreateGA(F, 0);
133   else
134     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
135 
136   Info.OrigRet = ArgInfo{ResRegs, RetTy, ISD::ArgFlagsTy{}};
137   if (!Info.OrigRet.Ty->isVoidTy())
138     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
139 
140   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
141   Info.CallConv = CallConv;
142   Info.SwiftErrorVReg = SwiftErrorVReg;
143   Info.IsMustTailCall = CB.isMustTailCall();
144   Info.IsTailCall = CanBeTailCalled;
145   Info.IsVarArg = IsVarArg;
146   return lowerCall(MIRBuilder, Info);
147 }
148 
149 template <typename FuncInfoTy>
150 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
151                                const DataLayout &DL,
152                                const FuncInfoTy &FuncInfo) const {
153   auto &Flags = Arg.Flags[0];
154   const AttributeList &Attrs = FuncInfo.getAttributes();
155   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
156 
157   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
158     Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
159 
160     auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType();
161     Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy));
162 
163     // For ByVal, alignment should be passed from FE.  BE will guess if
164     // this info is not there but there are cases it cannot get right.
165     Align FrameAlign;
166     if (auto ParamAlign = FuncInfo.getParamAlign(OpIdx - 1))
167       FrameAlign = *ParamAlign;
168     else
169       FrameAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
170     Flags.setByValAlign(FrameAlign);
171   }
172   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
173 
174   // Don't try to use the returned attribute if the argument is marked as
175   // swiftself, since it won't be passed in x0.
176   if (Flags.isSwiftSelf())
177     Flags.setReturned(false);
178 }
179 
180 template void
181 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
182                                     const DataLayout &DL,
183                                     const Function &FuncInfo) const;
184 
185 template void
186 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
187                                     const DataLayout &DL,
188                                     const CallBase &FuncInfo) const;
189 
190 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
191                                      SmallVectorImpl<ArgInfo> &SplitArgs,
192                                      const DataLayout &DL,
193                                      CallingConv::ID CallConv) const {
194   LLVMContext &Ctx = OrigArg.Ty->getContext();
195 
196   SmallVector<EVT, 4> SplitVTs;
197   SmallVector<uint64_t, 4> Offsets;
198   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
199 
200   if (SplitVTs.size() == 0)
201     return;
202 
203   if (SplitVTs.size() == 1) {
204     // No splitting to do, but we want to replace the original type (e.g. [1 x
205     // double] -> double).
206     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
207                            OrigArg.Flags[0], OrigArg.IsFixed);
208     return;
209   }
210 
211   // Create one ArgInfo for each virtual register in the original ArgInfo.
212   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
213 
214   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
215       OrigArg.Ty, CallConv, false);
216   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
217     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
218     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.Flags[0],
219                            OrigArg.IsFixed);
220     if (NeedsRegBlock)
221       SplitArgs.back().Flags[0].setInConsecutiveRegs();
222   }
223 
224   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
225 }
226 
227 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
228                               Type *PackedTy,
229                               MachineIRBuilder &MIRBuilder) const {
230   assert(DstRegs.size() > 1 && "Nothing to unpack");
231 
232   const DataLayout &DL = MIRBuilder.getDataLayout();
233 
234   SmallVector<LLT, 8> LLTs;
235   SmallVector<uint64_t, 8> Offsets;
236   computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
237   assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch");
238 
239   for (unsigned i = 0; i < DstRegs.size(); ++i)
240     MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]);
241 }
242 
243 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
244 static MachineInstrBuilder
245 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
246                             ArrayRef<Register> SrcRegs) {
247   MachineRegisterInfo &MRI = *B.getMRI();
248   LLT LLTy = MRI.getType(DstRegs[0]);
249   LLT PartLLT = MRI.getType(SrcRegs[0]);
250 
251   // Deal with v3s16 split into v2s16
252   LLT LCMTy = getLCMType(LLTy, PartLLT);
253   if (LCMTy == LLTy) {
254     // Common case where no padding is needed.
255     assert(DstRegs.size() == 1);
256     return B.buildConcatVectors(DstRegs[0], SrcRegs);
257   }
258 
259   // We need to create an unmerge to the result registers, which may require
260   // widening the original value.
261   Register UnmergeSrcReg;
262   if (LCMTy != PartLLT) {
263     // e.g. A <3 x s16> value was split to <2 x s16>
264     // %register_value0:_(<2 x s16>)
265     // %register_value1:_(<2 x s16>)
266     // %undef:_(<2 x s16>) = G_IMPLICIT_DEF
267     // %concat:_<6 x s16>) = G_CONCAT_VECTORS %reg_value0, %reg_value1, %undef
268     // %dst_reg:_(<3 x s16>), %dead:_(<3 x s16>) = G_UNMERGE_VALUES %concat
269     const int NumWide = LCMTy.getSizeInBits() / PartLLT.getSizeInBits();
270     Register Undef = B.buildUndef(PartLLT).getReg(0);
271 
272     // Build vector of undefs.
273     SmallVector<Register, 8> WidenedSrcs(NumWide, Undef);
274 
275     // Replace the first sources with the real registers.
276     std::copy(SrcRegs.begin(), SrcRegs.end(), WidenedSrcs.begin());
277     UnmergeSrcReg = B.buildConcatVectors(LCMTy, WidenedSrcs).getReg(0);
278   } else {
279     // We don't need to widen anything if we're extracting a scalar which was
280     // promoted to a vector e.g. s8 -> v4s8 -> s8
281     assert(SrcRegs.size() == 1);
282     UnmergeSrcReg = SrcRegs[0];
283   }
284 
285   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
286 
287   SmallVector<Register, 8> PadDstRegs(NumDst);
288   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
289 
290   // Create the excess dead defs for the unmerge.
291   for (int I = DstRegs.size(); I != NumDst; ++I)
292     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
293 
294   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
295 }
296 
297 /// Create a sequence of instructions to combine pieces split into register
298 /// typed values to the original IR value. \p OrigRegs contains the destination
299 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
300 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
301 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
302                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT) {
303   MachineRegisterInfo &MRI = *B.getMRI();
304 
305   // We could just insert a regular copy, but this is unreachable at the moment.
306   assert(LLTy != PartLLT && "identical part types shouldn't reach here");
307 
308   if (PartLLT.isVector() == LLTy.isVector() &&
309       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits()) {
310     assert(OrigRegs.size() == 1 && Regs.size() == 1);
311     B.buildTrunc(OrigRegs[0], Regs[0]);
312     return;
313   }
314 
315   if (!LLTy.isVector() && !PartLLT.isVector()) {
316     assert(OrigRegs.size() == 1);
317     LLT OrigTy = MRI.getType(OrigRegs[0]);
318 
319     unsigned SrcSize = PartLLT.getSizeInBits() * Regs.size();
320     if (SrcSize == OrigTy.getSizeInBits())
321       B.buildMerge(OrigRegs[0], Regs);
322     else {
323       auto Widened = B.buildMerge(LLT::scalar(SrcSize), Regs);
324       B.buildTrunc(OrigRegs[0], Widened);
325     }
326 
327     return;
328   }
329 
330   if (PartLLT.isVector()) {
331     assert(OrigRegs.size() == 1 &&
332            LLTy.getScalarType() == PartLLT.getElementType());
333     mergeVectorRegsToResultRegs(B, OrigRegs, Regs);
334     return;
335   }
336 
337   assert(LLTy.isVector() && !PartLLT.isVector());
338 
339   LLT DstEltTy = LLTy.getElementType();
340 
341   // Pointer information was discarded. We'll need to coerce some register types
342   // to avoid violating type constraints.
343   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
344 
345   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
346 
347   if (DstEltTy == PartLLT) {
348     // Vector was trivially scalarized.
349 
350     if (RealDstEltTy.isPointer()) {
351       for (Register Reg : Regs)
352         MRI.setType(Reg, RealDstEltTy);
353     }
354 
355     B.buildBuildVector(OrigRegs[0], Regs);
356   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
357     // Deal with vector with 64-bit elements decomposed to 32-bit
358     // registers. Need to create intermediate 64-bit elements.
359     SmallVector<Register, 8> EltMerges;
360     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
361 
362     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
363 
364     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
365       auto Merge = B.buildMerge(RealDstEltTy, Regs.take_front(PartsPerElt));
366       // Fix the type in case this is really a vector of pointers.
367       MRI.setType(Merge.getReg(0), RealDstEltTy);
368       EltMerges.push_back(Merge.getReg(0));
369       Regs = Regs.drop_front(PartsPerElt);
370     }
371 
372     B.buildBuildVector(OrigRegs[0], EltMerges);
373   } else {
374     // Vector was split, and elements promoted to a wider type.
375     // FIXME: Should handle floating point promotions.
376     LLT BVType = LLT::vector(LLTy.getNumElements(), PartLLT);
377     auto BV = B.buildBuildVector(BVType, Regs);
378     B.buildTrunc(OrigRegs[0], BV);
379   }
380 }
381 
382 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
383 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
384 /// contain the type of scalar value extension if necessary.
385 ///
386 /// This is used for outgoing values (vregs to physregs)
387 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
388                             Register SrcReg, LLT SrcTy, LLT PartTy,
389                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
390   // We could just insert a regular copy, but this is unreachable at the moment.
391   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
392 
393   const unsigned PartSize = PartTy.getSizeInBits();
394 
395   if (PartTy.isVector() == SrcTy.isVector() &&
396       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
397     assert(DstRegs.size() == 1);
398     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
399     return;
400   }
401 
402   if (SrcTy.isVector() && !PartTy.isVector() &&
403       PartSize > SrcTy.getElementType().getSizeInBits()) {
404     // Vector was scalarized, and the elements extended.
405     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
406     for (int i = 0, e = DstRegs.size(); i != e; ++i)
407       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
408     return;
409   }
410 
411   LLT GCDTy = getGCDType(SrcTy, PartTy);
412   if (GCDTy == PartTy) {
413     // If this already evenly divisible, we can create a simple unmerge.
414     B.buildUnmerge(DstRegs, SrcReg);
415     return;
416   }
417 
418   MachineRegisterInfo &MRI = *B.getMRI();
419   LLT DstTy = MRI.getType(DstRegs[0]);
420   LLT LCMTy = getLCMType(SrcTy, PartTy);
421 
422   const unsigned LCMSize = LCMTy.getSizeInBits();
423   const unsigned DstSize = DstTy.getSizeInBits();
424   const unsigned SrcSize = SrcTy.getSizeInBits();
425 
426   Register UnmergeSrc = SrcReg;
427   if (LCMSize != SrcSize) {
428     // Widen to the common type.
429     Register Undef = B.buildUndef(SrcTy).getReg(0);
430     SmallVector<Register, 8> MergeParts(1, SrcReg);
431     for (unsigned Size = SrcSize; Size != LCMSize; Size += SrcSize)
432       MergeParts.push_back(Undef);
433 
434     UnmergeSrc = B.buildMerge(LCMTy, MergeParts).getReg(0);
435   }
436 
437   // Unmerge to the original registers and pad with dead defs.
438   SmallVector<Register, 8> UnmergeResults(DstRegs.begin(), DstRegs.end());
439   for (unsigned Size = DstSize * DstRegs.size(); Size != LCMSize;
440        Size += DstSize) {
441     UnmergeResults.push_back(MRI.createGenericVirtualRegister(DstTy));
442   }
443 
444   B.buildUnmerge(UnmergeResults, UnmergeSrc);
445 }
446 
447 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder,
448                                      SmallVectorImpl<ArgInfo> &Args,
449                                      ValueHandler &Handler,
450                                      CallingConv::ID CallConv, bool IsVarArg,
451                                      Register ThisReturnReg) const {
452   MachineFunction &MF = MIRBuilder.getMF();
453   const Function &F = MF.getFunction();
454   SmallVector<CCValAssign, 16> ArgLocs;
455 
456   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
457   return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler,
458                            ThisReturnReg);
459 }
460 
461 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
462   if (Flags.isSExt())
463     return TargetOpcode::G_SEXT;
464   if (Flags.isZExt())
465     return TargetOpcode::G_ZEXT;
466   return TargetOpcode::G_ANYEXT;
467 }
468 
469 bool CallLowering::handleAssignments(CCState &CCInfo,
470                                      SmallVectorImpl<CCValAssign> &ArgLocs,
471                                      MachineIRBuilder &MIRBuilder,
472                                      SmallVectorImpl<ArgInfo> &Args,
473                                      ValueHandler &Handler,
474                                      Register ThisReturnReg) const {
475   MachineFunction &MF = MIRBuilder.getMF();
476   MachineRegisterInfo &MRI = MF.getRegInfo();
477   const Function &F = MF.getFunction();
478   const DataLayout &DL = F.getParent()->getDataLayout();
479 
480   unsigned NumArgs = Args.size();
481   for (unsigned i = 0; i != NumArgs; ++i) {
482     EVT CurVT = EVT::getEVT(Args[i].Ty);
483     if (CurVT.isSimple() &&
484         !Handler.assignArg(i, CurVT.getSimpleVT(), CurVT.getSimpleVT(),
485                            CCValAssign::Full, Args[i], Args[i].Flags[0],
486                            CCInfo))
487       continue;
488 
489     MVT NewVT = TLI->getRegisterTypeForCallingConv(
490         F.getContext(), CCInfo.getCallingConv(), EVT(CurVT));
491 
492     // If we need to split the type over multiple regs, check it's a scenario
493     // we currently support.
494     unsigned NumParts = TLI->getNumRegistersForCallingConv(
495         F.getContext(), CCInfo.getCallingConv(), CurVT);
496 
497     if (NumParts == 1) {
498       // Try to use the register type if we couldn't assign the VT.
499       if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
500                             Args[i].Flags[0], CCInfo))
501         return false;
502 
503       // If we couldn't directly assign this part, some casting may be
504       // necessary. Create the new register, but defer inserting the conversion
505       // instructions.
506       assert(Args[i].OrigRegs.empty());
507       Args[i].OrigRegs.push_back(Args[i].Regs[0]);
508       assert(Args[i].Regs.size() == 1);
509 
510       const LLT VATy(NewVT);
511       Args[i].Regs[0] = MRI.createGenericVirtualRegister(VATy);
512       continue;
513     }
514 
515     const LLT NewLLT(NewVT);
516 
517     // For incoming arguments (physregs to vregs), we could have values in
518     // physregs (or memlocs) which we want to extract and copy to vregs.
519     // During this, we might have to deal with the LLT being split across
520     // multiple regs, so we have to record this information for later.
521     //
522     // If we have outgoing args, then we have the opposite case. We have a
523     // vreg with an LLT which we want to assign to a physical location, and
524     // we might have to record that the value has to be split later.
525     if (Handler.isIncomingArgumentHandler()) {
526       // We're handling an incoming arg which is split over multiple regs.
527       // E.g. passing an s128 on AArch64.
528       ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
529       Args[i].OrigRegs.push_back(Args[i].Regs[0]);
530       Args[i].Regs.clear();
531       Args[i].Flags.clear();
532       // For each split register, create and assign a vreg that will store
533       // the incoming component of the larger value. These will later be
534       // merged to form the final vreg.
535       for (unsigned Part = 0; Part < NumParts; ++Part) {
536         Register Reg = MRI.createGenericVirtualRegister(NewLLT);
537         ISD::ArgFlagsTy Flags = OrigFlags;
538         if (Part == 0) {
539           Flags.setSplit();
540         } else {
541           Flags.setOrigAlign(Align(1));
542           if (Part == NumParts - 1)
543             Flags.setSplitEnd();
544         }
545         Args[i].Regs.push_back(Reg);
546         Args[i].Flags.push_back(Flags);
547         if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
548                               Args[i].Flags[Part], CCInfo)) {
549           // Still couldn't assign this smaller part type for some reason.
550           return false;
551         }
552       }
553     } else {
554       assert(Args[i].Regs.size() == 1);
555 
556       // This type is passed via multiple registers in the calling convention.
557       // We need to extract the individual parts.
558       assert(Args[i].OrigRegs.empty());
559       Args[i].OrigRegs.push_back(Args[i].Regs[0]);
560 
561       ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
562       // We're going to replace the regs and flags with the split ones.
563       Args[i].Regs.clear();
564       Args[i].Flags.clear();
565       for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) {
566         ISD::ArgFlagsTy Flags = OrigFlags;
567         if (PartIdx == 0) {
568           Flags.setSplit();
569         } else {
570           Flags.setOrigAlign(Align(1));
571           if (PartIdx == NumParts - 1)
572             Flags.setSplitEnd();
573         }
574 
575         // TODO: Also check if there is a valid extension that preserves the
576         // bits. However currently this call lowering doesn't support non-exact
577         // split parts, so that can't be tested.
578         if (OrigFlags.isReturned() &&
579             (NumParts * NewVT.getSizeInBits() != CurVT.getSizeInBits())) {
580           Flags.setReturned(false);
581         }
582 
583         Register NewReg = MRI.createGenericVirtualRegister(NewLLT);
584 
585         Args[i].Regs.push_back(NewReg);
586         Args[i].Flags.push_back(Flags);
587         if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full,
588                               Args[i], Args[i].Flags[PartIdx], CCInfo))
589           return false;
590       }
591     }
592   }
593 
594   for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) {
595     assert(j < ArgLocs.size() && "Skipped too many arg locs");
596 
597     CCValAssign &VA = ArgLocs[j];
598     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
599 
600     if (VA.needsCustom()) {
601       unsigned NumArgRegs =
602           Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j));
603       if (!NumArgRegs)
604         return false;
605       j += NumArgRegs;
606       continue;
607     }
608 
609     EVT VAVT = VA.getValVT();
610     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
611     const LLT VATy(VAVT.getSimpleVT());
612 
613     // Expected to be multiple regs for a single incoming arg.
614     // There should be Regs.size() ArgLocs per argument.
615     unsigned NumArgRegs = Args[i].Regs.size();
616     assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
617            "Too many regs for number of args");
618 
619     // Coerce into outgoing value types before register assignment.
620     if (!Handler.isIncomingArgumentHandler() && OrigTy != VATy) {
621       assert(Args[i].OrigRegs.size() == 1);
622       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
623                       VATy, extendOpFromFlags(Args[i].Flags[0]));
624     }
625 
626     for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
627       Register ArgReg = Args[i].Regs[Part];
628       // There should be Regs.size() ArgLocs per argument.
629       VA = ArgLocs[j + Part];
630       if (VA.isMemLoc()) {
631         // Individual pieces may have been spilled to the stack and others
632         // passed in registers.
633 
634         // FIXME: Use correct address space for pointer size
635         EVT LocVT = VA.getValVT();
636         unsigned MemSize = LocVT == MVT::iPTR ? DL.getPointerSize()
637                                               : LocVT.getStoreSize();
638         unsigned Offset = VA.getLocMemOffset();
639         MachinePointerInfo MPO;
640         Register StackAddr = Handler.getStackAddress(MemSize, Offset, MPO);
641         Handler.assignValueToAddress(Args[i], Part, StackAddr, MemSize, MPO,
642                                      VA);
643         continue;
644       }
645 
646       assert(VA.isRegLoc() && "custom loc should have been handled already");
647 
648       if (i == 0 && ThisReturnReg.isValid() &&
649           Handler.isIncomingArgumentHandler() &&
650           isTypeIsValidForThisReturn(VAVT)) {
651         Handler.assignValueToReg(Args[i].Regs[i], ThisReturnReg, VA);
652         continue;
653       }
654 
655       Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
656     }
657 
658     // Now that all pieces have been assigned, re-pack the register typed values
659     // into the original value typed registers.
660     if (Handler.isIncomingArgumentHandler() && OrigTy != VATy) {
661       // Merge the split registers into the expected larger result vregs of
662       // the original call.
663       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
664                         VATy);
665     }
666 
667     j += NumArgRegs - 1;
668   }
669 
670   return true;
671 }
672 
673 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
674                                    ArrayRef<Register> VRegs, Register DemoteReg,
675                                    int FI) const {
676   MachineFunction &MF = MIRBuilder.getMF();
677   MachineRegisterInfo &MRI = MF.getRegInfo();
678   const DataLayout &DL = MF.getDataLayout();
679 
680   SmallVector<EVT, 4> SplitVTs;
681   SmallVector<uint64_t, 4> Offsets;
682   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
683 
684   assert(VRegs.size() == SplitVTs.size());
685 
686   unsigned NumValues = SplitVTs.size();
687   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
688   Type *RetPtrTy = RetTy->getPointerTo(DL.getAllocaAddrSpace());
689   LLT OffsetLLTy = getLLTForType(*DL.getIntPtrType(RetPtrTy), DL);
690 
691   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
692 
693   for (unsigned I = 0; I < NumValues; ++I) {
694     Register Addr;
695     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
696     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
697                                         MRI.getType(VRegs[I]).getSizeInBytes(),
698                                         commonAlignment(BaseAlign, Offsets[I]));
699     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
700   }
701 }
702 
703 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
704                                     ArrayRef<Register> VRegs,
705                                     Register DemoteReg) const {
706   MachineFunction &MF = MIRBuilder.getMF();
707   MachineRegisterInfo &MRI = MF.getRegInfo();
708   const DataLayout &DL = MF.getDataLayout();
709 
710   SmallVector<EVT, 4> SplitVTs;
711   SmallVector<uint64_t, 4> Offsets;
712   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
713 
714   assert(VRegs.size() == SplitVTs.size());
715 
716   unsigned NumValues = SplitVTs.size();
717   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
718   unsigned AS = DL.getAllocaAddrSpace();
719   LLT OffsetLLTy =
720       getLLTForType(*DL.getIntPtrType(RetTy->getPointerTo(AS)), DL);
721 
722   MachinePointerInfo PtrInfo(AS);
723 
724   for (unsigned I = 0; I < NumValues; ++I) {
725     Register Addr;
726     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
727     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
728                                         MRI.getType(VRegs[I]).getSizeInBytes(),
729                                         commonAlignment(BaseAlign, Offsets[I]));
730     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
731   }
732 }
733 
734 void CallLowering::insertSRetIncomingArgument(
735     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
736     MachineRegisterInfo &MRI, const DataLayout &DL) const {
737   unsigned AS = DL.getAllocaAddrSpace();
738   DemoteReg = MRI.createGenericVirtualRegister(
739       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
740 
741   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
742 
743   SmallVector<EVT, 1> ValueVTs;
744   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
745 
746   // NOTE: Assume that a pointer won't get split into more than one VT.
747   assert(ValueVTs.size() == 1);
748 
749   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()));
750   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
751   DemoteArg.Flags[0].setSRet();
752   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
753 }
754 
755 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
756                                               const CallBase &CB,
757                                               CallLoweringInfo &Info) const {
758   const DataLayout &DL = MIRBuilder.getDataLayout();
759   Type *RetTy = CB.getType();
760   unsigned AS = DL.getAllocaAddrSpace();
761   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
762 
763   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
764       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
765 
766   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
767   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS));
768   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
769   DemoteArg.Flags[0].setSRet();
770 
771   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
772   Info.DemoteStackIndex = FI;
773   Info.DemoteRegister = DemoteReg;
774 }
775 
776 bool CallLowering::checkReturn(CCState &CCInfo,
777                                SmallVectorImpl<BaseArgInfo> &Outs,
778                                CCAssignFn *Fn) const {
779   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
780     MVT VT = MVT::getVT(Outs[I].Ty);
781     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
782       return false;
783   }
784   return true;
785 }
786 
787 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
788                                  AttributeList Attrs,
789                                  SmallVectorImpl<BaseArgInfo> &Outs,
790                                  const DataLayout &DL) const {
791   LLVMContext &Context = RetTy->getContext();
792   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
793 
794   SmallVector<EVT, 4> SplitVTs;
795   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
796   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
797 
798   for (EVT VT : SplitVTs) {
799     unsigned NumParts =
800         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
801     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
802     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
803 
804     for (unsigned I = 0; I < NumParts; ++I) {
805       Outs.emplace_back(PartTy, Flags);
806     }
807   }
808 }
809 
810 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
811   const auto &F = MF.getFunction();
812   Type *ReturnType = F.getReturnType();
813   CallingConv::ID CallConv = F.getCallingConv();
814 
815   SmallVector<BaseArgInfo, 4> SplitArgs;
816   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
817                 MF.getDataLayout());
818   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
819 }
820 
821 bool CallLowering::analyzeArgInfo(CCState &CCState,
822                                   SmallVectorImpl<ArgInfo> &Args,
823                                   CCAssignFn &AssignFnFixed,
824                                   CCAssignFn &AssignFnVarArg) const {
825   for (unsigned i = 0, e = Args.size(); i < e; ++i) {
826     MVT VT = MVT::getVT(Args[i].Ty);
827     CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
828     if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
829       // Bail out on anything we can't handle.
830       LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
831                         << " (arg number = " << i << "\n");
832       return false;
833     }
834   }
835   return true;
836 }
837 
838 bool CallLowering::parametersInCSRMatch(
839     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
840     const SmallVectorImpl<CCValAssign> &OutLocs,
841     const SmallVectorImpl<ArgInfo> &OutArgs) const {
842   for (unsigned i = 0; i < OutLocs.size(); ++i) {
843     auto &ArgLoc = OutLocs[i];
844     // If it's not a register, it's fine.
845     if (!ArgLoc.isRegLoc())
846       continue;
847 
848     MCRegister PhysReg = ArgLoc.getLocReg();
849 
850     // Only look at callee-saved registers.
851     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
852       continue;
853 
854     LLVM_DEBUG(
855         dbgs()
856         << "... Call has an argument passed in a callee-saved register.\n");
857 
858     // Check if it was copied from.
859     const ArgInfo &OutInfo = OutArgs[i];
860 
861     if (OutInfo.Regs.size() > 1) {
862       LLVM_DEBUG(
863           dbgs() << "... Cannot handle arguments in multiple registers.\n");
864       return false;
865     }
866 
867     // Check if we copy the register, walking through copies from virtual
868     // registers. Note that getDefIgnoringCopies does not ignore copies from
869     // physical registers.
870     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
871     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
872       LLVM_DEBUG(
873           dbgs()
874           << "... Parameter was not copied into a VReg, cannot tail call.\n");
875       return false;
876     }
877 
878     // Got a copy. Verify that it's the same as the register we want.
879     Register CopyRHS = RegDef->getOperand(1).getReg();
880     if (CopyRHS != PhysReg) {
881       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
882                            "VReg, cannot tail call.\n");
883       return false;
884     }
885   }
886 
887   return true;
888 }
889 
890 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
891                                      MachineFunction &MF,
892                                      SmallVectorImpl<ArgInfo> &InArgs,
893                                      CCAssignFn &CalleeAssignFnFixed,
894                                      CCAssignFn &CalleeAssignFnVarArg,
895                                      CCAssignFn &CallerAssignFnFixed,
896                                      CCAssignFn &CallerAssignFnVarArg) const {
897   const Function &F = MF.getFunction();
898   CallingConv::ID CalleeCC = Info.CallConv;
899   CallingConv::ID CallerCC = F.getCallingConv();
900 
901   if (CallerCC == CalleeCC)
902     return true;
903 
904   SmallVector<CCValAssign, 16> ArgLocs1;
905   CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
906   if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
907                       CalleeAssignFnVarArg))
908     return false;
909 
910   SmallVector<CCValAssign, 16> ArgLocs2;
911   CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
912   if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
913                       CalleeAssignFnVarArg))
914     return false;
915 
916   // We need the argument locations to match up exactly. If there's more in
917   // one than the other, then we are done.
918   if (ArgLocs1.size() != ArgLocs2.size())
919     return false;
920 
921   // Make sure that each location is passed in exactly the same way.
922   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
923     const CCValAssign &Loc1 = ArgLocs1[i];
924     const CCValAssign &Loc2 = ArgLocs2[i];
925 
926     // We need both of them to be the same. So if one is a register and one
927     // isn't, we're done.
928     if (Loc1.isRegLoc() != Loc2.isRegLoc())
929       return false;
930 
931     if (Loc1.isRegLoc()) {
932       // If they don't have the same register location, we're done.
933       if (Loc1.getLocReg() != Loc2.getLocReg())
934         return false;
935 
936       // They matched, so we can move to the next ArgLoc.
937       continue;
938     }
939 
940     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
941     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
942       return false;
943   }
944 
945   return true;
946 }
947 
948 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
949                                                     CCValAssign &VA,
950                                                     unsigned MaxSizeBits) {
951   LLT LocTy{VA.getLocVT()};
952   LLT ValTy = MRI.getType(ValReg);
953   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
954     return ValReg;
955 
956   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
957     if (MaxSizeBits <= ValTy.getSizeInBits())
958       return ValReg;
959     LocTy = LLT::scalar(MaxSizeBits);
960   }
961 
962   switch (VA.getLocInfo()) {
963   default: break;
964   case CCValAssign::Full:
965   case CCValAssign::BCvt:
966     // FIXME: bitconverting between vector types may or may not be a
967     // nop in big-endian situations.
968     return ValReg;
969   case CCValAssign::AExt: {
970     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
971     return MIB.getReg(0);
972   }
973   case CCValAssign::SExt: {
974     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
975     MIRBuilder.buildSExt(NewReg, ValReg);
976     return NewReg;
977   }
978   case CCValAssign::ZExt: {
979     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
980     MIRBuilder.buildZExt(NewReg, ValReg);
981     return NewReg;
982   }
983   }
984   llvm_unreachable("unable to extend register");
985 }
986 
987 void CallLowering::ValueHandler::anchor() {}
988 
989 Register CallLowering::IncomingValueHandler::buildExtensionHint(CCValAssign &VA,
990                                                                 Register SrcReg,
991                                                                 LLT NarrowTy) {
992   switch (VA.getLocInfo()) {
993   case CCValAssign::LocInfo::ZExt: {
994     return MIRBuilder
995         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
996                          NarrowTy.getScalarSizeInBits())
997         .getReg(0);
998   }
999   case CCValAssign::LocInfo::SExt: {
1000     return MIRBuilder
1001         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1002                          NarrowTy.getScalarSizeInBits())
1003         .getReg(0);
1004     break;
1005   }
1006   default:
1007     return SrcReg;
1008   }
1009 }
1010 
1011 void CallLowering::IncomingValueHandler::assignValueToReg(Register ValVReg,
1012                                                           Register PhysReg,
1013                                                           CCValAssign &VA) {
1014   const LLT LocTy(VA.getLocVT());
1015   const LLT ValTy = MRI.getType(ValVReg);
1016 
1017   if (ValTy.getSizeInBits() == LocTy.getSizeInBits()) {
1018     MIRBuilder.buildCopy(ValVReg, PhysReg);
1019     return;
1020   }
1021 
1022   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1023   auto Hint = buildExtensionHint(VA, Copy.getReg(0), ValTy);
1024   MIRBuilder.buildTrunc(ValVReg, Hint);
1025 }
1026