1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements some simple delegations needed for call lowering. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/Analysis.h" 15 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 16 #include "llvm/CodeGen/GlobalISel/Utils.h" 17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 18 #include "llvm/CodeGen/MachineOperand.h" 19 #include "llvm/CodeGen/MachineRegisterInfo.h" 20 #include "llvm/CodeGen/TargetLowering.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/Instructions.h" 23 #include "llvm/IR/LLVMContext.h" 24 #include "llvm/IR/Module.h" 25 #include "llvm/Target/TargetMachine.h" 26 27 #define DEBUG_TYPE "call-lowering" 28 29 using namespace llvm; 30 31 void CallLowering::anchor() {} 32 33 /// Helper function which updates \p Flags when \p AttrFn returns true. 34 static void 35 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, 36 const std::function<bool(Attribute::AttrKind)> &AttrFn) { 37 if (AttrFn(Attribute::SExt)) 38 Flags.setSExt(); 39 if (AttrFn(Attribute::ZExt)) 40 Flags.setZExt(); 41 if (AttrFn(Attribute::InReg)) 42 Flags.setInReg(); 43 if (AttrFn(Attribute::StructRet)) 44 Flags.setSRet(); 45 if (AttrFn(Attribute::Nest)) 46 Flags.setNest(); 47 if (AttrFn(Attribute::ByVal)) 48 Flags.setByVal(); 49 if (AttrFn(Attribute::Preallocated)) 50 Flags.setPreallocated(); 51 if (AttrFn(Attribute::InAlloca)) 52 Flags.setInAlloca(); 53 if (AttrFn(Attribute::Returned)) 54 Flags.setReturned(); 55 if (AttrFn(Attribute::SwiftSelf)) 56 Flags.setSwiftSelf(); 57 if (AttrFn(Attribute::SwiftError)) 58 Flags.setSwiftError(); 59 } 60 61 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, 62 unsigned ArgIdx) const { 63 ISD::ArgFlagsTy Flags; 64 addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) { 65 return Call.paramHasAttr(ArgIdx, Attr); 66 }); 67 return Flags; 68 } 69 70 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, 71 const AttributeList &Attrs, 72 unsigned OpIdx) const { 73 addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) { 74 return Attrs.hasAttribute(OpIdx, Attr); 75 }); 76 } 77 78 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, 79 ArrayRef<Register> ResRegs, 80 ArrayRef<ArrayRef<Register>> ArgRegs, 81 Register SwiftErrorVReg, 82 std::function<unsigned()> GetCalleeReg) const { 83 CallLoweringInfo Info; 84 const DataLayout &DL = MIRBuilder.getDataLayout(); 85 MachineFunction &MF = MIRBuilder.getMF(); 86 bool CanBeTailCalled = CB.isTailCall() && 87 isInTailCallPosition(CB, MF.getTarget()) && 88 (MF.getFunction() 89 .getFnAttribute("disable-tail-calls") 90 .getValueAsString() != "true"); 91 92 CallingConv::ID CallConv = CB.getCallingConv(); 93 Type *RetTy = CB.getType(); 94 bool IsVarArg = CB.getFunctionType()->isVarArg(); 95 96 SmallVector<BaseArgInfo, 4> SplitArgs; 97 getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL); 98 Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg); 99 100 if (!Info.CanLowerReturn) { 101 // Callee requires sret demotion. 102 insertSRetOutgoingArgument(MIRBuilder, CB, Info); 103 104 // The sret demotion isn't compatible with tail-calls, since the sret 105 // argument points into the caller's stack frame. 106 CanBeTailCalled = false; 107 } 108 109 // First step is to marshall all the function's parameters into the correct 110 // physregs and memory locations. Gather the sequence of argument types that 111 // we'll pass to the assigner function. 112 unsigned i = 0; 113 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); 114 for (auto &Arg : CB.args()) { 115 ArgInfo OrigArg{ArgRegs[i], Arg->getType(), getAttributesForArgIdx(CB, i), 116 i < NumFixedArgs}; 117 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB); 118 119 // If we have an explicit sret argument that is an Instruction, (i.e., it 120 // might point to function-local memory), we can't meaningfully tail-call. 121 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg)) 122 CanBeTailCalled = false; 123 124 Info.OrigArgs.push_back(OrigArg); 125 ++i; 126 } 127 128 // Try looking through a bitcast from one function type to another. 129 // Commonly happens with calls to objc_msgSend(). 130 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); 131 if (const Function *F = dyn_cast<Function>(CalleeV)) 132 Info.Callee = MachineOperand::CreateGA(F, 0); 133 else 134 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); 135 136 Info.OrigRet = ArgInfo{ResRegs, RetTy, ISD::ArgFlagsTy{}}; 137 if (!Info.OrigRet.Ty->isVoidTy()) 138 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB); 139 140 Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); 141 Info.CallConv = CallConv; 142 Info.SwiftErrorVReg = SwiftErrorVReg; 143 Info.IsMustTailCall = CB.isMustTailCall(); 144 Info.IsTailCall = CanBeTailCalled; 145 Info.IsVarArg = IsVarArg; 146 return lowerCall(MIRBuilder, Info); 147 } 148 149 template <typename FuncInfoTy> 150 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, 151 const DataLayout &DL, 152 const FuncInfoTy &FuncInfo) const { 153 auto &Flags = Arg.Flags[0]; 154 const AttributeList &Attrs = FuncInfo.getAttributes(); 155 addArgFlagsFromAttributes(Flags, Attrs, OpIdx); 156 157 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { 158 Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType(); 159 160 auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType(); 161 Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy)); 162 163 // For ByVal, alignment should be passed from FE. BE will guess if 164 // this info is not there but there are cases it cannot get right. 165 Align FrameAlign; 166 if (auto ParamAlign = FuncInfo.getParamAlign(OpIdx - 1)) 167 FrameAlign = *ParamAlign; 168 else 169 FrameAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL)); 170 Flags.setByValAlign(FrameAlign); 171 } 172 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty)); 173 174 // Don't try to use the returned attribute if the argument is marked as 175 // swiftself, since it won't be passed in x0. 176 if (Flags.isSwiftSelf()) 177 Flags.setReturned(false); 178 } 179 180 template void 181 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 182 const DataLayout &DL, 183 const Function &FuncInfo) const; 184 185 template void 186 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 187 const DataLayout &DL, 188 const CallBase &FuncInfo) const; 189 190 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 191 SmallVectorImpl<ArgInfo> &SplitArgs, 192 const DataLayout &DL, 193 CallingConv::ID CallConv) const { 194 LLVMContext &Ctx = OrigArg.Ty->getContext(); 195 196 SmallVector<EVT, 4> SplitVTs; 197 SmallVector<uint64_t, 4> Offsets; 198 ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 199 200 if (SplitVTs.size() == 0) 201 return; 202 203 if (SplitVTs.size() == 1) { 204 // No splitting to do, but we want to replace the original type (e.g. [1 x 205 // double] -> double). 206 SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), 207 OrigArg.Flags[0], OrigArg.IsFixed); 208 return; 209 } 210 211 // Create one ArgInfo for each virtual register in the original ArgInfo. 212 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch"); 213 214 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( 215 OrigArg.Ty, CallConv, false); 216 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { 217 Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); 218 SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.Flags[0], 219 OrigArg.IsFixed); 220 if (NeedsRegBlock) 221 SplitArgs.back().Flags[0].setInConsecutiveRegs(); 222 } 223 224 SplitArgs.back().Flags[0].setInConsecutiveRegsLast(); 225 } 226 227 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg, 228 Type *PackedTy, 229 MachineIRBuilder &MIRBuilder) const { 230 assert(DstRegs.size() > 1 && "Nothing to unpack"); 231 232 const DataLayout &DL = MIRBuilder.getDataLayout(); 233 234 SmallVector<LLT, 8> LLTs; 235 SmallVector<uint64_t, 8> Offsets; 236 computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); 237 assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch"); 238 239 for (unsigned i = 0; i < DstRegs.size(); ++i) 240 MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]); 241 } 242 243 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs. 244 static MachineInstrBuilder 245 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 246 ArrayRef<Register> SrcRegs) { 247 MachineRegisterInfo &MRI = *B.getMRI(); 248 LLT LLTy = MRI.getType(DstRegs[0]); 249 LLT PartLLT = MRI.getType(SrcRegs[0]); 250 251 // Deal with v3s16 split into v2s16 252 LLT LCMTy = getLCMType(LLTy, PartLLT); 253 if (LCMTy == LLTy) { 254 // Common case where no padding is needed. 255 assert(DstRegs.size() == 1); 256 return B.buildConcatVectors(DstRegs[0], SrcRegs); 257 } 258 259 // We need to create an unmerge to the result registers, which may require 260 // widening the original value. 261 Register UnmergeSrcReg; 262 if (LCMTy != PartLLT) { 263 // e.g. A <3 x s16> value was split to <2 x s16> 264 // %register_value0:_(<2 x s16>) 265 // %register_value1:_(<2 x s16>) 266 // %undef:_(<2 x s16>) = G_IMPLICIT_DEF 267 // %concat:_<6 x s16>) = G_CONCAT_VECTORS %reg_value0, %reg_value1, %undef 268 // %dst_reg:_(<3 x s16>), %dead:_(<3 x s16>) = G_UNMERGE_VALUES %concat 269 const int NumWide = LCMTy.getSizeInBits() / PartLLT.getSizeInBits(); 270 Register Undef = B.buildUndef(PartLLT).getReg(0); 271 272 // Build vector of undefs. 273 SmallVector<Register, 8> WidenedSrcs(NumWide, Undef); 274 275 // Replace the first sources with the real registers. 276 std::copy(SrcRegs.begin(), SrcRegs.end(), WidenedSrcs.begin()); 277 UnmergeSrcReg = B.buildConcatVectors(LCMTy, WidenedSrcs).getReg(0); 278 } else { 279 // We don't need to widen anything if we're extracting a scalar which was 280 // promoted to a vector e.g. s8 -> v4s8 -> s8 281 assert(SrcRegs.size() == 1); 282 UnmergeSrcReg = SrcRegs[0]; 283 } 284 285 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits(); 286 287 SmallVector<Register, 8> PadDstRegs(NumDst); 288 std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin()); 289 290 // Create the excess dead defs for the unmerge. 291 for (int I = DstRegs.size(); I != NumDst; ++I) 292 PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy); 293 294 return B.buildUnmerge(PadDstRegs, UnmergeSrcReg); 295 } 296 297 /// Create a sequence of instructions to combine pieces split into register 298 /// typed values to the original IR value. \p OrigRegs contains the destination 299 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces 300 /// with type \p PartLLT. This is used for incoming values (physregs to vregs). 301 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs, 302 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT) { 303 MachineRegisterInfo &MRI = *B.getMRI(); 304 305 // We could just insert a regular copy, but this is unreachable at the moment. 306 assert(LLTy != PartLLT && "identical part types shouldn't reach here"); 307 308 if (PartLLT.isVector() == LLTy.isVector() && 309 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits()) { 310 assert(OrigRegs.size() == 1 && Regs.size() == 1); 311 B.buildTrunc(OrigRegs[0], Regs[0]); 312 return; 313 } 314 315 if (!LLTy.isVector() && !PartLLT.isVector()) { 316 assert(OrigRegs.size() == 1); 317 LLT OrigTy = MRI.getType(OrigRegs[0]); 318 319 unsigned SrcSize = PartLLT.getSizeInBits() * Regs.size(); 320 if (SrcSize == OrigTy.getSizeInBits()) 321 B.buildMerge(OrigRegs[0], Regs); 322 else { 323 auto Widened = B.buildMerge(LLT::scalar(SrcSize), Regs); 324 B.buildTrunc(OrigRegs[0], Widened); 325 } 326 327 return; 328 } 329 330 if (PartLLT.isVector()) { 331 assert(OrigRegs.size() == 1 && 332 LLTy.getScalarType() == PartLLT.getElementType()); 333 mergeVectorRegsToResultRegs(B, OrigRegs, Regs); 334 return; 335 } 336 337 assert(LLTy.isVector() && !PartLLT.isVector()); 338 339 LLT DstEltTy = LLTy.getElementType(); 340 341 // Pointer information was discarded. We'll need to coerce some register types 342 // to avoid violating type constraints. 343 LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType(); 344 345 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); 346 347 if (DstEltTy == PartLLT) { 348 // Vector was trivially scalarized. 349 350 if (RealDstEltTy.isPointer()) { 351 for (Register Reg : Regs) 352 MRI.setType(Reg, RealDstEltTy); 353 } 354 355 B.buildBuildVector(OrigRegs[0], Regs); 356 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { 357 // Deal with vector with 64-bit elements decomposed to 32-bit 358 // registers. Need to create intermediate 64-bit elements. 359 SmallVector<Register, 8> EltMerges; 360 int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits(); 361 362 assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); 363 364 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { 365 auto Merge = B.buildMerge(RealDstEltTy, Regs.take_front(PartsPerElt)); 366 // Fix the type in case this is really a vector of pointers. 367 MRI.setType(Merge.getReg(0), RealDstEltTy); 368 EltMerges.push_back(Merge.getReg(0)); 369 Regs = Regs.drop_front(PartsPerElt); 370 } 371 372 B.buildBuildVector(OrigRegs[0], EltMerges); 373 } else { 374 // Vector was split, and elements promoted to a wider type. 375 // FIXME: Should handle floating point promotions. 376 LLT BVType = LLT::vector(LLTy.getNumElements(), PartLLT); 377 auto BV = B.buildBuildVector(BVType, Regs); 378 B.buildTrunc(OrigRegs[0], BV); 379 } 380 } 381 382 /// Create a sequence of instructions to expand the value in \p SrcReg (of type 383 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should 384 /// contain the type of scalar value extension if necessary. 385 /// 386 /// This is used for outgoing values (vregs to physregs) 387 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 388 Register SrcReg, LLT SrcTy, LLT PartTy, 389 unsigned ExtendOp = TargetOpcode::G_ANYEXT) { 390 // We could just insert a regular copy, but this is unreachable at the moment. 391 assert(SrcTy != PartTy && "identical part types shouldn't reach here"); 392 393 const unsigned PartSize = PartTy.getSizeInBits(); 394 395 if (PartTy.isVector() == SrcTy.isVector() && 396 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) { 397 assert(DstRegs.size() == 1); 398 B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg}); 399 return; 400 } 401 402 if (SrcTy.isVector() && !PartTy.isVector() && 403 PartSize > SrcTy.getElementType().getSizeInBits()) { 404 // Vector was scalarized, and the elements extended. 405 auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg); 406 for (int i = 0, e = DstRegs.size(); i != e; ++i) 407 B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i)); 408 return; 409 } 410 411 LLT GCDTy = getGCDType(SrcTy, PartTy); 412 if (GCDTy == PartTy) { 413 // If this already evenly divisible, we can create a simple unmerge. 414 B.buildUnmerge(DstRegs, SrcReg); 415 return; 416 } 417 418 MachineRegisterInfo &MRI = *B.getMRI(); 419 LLT DstTy = MRI.getType(DstRegs[0]); 420 LLT LCMTy = getLCMType(SrcTy, PartTy); 421 422 const unsigned LCMSize = LCMTy.getSizeInBits(); 423 const unsigned DstSize = DstTy.getSizeInBits(); 424 const unsigned SrcSize = SrcTy.getSizeInBits(); 425 426 Register UnmergeSrc = SrcReg; 427 if (LCMSize != SrcSize) { 428 // Widen to the common type. 429 Register Undef = B.buildUndef(SrcTy).getReg(0); 430 SmallVector<Register, 8> MergeParts(1, SrcReg); 431 for (unsigned Size = SrcSize; Size != LCMSize; Size += SrcSize) 432 MergeParts.push_back(Undef); 433 434 UnmergeSrc = B.buildMerge(LCMTy, MergeParts).getReg(0); 435 } 436 437 // Unmerge to the original registers and pad with dead defs. 438 SmallVector<Register, 8> UnmergeResults(DstRegs.begin(), DstRegs.end()); 439 for (unsigned Size = DstSize * DstRegs.size(); Size != LCMSize; 440 Size += DstSize) { 441 UnmergeResults.push_back(MRI.createGenericVirtualRegister(DstTy)); 442 } 443 444 B.buildUnmerge(UnmergeResults, UnmergeSrc); 445 } 446 447 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, 448 SmallVectorImpl<ArgInfo> &Args, 449 ValueHandler &Handler, 450 CallingConv::ID CallConv, bool IsVarArg, 451 Register ThisReturnReg) const { 452 MachineFunction &MF = MIRBuilder.getMF(); 453 const Function &F = MF.getFunction(); 454 SmallVector<CCValAssign, 16> ArgLocs; 455 456 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext()); 457 return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler, 458 ThisReturnReg); 459 } 460 461 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) { 462 if (Flags.isSExt()) 463 return TargetOpcode::G_SEXT; 464 if (Flags.isZExt()) 465 return TargetOpcode::G_ZEXT; 466 return TargetOpcode::G_ANYEXT; 467 } 468 469 bool CallLowering::handleAssignments(CCState &CCInfo, 470 SmallVectorImpl<CCValAssign> &ArgLocs, 471 MachineIRBuilder &MIRBuilder, 472 SmallVectorImpl<ArgInfo> &Args, 473 ValueHandler &Handler, 474 Register ThisReturnReg) const { 475 MachineFunction &MF = MIRBuilder.getMF(); 476 MachineRegisterInfo &MRI = MF.getRegInfo(); 477 const Function &F = MF.getFunction(); 478 const DataLayout &DL = F.getParent()->getDataLayout(); 479 480 unsigned NumArgs = Args.size(); 481 for (unsigned i = 0; i != NumArgs; ++i) { 482 EVT CurVT = EVT::getEVT(Args[i].Ty); 483 if (CurVT.isSimple() && 484 !Handler.assignArg(i, CurVT.getSimpleVT(), CurVT.getSimpleVT(), 485 CCValAssign::Full, Args[i], Args[i].Flags[0], 486 CCInfo)) 487 continue; 488 489 MVT NewVT = TLI->getRegisterTypeForCallingConv( 490 F.getContext(), CCInfo.getCallingConv(), EVT(CurVT)); 491 492 // If we need to split the type over multiple regs, check it's a scenario 493 // we currently support. 494 unsigned NumParts = TLI->getNumRegistersForCallingConv( 495 F.getContext(), CCInfo.getCallingConv(), CurVT); 496 497 if (NumParts == 1) { 498 // Try to use the register type if we couldn't assign the VT. 499 if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i], 500 Args[i].Flags[0], CCInfo)) 501 return false; 502 503 // If we couldn't directly assign this part, some casting may be 504 // necessary. Create the new register, but defer inserting the conversion 505 // instructions. 506 assert(Args[i].OrigRegs.empty()); 507 Args[i].OrigRegs.push_back(Args[i].Regs[0]); 508 assert(Args[i].Regs.size() == 1); 509 510 const LLT VATy(NewVT); 511 Args[i].Regs[0] = MRI.createGenericVirtualRegister(VATy); 512 continue; 513 } 514 515 const LLT NewLLT(NewVT); 516 517 // For incoming arguments (physregs to vregs), we could have values in 518 // physregs (or memlocs) which we want to extract and copy to vregs. 519 // During this, we might have to deal with the LLT being split across 520 // multiple regs, so we have to record this information for later. 521 // 522 // If we have outgoing args, then we have the opposite case. We have a 523 // vreg with an LLT which we want to assign to a physical location, and 524 // we might have to record that the value has to be split later. 525 if (Handler.isIncomingArgumentHandler()) { 526 // We're handling an incoming arg which is split over multiple regs. 527 // E.g. passing an s128 on AArch64. 528 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 529 Args[i].OrigRegs.push_back(Args[i].Regs[0]); 530 Args[i].Regs.clear(); 531 Args[i].Flags.clear(); 532 // For each split register, create and assign a vreg that will store 533 // the incoming component of the larger value. These will later be 534 // merged to form the final vreg. 535 for (unsigned Part = 0; Part < NumParts; ++Part) { 536 Register Reg = MRI.createGenericVirtualRegister(NewLLT); 537 ISD::ArgFlagsTy Flags = OrigFlags; 538 if (Part == 0) { 539 Flags.setSplit(); 540 } else { 541 Flags.setOrigAlign(Align(1)); 542 if (Part == NumParts - 1) 543 Flags.setSplitEnd(); 544 } 545 Args[i].Regs.push_back(Reg); 546 Args[i].Flags.push_back(Flags); 547 if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i], 548 Args[i].Flags[Part], CCInfo)) { 549 // Still couldn't assign this smaller part type for some reason. 550 return false; 551 } 552 } 553 } else { 554 assert(Args[i].Regs.size() == 1); 555 556 // This type is passed via multiple registers in the calling convention. 557 // We need to extract the individual parts. 558 assert(Args[i].OrigRegs.empty()); 559 Args[i].OrigRegs.push_back(Args[i].Regs[0]); 560 561 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 562 // We're going to replace the regs and flags with the split ones. 563 Args[i].Regs.clear(); 564 Args[i].Flags.clear(); 565 for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) { 566 ISD::ArgFlagsTy Flags = OrigFlags; 567 if (PartIdx == 0) { 568 Flags.setSplit(); 569 } else { 570 Flags.setOrigAlign(Align(1)); 571 if (PartIdx == NumParts - 1) 572 Flags.setSplitEnd(); 573 } 574 575 // TODO: Also check if there is a valid extension that preserves the 576 // bits. However currently this call lowering doesn't support non-exact 577 // split parts, so that can't be tested. 578 if (OrigFlags.isReturned() && 579 (NumParts * NewVT.getSizeInBits() != CurVT.getSizeInBits())) { 580 Flags.setReturned(false); 581 } 582 583 Register NewReg = MRI.createGenericVirtualRegister(NewLLT); 584 585 Args[i].Regs.push_back(NewReg); 586 Args[i].Flags.push_back(Flags); 587 if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, 588 Args[i], Args[i].Flags[PartIdx], CCInfo)) 589 return false; 590 } 591 } 592 } 593 594 for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) { 595 assert(j < ArgLocs.size() && "Skipped too many arg locs"); 596 597 CCValAssign &VA = ArgLocs[j]; 598 assert(VA.getValNo() == i && "Location doesn't correspond to current arg"); 599 600 if (VA.needsCustom()) { 601 unsigned NumArgRegs = 602 Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j)); 603 if (!NumArgRegs) 604 return false; 605 j += NumArgRegs; 606 continue; 607 } 608 609 EVT VAVT = VA.getValVT(); 610 const LLT OrigTy = getLLTForType(*Args[i].Ty, DL); 611 const LLT VATy(VAVT.getSimpleVT()); 612 613 // Expected to be multiple regs for a single incoming arg. 614 // There should be Regs.size() ArgLocs per argument. 615 unsigned NumArgRegs = Args[i].Regs.size(); 616 assert((j + (NumArgRegs - 1)) < ArgLocs.size() && 617 "Too many regs for number of args"); 618 619 // Coerce into outgoing value types before register assignment. 620 if (!Handler.isIncomingArgumentHandler() && OrigTy != VATy) { 621 assert(Args[i].OrigRegs.size() == 1); 622 buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy, 623 VATy, extendOpFromFlags(Args[i].Flags[0])); 624 } 625 626 for (unsigned Part = 0; Part < NumArgRegs; ++Part) { 627 Register ArgReg = Args[i].Regs[Part]; 628 // There should be Regs.size() ArgLocs per argument. 629 VA = ArgLocs[j + Part]; 630 if (VA.isMemLoc()) { 631 // Individual pieces may have been spilled to the stack and others 632 // passed in registers. 633 634 // FIXME: Use correct address space for pointer size 635 EVT LocVT = VA.getValVT(); 636 unsigned MemSize = LocVT == MVT::iPTR ? DL.getPointerSize() 637 : LocVT.getStoreSize(); 638 unsigned Offset = VA.getLocMemOffset(); 639 MachinePointerInfo MPO; 640 Register StackAddr = Handler.getStackAddress(MemSize, Offset, MPO); 641 Handler.assignValueToAddress(Args[i], Part, StackAddr, MemSize, MPO, 642 VA); 643 continue; 644 } 645 646 assert(VA.isRegLoc() && "custom loc should have been handled already"); 647 648 if (i == 0 && ThisReturnReg.isValid() && 649 Handler.isIncomingArgumentHandler() && 650 isTypeIsValidForThisReturn(VAVT)) { 651 Handler.assignValueToReg(Args[i].Regs[i], ThisReturnReg, VA); 652 continue; 653 } 654 655 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 656 } 657 658 // Now that all pieces have been assigned, re-pack the register typed values 659 // into the original value typed registers. 660 if (Handler.isIncomingArgumentHandler() && OrigTy != VATy) { 661 // Merge the split registers into the expected larger result vregs of 662 // the original call. 663 buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy, 664 VATy); 665 } 666 667 j += NumArgRegs - 1; 668 } 669 670 return true; 671 } 672 673 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, 674 ArrayRef<Register> VRegs, Register DemoteReg, 675 int FI) const { 676 MachineFunction &MF = MIRBuilder.getMF(); 677 MachineRegisterInfo &MRI = MF.getRegInfo(); 678 const DataLayout &DL = MF.getDataLayout(); 679 680 SmallVector<EVT, 4> SplitVTs; 681 SmallVector<uint64_t, 4> Offsets; 682 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 683 684 assert(VRegs.size() == SplitVTs.size()); 685 686 unsigned NumValues = SplitVTs.size(); 687 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 688 Type *RetPtrTy = RetTy->getPointerTo(DL.getAllocaAddrSpace()); 689 LLT OffsetLLTy = getLLTForType(*DL.getIntPtrType(RetPtrTy), DL); 690 691 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); 692 693 for (unsigned I = 0; I < NumValues; ++I) { 694 Register Addr; 695 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 696 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 697 MRI.getType(VRegs[I]).getSizeInBytes(), 698 commonAlignment(BaseAlign, Offsets[I])); 699 MIRBuilder.buildLoad(VRegs[I], Addr, *MMO); 700 } 701 } 702 703 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, 704 ArrayRef<Register> VRegs, 705 Register DemoteReg) const { 706 MachineFunction &MF = MIRBuilder.getMF(); 707 MachineRegisterInfo &MRI = MF.getRegInfo(); 708 const DataLayout &DL = MF.getDataLayout(); 709 710 SmallVector<EVT, 4> SplitVTs; 711 SmallVector<uint64_t, 4> Offsets; 712 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 713 714 assert(VRegs.size() == SplitVTs.size()); 715 716 unsigned NumValues = SplitVTs.size(); 717 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 718 unsigned AS = DL.getAllocaAddrSpace(); 719 LLT OffsetLLTy = 720 getLLTForType(*DL.getIntPtrType(RetTy->getPointerTo(AS)), DL); 721 722 MachinePointerInfo PtrInfo(AS); 723 724 for (unsigned I = 0; I < NumValues; ++I) { 725 Register Addr; 726 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 727 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 728 MRI.getType(VRegs[I]).getSizeInBytes(), 729 commonAlignment(BaseAlign, Offsets[I])); 730 MIRBuilder.buildStore(VRegs[I], Addr, *MMO); 731 } 732 } 733 734 void CallLowering::insertSRetIncomingArgument( 735 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg, 736 MachineRegisterInfo &MRI, const DataLayout &DL) const { 737 unsigned AS = DL.getAllocaAddrSpace(); 738 DemoteReg = MRI.createGenericVirtualRegister( 739 LLT::pointer(AS, DL.getPointerSizeInBits(AS))); 740 741 Type *PtrTy = PointerType::get(F.getReturnType(), AS); 742 743 SmallVector<EVT, 1> ValueVTs; 744 ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs); 745 746 // NOTE: Assume that a pointer won't get split into more than one VT. 747 assert(ValueVTs.size() == 1); 748 749 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext())); 750 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F); 751 DemoteArg.Flags[0].setSRet(); 752 SplitArgs.insert(SplitArgs.begin(), DemoteArg); 753 } 754 755 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder, 756 const CallBase &CB, 757 CallLoweringInfo &Info) const { 758 const DataLayout &DL = MIRBuilder.getDataLayout(); 759 Type *RetTy = CB.getType(); 760 unsigned AS = DL.getAllocaAddrSpace(); 761 LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); 762 763 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject( 764 DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false); 765 766 Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0); 767 ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS)); 768 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB); 769 DemoteArg.Flags[0].setSRet(); 770 771 Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg); 772 Info.DemoteStackIndex = FI; 773 Info.DemoteRegister = DemoteReg; 774 } 775 776 bool CallLowering::checkReturn(CCState &CCInfo, 777 SmallVectorImpl<BaseArgInfo> &Outs, 778 CCAssignFn *Fn) const { 779 for (unsigned I = 0, E = Outs.size(); I < E; ++I) { 780 MVT VT = MVT::getVT(Outs[I].Ty); 781 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo)) 782 return false; 783 } 784 return true; 785 } 786 787 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy, 788 AttributeList Attrs, 789 SmallVectorImpl<BaseArgInfo> &Outs, 790 const DataLayout &DL) const { 791 LLVMContext &Context = RetTy->getContext(); 792 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 793 794 SmallVector<EVT, 4> SplitVTs; 795 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs); 796 addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex); 797 798 for (EVT VT : SplitVTs) { 799 unsigned NumParts = 800 TLI->getNumRegistersForCallingConv(Context, CallConv, VT); 801 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT); 802 Type *PartTy = EVT(RegVT).getTypeForEVT(Context); 803 804 for (unsigned I = 0; I < NumParts; ++I) { 805 Outs.emplace_back(PartTy, Flags); 806 } 807 } 808 } 809 810 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const { 811 const auto &F = MF.getFunction(); 812 Type *ReturnType = F.getReturnType(); 813 CallingConv::ID CallConv = F.getCallingConv(); 814 815 SmallVector<BaseArgInfo, 4> SplitArgs; 816 getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs, 817 MF.getDataLayout()); 818 return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg()); 819 } 820 821 bool CallLowering::analyzeArgInfo(CCState &CCState, 822 SmallVectorImpl<ArgInfo> &Args, 823 CCAssignFn &AssignFnFixed, 824 CCAssignFn &AssignFnVarArg) const { 825 for (unsigned i = 0, e = Args.size(); i < e; ++i) { 826 MVT VT = MVT::getVT(Args[i].Ty); 827 CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg; 828 if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) { 829 // Bail out on anything we can't handle. 830 LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString() 831 << " (arg number = " << i << "\n"); 832 return false; 833 } 834 } 835 return true; 836 } 837 838 bool CallLowering::parametersInCSRMatch( 839 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, 840 const SmallVectorImpl<CCValAssign> &OutLocs, 841 const SmallVectorImpl<ArgInfo> &OutArgs) const { 842 for (unsigned i = 0; i < OutLocs.size(); ++i) { 843 auto &ArgLoc = OutLocs[i]; 844 // If it's not a register, it's fine. 845 if (!ArgLoc.isRegLoc()) 846 continue; 847 848 MCRegister PhysReg = ArgLoc.getLocReg(); 849 850 // Only look at callee-saved registers. 851 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg)) 852 continue; 853 854 LLVM_DEBUG( 855 dbgs() 856 << "... Call has an argument passed in a callee-saved register.\n"); 857 858 // Check if it was copied from. 859 const ArgInfo &OutInfo = OutArgs[i]; 860 861 if (OutInfo.Regs.size() > 1) { 862 LLVM_DEBUG( 863 dbgs() << "... Cannot handle arguments in multiple registers.\n"); 864 return false; 865 } 866 867 // Check if we copy the register, walking through copies from virtual 868 // registers. Note that getDefIgnoringCopies does not ignore copies from 869 // physical registers. 870 MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI); 871 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) { 872 LLVM_DEBUG( 873 dbgs() 874 << "... Parameter was not copied into a VReg, cannot tail call.\n"); 875 return false; 876 } 877 878 // Got a copy. Verify that it's the same as the register we want. 879 Register CopyRHS = RegDef->getOperand(1).getReg(); 880 if (CopyRHS != PhysReg) { 881 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into " 882 "VReg, cannot tail call.\n"); 883 return false; 884 } 885 } 886 887 return true; 888 } 889 890 bool CallLowering::resultsCompatible(CallLoweringInfo &Info, 891 MachineFunction &MF, 892 SmallVectorImpl<ArgInfo> &InArgs, 893 CCAssignFn &CalleeAssignFnFixed, 894 CCAssignFn &CalleeAssignFnVarArg, 895 CCAssignFn &CallerAssignFnFixed, 896 CCAssignFn &CallerAssignFnVarArg) const { 897 const Function &F = MF.getFunction(); 898 CallingConv::ID CalleeCC = Info.CallConv; 899 CallingConv::ID CallerCC = F.getCallingConv(); 900 901 if (CallerCC == CalleeCC) 902 return true; 903 904 SmallVector<CCValAssign, 16> ArgLocs1; 905 CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext()); 906 if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed, 907 CalleeAssignFnVarArg)) 908 return false; 909 910 SmallVector<CCValAssign, 16> ArgLocs2; 911 CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext()); 912 if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed, 913 CalleeAssignFnVarArg)) 914 return false; 915 916 // We need the argument locations to match up exactly. If there's more in 917 // one than the other, then we are done. 918 if (ArgLocs1.size() != ArgLocs2.size()) 919 return false; 920 921 // Make sure that each location is passed in exactly the same way. 922 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { 923 const CCValAssign &Loc1 = ArgLocs1[i]; 924 const CCValAssign &Loc2 = ArgLocs2[i]; 925 926 // We need both of them to be the same. So if one is a register and one 927 // isn't, we're done. 928 if (Loc1.isRegLoc() != Loc2.isRegLoc()) 929 return false; 930 931 if (Loc1.isRegLoc()) { 932 // If they don't have the same register location, we're done. 933 if (Loc1.getLocReg() != Loc2.getLocReg()) 934 return false; 935 936 // They matched, so we can move to the next ArgLoc. 937 continue; 938 } 939 940 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. 941 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) 942 return false; 943 } 944 945 return true; 946 } 947 948 Register CallLowering::ValueHandler::extendRegister(Register ValReg, 949 CCValAssign &VA, 950 unsigned MaxSizeBits) { 951 LLT LocTy{VA.getLocVT()}; 952 LLT ValTy = MRI.getType(ValReg); 953 if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) 954 return ValReg; 955 956 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { 957 if (MaxSizeBits <= ValTy.getSizeInBits()) 958 return ValReg; 959 LocTy = LLT::scalar(MaxSizeBits); 960 } 961 962 switch (VA.getLocInfo()) { 963 default: break; 964 case CCValAssign::Full: 965 case CCValAssign::BCvt: 966 // FIXME: bitconverting between vector types may or may not be a 967 // nop in big-endian situations. 968 return ValReg; 969 case CCValAssign::AExt: { 970 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg); 971 return MIB.getReg(0); 972 } 973 case CCValAssign::SExt: { 974 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 975 MIRBuilder.buildSExt(NewReg, ValReg); 976 return NewReg; 977 } 978 case CCValAssign::ZExt: { 979 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 980 MIRBuilder.buildZExt(NewReg, ValReg); 981 return NewReg; 982 } 983 } 984 llvm_unreachable("unable to extend register"); 985 } 986 987 void CallLowering::ValueHandler::anchor() {} 988 989 Register CallLowering::IncomingValueHandler::buildExtensionHint(CCValAssign &VA, 990 Register SrcReg, 991 LLT NarrowTy) { 992 switch (VA.getLocInfo()) { 993 case CCValAssign::LocInfo::ZExt: { 994 return MIRBuilder 995 .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 996 NarrowTy.getScalarSizeInBits()) 997 .getReg(0); 998 } 999 case CCValAssign::LocInfo::SExt: { 1000 return MIRBuilder 1001 .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 1002 NarrowTy.getScalarSizeInBits()) 1003 .getReg(0); 1004 break; 1005 } 1006 default: 1007 return SrcReg; 1008 } 1009 } 1010 1011 void CallLowering::IncomingValueHandler::assignValueToReg(Register ValVReg, 1012 Register PhysReg, 1013 CCValAssign &VA) { 1014 const LLT LocTy(VA.getLocVT()); 1015 const LLT ValTy = MRI.getType(ValVReg); 1016 1017 if (ValTy.getSizeInBits() == LocTy.getSizeInBits()) { 1018 MIRBuilder.buildCopy(ValVReg, PhysReg); 1019 return; 1020 } 1021 1022 auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg); 1023 auto Hint = buildExtensionHint(VA, Copy.getReg(0), ValTy); 1024 MIRBuilder.buildTrunc(ValVReg, Hint); 1025 } 1026