1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements some simple delegations needed for call lowering. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 15 #include "llvm/CodeGen/Analysis.h" 16 #include "llvm/CodeGen/CallingConvLower.h" 17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 18 #include "llvm/CodeGen/GlobalISel/Utils.h" 19 #include "llvm/CodeGen/MachineFrameInfo.h" 20 #include "llvm/CodeGen/MachineOperand.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/TargetLowering.h" 23 #include "llvm/IR/DataLayout.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/LLVMContext.h" 26 #include "llvm/IR/Module.h" 27 #include "llvm/Target/TargetMachine.h" 28 29 #define DEBUG_TYPE "call-lowering" 30 31 using namespace llvm; 32 33 void CallLowering::anchor() {} 34 35 /// Helper function which updates \p Flags when \p AttrFn returns true. 36 static void 37 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, 38 const std::function<bool(Attribute::AttrKind)> &AttrFn) { 39 if (AttrFn(Attribute::SExt)) 40 Flags.setSExt(); 41 if (AttrFn(Attribute::ZExt)) 42 Flags.setZExt(); 43 if (AttrFn(Attribute::InReg)) 44 Flags.setInReg(); 45 if (AttrFn(Attribute::StructRet)) 46 Flags.setSRet(); 47 if (AttrFn(Attribute::Nest)) 48 Flags.setNest(); 49 if (AttrFn(Attribute::ByVal)) 50 Flags.setByVal(); 51 if (AttrFn(Attribute::Preallocated)) 52 Flags.setPreallocated(); 53 if (AttrFn(Attribute::InAlloca)) 54 Flags.setInAlloca(); 55 if (AttrFn(Attribute::Returned)) 56 Flags.setReturned(); 57 if (AttrFn(Attribute::SwiftSelf)) 58 Flags.setSwiftSelf(); 59 if (AttrFn(Attribute::SwiftAsync)) 60 Flags.setSwiftAsync(); 61 if (AttrFn(Attribute::SwiftError)) 62 Flags.setSwiftError(); 63 } 64 65 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, 66 unsigned ArgIdx) const { 67 ISD::ArgFlagsTy Flags; 68 addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) { 69 return Call.paramHasAttr(ArgIdx, Attr); 70 }); 71 return Flags; 72 } 73 74 ISD::ArgFlagsTy 75 CallLowering::getAttributesForReturn(const CallBase &Call) const { 76 ISD::ArgFlagsTy Flags; 77 addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) { 78 return Call.hasRetAttr(Attr); 79 }); 80 return Flags; 81 } 82 83 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, 84 const AttributeList &Attrs, 85 unsigned OpIdx) const { 86 addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) { 87 return Attrs.hasAttributeAtIndex(OpIdx, Attr); 88 }); 89 } 90 91 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, 92 ArrayRef<Register> ResRegs, 93 ArrayRef<ArrayRef<Register>> ArgRegs, 94 Register SwiftErrorVReg, 95 std::optional<PtrAuthInfo> PAI, 96 Register ConvergenceCtrlToken, 97 std::function<unsigned()> GetCalleeReg) const { 98 CallLoweringInfo Info; 99 const DataLayout &DL = MIRBuilder.getDataLayout(); 100 MachineFunction &MF = MIRBuilder.getMF(); 101 MachineRegisterInfo &MRI = MF.getRegInfo(); 102 bool CanBeTailCalled = CB.isTailCall() && 103 isInTailCallPosition(CB, MF.getTarget()) && 104 (MF.getFunction() 105 .getFnAttribute("disable-tail-calls") 106 .getValueAsString() != "true"); 107 108 CallingConv::ID CallConv = CB.getCallingConv(); 109 Type *RetTy = CB.getType(); 110 bool IsVarArg = CB.getFunctionType()->isVarArg(); 111 112 SmallVector<BaseArgInfo, 4> SplitArgs; 113 getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL); 114 Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg); 115 116 Info.IsConvergent = CB.isConvergent(); 117 118 if (!Info.CanLowerReturn) { 119 // Callee requires sret demotion. 120 insertSRetOutgoingArgument(MIRBuilder, CB, Info); 121 122 // The sret demotion isn't compatible with tail-calls, since the sret 123 // argument points into the caller's stack frame. 124 CanBeTailCalled = false; 125 } 126 127 // First step is to marshall all the function's parameters into the correct 128 // physregs and memory locations. Gather the sequence of argument types that 129 // we'll pass to the assigner function. 130 unsigned i = 0; 131 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); 132 for (const auto &Arg : CB.args()) { 133 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i), 134 i < NumFixedArgs}; 135 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB); 136 137 // If we have an explicit sret argument that is an Instruction, (i.e., it 138 // might point to function-local memory), we can't meaningfully tail-call. 139 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg)) 140 CanBeTailCalled = false; 141 142 Info.OrigArgs.push_back(OrigArg); 143 ++i; 144 } 145 146 // Try looking through a bitcast from one function type to another. 147 // Commonly happens with calls to objc_msgSend(). 148 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); 149 if (const Function *F = dyn_cast<Function>(CalleeV)) { 150 if (F->hasFnAttribute(Attribute::NonLazyBind)) { 151 LLT Ty = getLLTForType(*F->getType(), DL); 152 Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0); 153 Info.Callee = MachineOperand::CreateReg(Reg, false); 154 } else { 155 Info.Callee = MachineOperand::CreateGA(F, 0); 156 } 157 } else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) { 158 // IR IFuncs and Aliases can't be forward declared (only defined), so the 159 // callee must be in the same TU and therefore we can direct-call it without 160 // worrying about it being out of range. 161 Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0); 162 } else 163 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); 164 165 Register ReturnHintAlignReg; 166 Align ReturnHintAlign; 167 168 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)}; 169 170 if (!Info.OrigRet.Ty->isVoidTy()) { 171 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB); 172 173 if (MaybeAlign Alignment = CB.getRetAlign()) { 174 if (*Alignment > Align(1)) { 175 ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]); 176 Info.OrigRet.Regs[0] = ReturnHintAlignReg; 177 ReturnHintAlign = *Alignment; 178 } 179 } 180 } 181 182 auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi); 183 if (Bundle && CB.isIndirectCall()) { 184 Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]); 185 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); 186 } 187 188 Info.CB = &CB; 189 Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); 190 Info.CallConv = CallConv; 191 Info.SwiftErrorVReg = SwiftErrorVReg; 192 Info.PAI = PAI; 193 Info.ConvergenceCtrlToken = ConvergenceCtrlToken; 194 Info.IsMustTailCall = CB.isMustTailCall(); 195 Info.IsTailCall = CanBeTailCalled; 196 Info.IsVarArg = IsVarArg; 197 if (!lowerCall(MIRBuilder, Info)) 198 return false; 199 200 if (ReturnHintAlignReg && !Info.LoweredTailCall) { 201 MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg, 202 ReturnHintAlign); 203 } 204 205 return true; 206 } 207 208 template <typename FuncInfoTy> 209 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, 210 const DataLayout &DL, 211 const FuncInfoTy &FuncInfo) const { 212 auto &Flags = Arg.Flags[0]; 213 const AttributeList &Attrs = FuncInfo.getAttributes(); 214 addArgFlagsFromAttributes(Flags, Attrs, OpIdx); 215 216 PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType()); 217 if (PtrTy) { 218 Flags.setPointer(); 219 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace()); 220 } 221 222 Align MemAlign = DL.getABITypeAlign(Arg.Ty); 223 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { 224 assert(OpIdx >= AttributeList::FirstArgIndex); 225 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; 226 227 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); 228 if (!ElementTy) 229 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); 230 if (!ElementTy) 231 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); 232 assert(ElementTy && "Must have byval, inalloca or preallocated type"); 233 Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); 234 235 // For ByVal, alignment should be passed from FE. BE will guess if 236 // this info is not there but there are cases it cannot get right. 237 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx)) 238 MemAlign = *ParamAlign; 239 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx))) 240 MemAlign = *ParamAlign; 241 else 242 MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL)); 243 } else if (OpIdx >= AttributeList::FirstArgIndex) { 244 if (auto ParamAlign = 245 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex)) 246 MemAlign = *ParamAlign; 247 } 248 Flags.setMemAlign(MemAlign); 249 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty)); 250 251 // Don't try to use the returned attribute if the argument is marked as 252 // swiftself, since it won't be passed in x0. 253 if (Flags.isSwiftSelf()) 254 Flags.setReturned(false); 255 } 256 257 template void 258 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 259 const DataLayout &DL, 260 const Function &FuncInfo) const; 261 262 template void 263 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 264 const DataLayout &DL, 265 const CallBase &FuncInfo) const; 266 267 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 268 SmallVectorImpl<ArgInfo> &SplitArgs, 269 const DataLayout &DL, 270 CallingConv::ID CallConv, 271 SmallVectorImpl<uint64_t> *Offsets) const { 272 LLVMContext &Ctx = OrigArg.Ty->getContext(); 273 274 SmallVector<EVT, 4> SplitVTs; 275 ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0); 276 277 if (SplitVTs.size() == 0) 278 return; 279 280 if (SplitVTs.size() == 1) { 281 // No splitting to do, but we want to replace the original type (e.g. [1 x 282 // double] -> double). 283 SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), 284 OrigArg.OrigArgIndex, OrigArg.Flags[0], 285 OrigArg.IsFixed, OrigArg.OrigValue); 286 return; 287 } 288 289 // Create one ArgInfo for each virtual register in the original ArgInfo. 290 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch"); 291 292 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( 293 OrigArg.Ty, CallConv, false, DL); 294 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { 295 Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); 296 SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex, 297 OrigArg.Flags[0], OrigArg.IsFixed); 298 if (NeedsRegBlock) 299 SplitArgs.back().Flags[0].setInConsecutiveRegs(); 300 } 301 302 SplitArgs.back().Flags[0].setInConsecutiveRegsLast(); 303 } 304 305 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs. 306 static MachineInstrBuilder 307 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 308 ArrayRef<Register> SrcRegs) { 309 MachineRegisterInfo &MRI = *B.getMRI(); 310 LLT LLTy = MRI.getType(DstRegs[0]); 311 LLT PartLLT = MRI.getType(SrcRegs[0]); 312 313 // Deal with v3s16 split into v2s16 314 LLT LCMTy = getCoverTy(LLTy, PartLLT); 315 if (LCMTy == LLTy) { 316 // Common case where no padding is needed. 317 assert(DstRegs.size() == 1); 318 return B.buildConcatVectors(DstRegs[0], SrcRegs); 319 } 320 321 // We need to create an unmerge to the result registers, which may require 322 // widening the original value. 323 Register UnmergeSrcReg; 324 if (LCMTy != PartLLT) { 325 assert(DstRegs.size() == 1); 326 return B.buildDeleteTrailingVectorElements( 327 DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs)); 328 } else { 329 // We don't need to widen anything if we're extracting a scalar which was 330 // promoted to a vector e.g. s8 -> v4s8 -> s8 331 assert(SrcRegs.size() == 1); 332 UnmergeSrcReg = SrcRegs[0]; 333 } 334 335 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits(); 336 337 SmallVector<Register, 8> PadDstRegs(NumDst); 338 std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin()); 339 340 // Create the excess dead defs for the unmerge. 341 for (int I = DstRegs.size(); I != NumDst; ++I) 342 PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy); 343 344 if (PadDstRegs.size() == 1) 345 return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg); 346 return B.buildUnmerge(PadDstRegs, UnmergeSrcReg); 347 } 348 349 /// Create a sequence of instructions to combine pieces split into register 350 /// typed values to the original IR value. \p OrigRegs contains the destination 351 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces 352 /// with type \p PartLLT. This is used for incoming values (physregs to vregs). 353 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs, 354 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, 355 const ISD::ArgFlagsTy Flags) { 356 MachineRegisterInfo &MRI = *B.getMRI(); 357 358 if (PartLLT == LLTy) { 359 // We should have avoided introducing a new virtual register, and just 360 // directly assigned here. 361 assert(OrigRegs[0] == Regs[0]); 362 return; 363 } 364 365 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 && 366 Regs.size() == 1) { 367 B.buildBitcast(OrigRegs[0], Regs[0]); 368 return; 369 } 370 371 // A vector PartLLT needs extending to LLTy's element size. 372 // E.g. <2 x s64> = G_SEXT <2 x s32>. 373 if (PartLLT.isVector() == LLTy.isVector() && 374 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() && 375 (!PartLLT.isVector() || 376 PartLLT.getElementCount() == LLTy.getElementCount()) && 377 OrigRegs.size() == 1 && Regs.size() == 1) { 378 Register SrcReg = Regs[0]; 379 380 LLT LocTy = MRI.getType(SrcReg); 381 382 if (Flags.isSExt()) { 383 SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits()) 384 .getReg(0); 385 } else if (Flags.isZExt()) { 386 SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits()) 387 .getReg(0); 388 } 389 390 // Sometimes pointers are passed zero extended. 391 LLT OrigTy = MRI.getType(OrigRegs[0]); 392 if (OrigTy.isPointer()) { 393 LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits()); 394 B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg)); 395 return; 396 } 397 398 B.buildTrunc(OrigRegs[0], SrcReg); 399 return; 400 } 401 402 if (!LLTy.isVector() && !PartLLT.isVector()) { 403 assert(OrigRegs.size() == 1); 404 LLT OrigTy = MRI.getType(OrigRegs[0]); 405 406 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size(); 407 if (SrcSize == OrigTy.getSizeInBits()) 408 B.buildMergeValues(OrigRegs[0], Regs); 409 else { 410 auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs); 411 B.buildTrunc(OrigRegs[0], Widened); 412 } 413 414 return; 415 } 416 417 if (PartLLT.isVector()) { 418 assert(OrigRegs.size() == 1); 419 SmallVector<Register> CastRegs(Regs.begin(), Regs.end()); 420 421 // If PartLLT is a mismatched vector in both number of elements and element 422 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to 423 // have the same elt type, i.e. v4s32. 424 // TODO: Extend this coersion to element multiples other than just 2. 425 if (TypeSize::isKnownGT(PartLLT.getSizeInBits(), LLTy.getSizeInBits()) && 426 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 && 427 Regs.size() == 1) { 428 LLT NewTy = PartLLT.changeElementType(LLTy.getElementType()) 429 .changeElementCount(PartLLT.getElementCount() * 2); 430 CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0); 431 PartLLT = NewTy; 432 } 433 434 if (LLTy.getScalarType() == PartLLT.getElementType()) { 435 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs); 436 } else { 437 unsigned I = 0; 438 LLT GCDTy = getGCDType(LLTy, PartLLT); 439 440 // We are both splitting a vector, and bitcasting its element types. Cast 441 // the source pieces into the appropriate number of pieces with the result 442 // element type. 443 for (Register SrcReg : CastRegs) 444 CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0); 445 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs); 446 } 447 448 return; 449 } 450 451 assert(LLTy.isVector() && !PartLLT.isVector()); 452 453 LLT DstEltTy = LLTy.getElementType(); 454 455 // Pointer information was discarded. We'll need to coerce some register types 456 // to avoid violating type constraints. 457 LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType(); 458 459 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); 460 461 if (DstEltTy == PartLLT) { 462 // Vector was trivially scalarized. 463 464 if (RealDstEltTy.isPointer()) { 465 for (Register Reg : Regs) 466 MRI.setType(Reg, RealDstEltTy); 467 } 468 469 B.buildBuildVector(OrigRegs[0], Regs); 470 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { 471 // Deal with vector with 64-bit elements decomposed to 32-bit 472 // registers. Need to create intermediate 64-bit elements. 473 SmallVector<Register, 8> EltMerges; 474 int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits(); 475 476 assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); 477 478 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { 479 auto Merge = 480 B.buildMergeLikeInstr(RealDstEltTy, Regs.take_front(PartsPerElt)); 481 // Fix the type in case this is really a vector of pointers. 482 MRI.setType(Merge.getReg(0), RealDstEltTy); 483 EltMerges.push_back(Merge.getReg(0)); 484 Regs = Regs.drop_front(PartsPerElt); 485 } 486 487 B.buildBuildVector(OrigRegs[0], EltMerges); 488 } else { 489 // Vector was split, and elements promoted to a wider type. 490 // FIXME: Should handle floating point promotions. 491 unsigned NumElts = LLTy.getNumElements(); 492 LLT BVType = LLT::fixed_vector(NumElts, PartLLT); 493 494 Register BuildVec; 495 if (NumElts == Regs.size()) 496 BuildVec = B.buildBuildVector(BVType, Regs).getReg(0); 497 else { 498 // Vector elements are packed in the inputs. 499 // e.g. we have a <4 x s16> but 2 x s32 in regs. 500 assert(NumElts > Regs.size()); 501 LLT SrcEltTy = MRI.getType(Regs[0]); 502 503 LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType(); 504 505 // Input registers contain packed elements. 506 // Determine how many elements per reg. 507 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0); 508 unsigned EltPerReg = 509 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits()); 510 511 SmallVector<Register, 0> BVRegs; 512 BVRegs.reserve(Regs.size() * EltPerReg); 513 for (Register R : Regs) { 514 auto Unmerge = B.buildUnmerge(OriginalEltTy, R); 515 for (unsigned K = 0; K < EltPerReg; ++K) 516 BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0)); 517 } 518 519 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces 520 // for a <3 x s16> vector. We should have less than EltPerReg extra items. 521 if (BVRegs.size() > NumElts) { 522 assert((BVRegs.size() - NumElts) < EltPerReg); 523 BVRegs.truncate(NumElts); 524 } 525 BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0); 526 } 527 B.buildTrunc(OrigRegs[0], BuildVec); 528 } 529 } 530 531 /// Create a sequence of instructions to expand the value in \p SrcReg (of type 532 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should 533 /// contain the type of scalar value extension if necessary. 534 /// 535 /// This is used for outgoing values (vregs to physregs) 536 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 537 Register SrcReg, LLT SrcTy, LLT PartTy, 538 unsigned ExtendOp = TargetOpcode::G_ANYEXT) { 539 // We could just insert a regular copy, but this is unreachable at the moment. 540 assert(SrcTy != PartTy && "identical part types shouldn't reach here"); 541 542 const TypeSize PartSize = PartTy.getSizeInBits(); 543 544 if (PartTy.isVector() == SrcTy.isVector() && 545 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) { 546 assert(DstRegs.size() == 1); 547 B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg}); 548 return; 549 } 550 551 if (SrcTy.isVector() && !PartTy.isVector() && 552 TypeSize::isKnownGT(PartSize, SrcTy.getElementType().getSizeInBits())) { 553 // Vector was scalarized, and the elements extended. 554 auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg); 555 for (int i = 0, e = DstRegs.size(); i != e; ++i) 556 B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i)); 557 return; 558 } 559 560 if (SrcTy.isVector() && PartTy.isVector() && 561 PartTy.getSizeInBits() == SrcTy.getSizeInBits() && 562 ElementCount::isKnownLT(SrcTy.getElementCount(), 563 PartTy.getElementCount())) { 564 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32 565 Register DstReg = DstRegs.front(); 566 B.buildPadVectorWithUndefElements(DstReg, SrcReg); 567 return; 568 } 569 570 LLT GCDTy = getGCDType(SrcTy, PartTy); 571 if (GCDTy == PartTy) { 572 // If this already evenly divisible, we can create a simple unmerge. 573 B.buildUnmerge(DstRegs, SrcReg); 574 return; 575 } 576 577 MachineRegisterInfo &MRI = *B.getMRI(); 578 LLT DstTy = MRI.getType(DstRegs[0]); 579 LLT LCMTy = getCoverTy(SrcTy, PartTy); 580 581 if (PartTy.isVector() && LCMTy == PartTy) { 582 assert(DstRegs.size() == 1); 583 B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg); 584 return; 585 } 586 587 const unsigned DstSize = DstTy.getSizeInBits(); 588 const unsigned SrcSize = SrcTy.getSizeInBits(); 589 unsigned CoveringSize = LCMTy.getSizeInBits(); 590 591 Register UnmergeSrc = SrcReg; 592 593 if (!LCMTy.isVector() && CoveringSize != SrcSize) { 594 // For scalars, it's common to be able to use a simple extension. 595 if (SrcTy.isScalar() && DstTy.isScalar()) { 596 CoveringSize = alignTo(SrcSize, DstSize); 597 LLT CoverTy = LLT::scalar(CoveringSize); 598 UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0); 599 } else { 600 // Widen to the common type. 601 // FIXME: This should respect the extend type 602 Register Undef = B.buildUndef(SrcTy).getReg(0); 603 SmallVector<Register, 8> MergeParts(1, SrcReg); 604 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize) 605 MergeParts.push_back(Undef); 606 UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0); 607 } 608 } 609 610 if (LCMTy.isVector() && CoveringSize != SrcSize) 611 UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0); 612 613 B.buildUnmerge(DstRegs, UnmergeSrc); 614 } 615 616 bool CallLowering::determineAndHandleAssignments( 617 ValueHandler &Handler, ValueAssigner &Assigner, 618 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder, 619 CallingConv::ID CallConv, bool IsVarArg, 620 ArrayRef<Register> ThisReturnRegs) const { 621 MachineFunction &MF = MIRBuilder.getMF(); 622 const Function &F = MF.getFunction(); 623 SmallVector<CCValAssign, 16> ArgLocs; 624 625 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext()); 626 if (!determineAssignments(Assigner, Args, CCInfo)) 627 return false; 628 629 return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder, 630 ThisReturnRegs); 631 } 632 633 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) { 634 if (Flags.isSExt()) 635 return TargetOpcode::G_SEXT; 636 if (Flags.isZExt()) 637 return TargetOpcode::G_ZEXT; 638 return TargetOpcode::G_ANYEXT; 639 } 640 641 bool CallLowering::determineAssignments(ValueAssigner &Assigner, 642 SmallVectorImpl<ArgInfo> &Args, 643 CCState &CCInfo) const { 644 LLVMContext &Ctx = CCInfo.getContext(); 645 const CallingConv::ID CallConv = CCInfo.getCallingConv(); 646 647 unsigned NumArgs = Args.size(); 648 for (unsigned i = 0; i != NumArgs; ++i) { 649 EVT CurVT = EVT::getEVT(Args[i].Ty); 650 651 MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT); 652 653 // If we need to split the type over multiple regs, check it's a scenario 654 // we currently support. 655 unsigned NumParts = 656 TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT); 657 658 if (NumParts == 1) { 659 // Try to use the register type if we couldn't assign the VT. 660 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i], 661 Args[i].Flags[0], CCInfo)) 662 return false; 663 continue; 664 } 665 666 // For incoming arguments (physregs to vregs), we could have values in 667 // physregs (or memlocs) which we want to extract and copy to vregs. 668 // During this, we might have to deal with the LLT being split across 669 // multiple regs, so we have to record this information for later. 670 // 671 // If we have outgoing args, then we have the opposite case. We have a 672 // vreg with an LLT which we want to assign to a physical location, and 673 // we might have to record that the value has to be split later. 674 675 // We're handling an incoming arg which is split over multiple regs. 676 // E.g. passing an s128 on AArch64. 677 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 678 Args[i].Flags.clear(); 679 680 for (unsigned Part = 0; Part < NumParts; ++Part) { 681 ISD::ArgFlagsTy Flags = OrigFlags; 682 if (Part == 0) { 683 Flags.setSplit(); 684 } else { 685 Flags.setOrigAlign(Align(1)); 686 if (Part == NumParts - 1) 687 Flags.setSplitEnd(); 688 } 689 690 Args[i].Flags.push_back(Flags); 691 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i], 692 Args[i].Flags[Part], CCInfo)) { 693 // Still couldn't assign this smaller part type for some reason. 694 return false; 695 } 696 } 697 } 698 699 return true; 700 } 701 702 bool CallLowering::handleAssignments(ValueHandler &Handler, 703 SmallVectorImpl<ArgInfo> &Args, 704 CCState &CCInfo, 705 SmallVectorImpl<CCValAssign> &ArgLocs, 706 MachineIRBuilder &MIRBuilder, 707 ArrayRef<Register> ThisReturnRegs) const { 708 MachineFunction &MF = MIRBuilder.getMF(); 709 MachineRegisterInfo &MRI = MF.getRegInfo(); 710 const Function &F = MF.getFunction(); 711 const DataLayout &DL = F.getParent()->getDataLayout(); 712 713 const unsigned NumArgs = Args.size(); 714 715 // Stores thunks for outgoing register assignments. This is used so we delay 716 // generating register copies until mem loc assignments are done. We do this 717 // so that if the target is using the delayed stack protector feature, we can 718 // find the split point of the block accurately. E.g. if we have: 719 // G_STORE %val, %memloc 720 // $x0 = COPY %foo 721 // $x1 = COPY %bar 722 // CALL func 723 // ... then the split point for the block will correctly be at, and including, 724 // the copy to $x0. If instead the G_STORE instruction immediately precedes 725 // the CALL, then we'd prematurely choose the CALL as the split point, thus 726 // generating a split block with a CALL that uses undefined physregs. 727 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments; 728 729 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) { 730 assert(j < ArgLocs.size() && "Skipped too many arg locs"); 731 CCValAssign &VA = ArgLocs[j]; 732 assert(VA.getValNo() == i && "Location doesn't correspond to current arg"); 733 734 if (VA.needsCustom()) { 735 std::function<void()> Thunk; 736 unsigned NumArgRegs = Handler.assignCustomValue( 737 Args[i], ArrayRef(ArgLocs).slice(j), &Thunk); 738 if (Thunk) 739 DelayedOutgoingRegAssignments.emplace_back(Thunk); 740 if (!NumArgRegs) 741 return false; 742 j += (NumArgRegs - 1); 743 continue; 744 } 745 746 const MVT ValVT = VA.getValVT(); 747 const MVT LocVT = VA.getLocVT(); 748 749 const LLT LocTy(LocVT); 750 const LLT ValTy(ValVT); 751 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy; 752 const EVT OrigVT = EVT::getEVT(Args[i].Ty); 753 const LLT OrigTy = getLLTForType(*Args[i].Ty, DL); 754 755 // Expected to be multiple regs for a single incoming arg. 756 // There should be Regs.size() ArgLocs per argument. 757 // This should be the same as getNumRegistersForCallingConv 758 const unsigned NumParts = Args[i].Flags.size(); 759 760 // Now split the registers into the assigned types. 761 Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end()); 762 763 if (NumParts != 1 || NewLLT != OrigTy) { 764 // If we can't directly assign the register, we need one or more 765 // intermediate values. 766 Args[i].Regs.resize(NumParts); 767 768 // For each split register, create and assign a vreg that will store 769 // the incoming component of the larger value. These will later be 770 // merged to form the final vreg. 771 for (unsigned Part = 0; Part < NumParts; ++Part) 772 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT); 773 } 774 775 assert((j + (NumParts - 1)) < ArgLocs.size() && 776 "Too many regs for number of args"); 777 778 // Coerce into outgoing value types before register assignment. 779 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) { 780 assert(Args[i].OrigRegs.size() == 1); 781 buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy, 782 ValTy, extendOpFromFlags(Args[i].Flags[0])); 783 } 784 785 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL); 786 for (unsigned Part = 0; Part < NumParts; ++Part) { 787 Register ArgReg = Args[i].Regs[Part]; 788 // There should be Regs.size() ArgLocs per argument. 789 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part; 790 CCValAssign &VA = ArgLocs[j + Idx]; 791 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part]; 792 793 if (VA.isMemLoc() && !Flags.isByVal()) { 794 // Individual pieces may have been spilled to the stack and others 795 // passed in registers. 796 797 // TODO: The memory size may be larger than the value we need to 798 // store. We may need to adjust the offset for big endian targets. 799 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags); 800 801 MachinePointerInfo MPO; 802 Register StackAddr = Handler.getStackAddress( 803 MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags); 804 805 Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA); 806 continue; 807 } 808 809 if (VA.isMemLoc() && Flags.isByVal()) { 810 assert(Args[i].Regs.size() == 1 && 811 "didn't expect split byval pointer"); 812 813 if (Handler.isIncomingArgumentHandler()) { 814 // We just need to copy the frame index value to the pointer. 815 MachinePointerInfo MPO; 816 Register StackAddr = Handler.getStackAddress( 817 Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags); 818 MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr); 819 } else { 820 // For outgoing byval arguments, insert the implicit copy byval 821 // implies, such that writes in the callee do not modify the caller's 822 // value. 823 uint64_t MemSize = Flags.getByValSize(); 824 int64_t Offset = VA.getLocMemOffset(); 825 826 MachinePointerInfo DstMPO; 827 Register StackAddr = 828 Handler.getStackAddress(MemSize, Offset, DstMPO, Flags); 829 830 MachinePointerInfo SrcMPO(Args[i].OrigValue); 831 if (!Args[i].OrigValue) { 832 // We still need to accurately track the stack address space if we 833 // don't know the underlying value. 834 const LLT PtrTy = MRI.getType(StackAddr); 835 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace()); 836 } 837 838 Align DstAlign = std::max(Flags.getNonZeroByValAlign(), 839 inferAlignFromPtrInfo(MF, DstMPO)); 840 841 Align SrcAlign = std::max(Flags.getNonZeroByValAlign(), 842 inferAlignFromPtrInfo(MF, SrcMPO)); 843 844 Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0], 845 DstMPO, DstAlign, SrcMPO, SrcAlign, 846 MemSize, VA); 847 } 848 continue; 849 } 850 851 assert(!VA.needsCustom() && "custom loc should have been handled already"); 852 853 if (i == 0 && !ThisReturnRegs.empty() && 854 Handler.isIncomingArgumentHandler() && 855 isTypeIsValidForThisReturn(ValVT)) { 856 Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA); 857 continue; 858 } 859 860 if (Handler.isIncomingArgumentHandler()) 861 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 862 else { 863 DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() { 864 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 865 }); 866 } 867 } 868 869 // Now that all pieces have been assigned, re-pack the register typed values 870 // into the original value typed registers. 871 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) { 872 // Merge the split registers into the expected larger result vregs of 873 // the original call. 874 buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy, 875 LocTy, Args[i].Flags[0]); 876 } 877 878 j += NumParts - 1; 879 } 880 for (auto &Fn : DelayedOutgoingRegAssignments) 881 Fn(); 882 883 return true; 884 } 885 886 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, 887 ArrayRef<Register> VRegs, Register DemoteReg, 888 int FI) const { 889 MachineFunction &MF = MIRBuilder.getMF(); 890 MachineRegisterInfo &MRI = MF.getRegInfo(); 891 const DataLayout &DL = MF.getDataLayout(); 892 893 SmallVector<EVT, 4> SplitVTs; 894 SmallVector<uint64_t, 4> Offsets; 895 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 896 897 assert(VRegs.size() == SplitVTs.size()); 898 899 unsigned NumValues = SplitVTs.size(); 900 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 901 Type *RetPtrTy = 902 PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace()); 903 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL); 904 905 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); 906 907 for (unsigned I = 0; I < NumValues; ++I) { 908 Register Addr; 909 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 910 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 911 MRI.getType(VRegs[I]), 912 commonAlignment(BaseAlign, Offsets[I])); 913 MIRBuilder.buildLoad(VRegs[I], Addr, *MMO); 914 } 915 } 916 917 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, 918 ArrayRef<Register> VRegs, 919 Register DemoteReg) const { 920 MachineFunction &MF = MIRBuilder.getMF(); 921 MachineRegisterInfo &MRI = MF.getRegInfo(); 922 const DataLayout &DL = MF.getDataLayout(); 923 924 SmallVector<EVT, 4> SplitVTs; 925 SmallVector<uint64_t, 4> Offsets; 926 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 927 928 assert(VRegs.size() == SplitVTs.size()); 929 930 unsigned NumValues = SplitVTs.size(); 931 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 932 unsigned AS = DL.getAllocaAddrSpace(); 933 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL); 934 935 MachinePointerInfo PtrInfo(AS); 936 937 for (unsigned I = 0; I < NumValues; ++I) { 938 Register Addr; 939 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 940 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 941 MRI.getType(VRegs[I]), 942 commonAlignment(BaseAlign, Offsets[I])); 943 MIRBuilder.buildStore(VRegs[I], Addr, *MMO); 944 } 945 } 946 947 void CallLowering::insertSRetIncomingArgument( 948 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg, 949 MachineRegisterInfo &MRI, const DataLayout &DL) const { 950 unsigned AS = DL.getAllocaAddrSpace(); 951 DemoteReg = MRI.createGenericVirtualRegister( 952 LLT::pointer(AS, DL.getPointerSizeInBits(AS))); 953 954 Type *PtrTy = PointerType::get(F.getReturnType(), AS); 955 956 SmallVector<EVT, 1> ValueVTs; 957 ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs); 958 959 // NOTE: Assume that a pointer won't get split into more than one VT. 960 assert(ValueVTs.size() == 1); 961 962 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()), 963 ArgInfo::NoArgIndex); 964 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F); 965 DemoteArg.Flags[0].setSRet(); 966 SplitArgs.insert(SplitArgs.begin(), DemoteArg); 967 } 968 969 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder, 970 const CallBase &CB, 971 CallLoweringInfo &Info) const { 972 const DataLayout &DL = MIRBuilder.getDataLayout(); 973 Type *RetTy = CB.getType(); 974 unsigned AS = DL.getAllocaAddrSpace(); 975 LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); 976 977 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject( 978 DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false); 979 980 Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0); 981 ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS), 982 ArgInfo::NoArgIndex); 983 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB); 984 DemoteArg.Flags[0].setSRet(); 985 986 Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg); 987 Info.DemoteStackIndex = FI; 988 Info.DemoteRegister = DemoteReg; 989 } 990 991 bool CallLowering::checkReturn(CCState &CCInfo, 992 SmallVectorImpl<BaseArgInfo> &Outs, 993 CCAssignFn *Fn) const { 994 for (unsigned I = 0, E = Outs.size(); I < E; ++I) { 995 MVT VT = MVT::getVT(Outs[I].Ty); 996 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo)) 997 return false; 998 } 999 return true; 1000 } 1001 1002 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy, 1003 AttributeList Attrs, 1004 SmallVectorImpl<BaseArgInfo> &Outs, 1005 const DataLayout &DL) const { 1006 LLVMContext &Context = RetTy->getContext(); 1007 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1008 1009 SmallVector<EVT, 4> SplitVTs; 1010 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs); 1011 addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex); 1012 1013 for (EVT VT : SplitVTs) { 1014 unsigned NumParts = 1015 TLI->getNumRegistersForCallingConv(Context, CallConv, VT); 1016 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT); 1017 Type *PartTy = EVT(RegVT).getTypeForEVT(Context); 1018 1019 for (unsigned I = 0; I < NumParts; ++I) { 1020 Outs.emplace_back(PartTy, Flags); 1021 } 1022 } 1023 } 1024 1025 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const { 1026 const auto &F = MF.getFunction(); 1027 Type *ReturnType = F.getReturnType(); 1028 CallingConv::ID CallConv = F.getCallingConv(); 1029 1030 SmallVector<BaseArgInfo, 4> SplitArgs; 1031 getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs, 1032 MF.getDataLayout()); 1033 return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg()); 1034 } 1035 1036 bool CallLowering::parametersInCSRMatch( 1037 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, 1038 const SmallVectorImpl<CCValAssign> &OutLocs, 1039 const SmallVectorImpl<ArgInfo> &OutArgs) const { 1040 for (unsigned i = 0; i < OutLocs.size(); ++i) { 1041 const auto &ArgLoc = OutLocs[i]; 1042 // If it's not a register, it's fine. 1043 if (!ArgLoc.isRegLoc()) 1044 continue; 1045 1046 MCRegister PhysReg = ArgLoc.getLocReg(); 1047 1048 // Only look at callee-saved registers. 1049 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg)) 1050 continue; 1051 1052 LLVM_DEBUG( 1053 dbgs() 1054 << "... Call has an argument passed in a callee-saved register.\n"); 1055 1056 // Check if it was copied from. 1057 const ArgInfo &OutInfo = OutArgs[i]; 1058 1059 if (OutInfo.Regs.size() > 1) { 1060 LLVM_DEBUG( 1061 dbgs() << "... Cannot handle arguments in multiple registers.\n"); 1062 return false; 1063 } 1064 1065 // Check if we copy the register, walking through copies from virtual 1066 // registers. Note that getDefIgnoringCopies does not ignore copies from 1067 // physical registers. 1068 MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI); 1069 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) { 1070 LLVM_DEBUG( 1071 dbgs() 1072 << "... Parameter was not copied into a VReg, cannot tail call.\n"); 1073 return false; 1074 } 1075 1076 // Got a copy. Verify that it's the same as the register we want. 1077 Register CopyRHS = RegDef->getOperand(1).getReg(); 1078 if (CopyRHS != PhysReg) { 1079 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into " 1080 "VReg, cannot tail call.\n"); 1081 return false; 1082 } 1083 } 1084 1085 return true; 1086 } 1087 1088 bool CallLowering::resultsCompatible(CallLoweringInfo &Info, 1089 MachineFunction &MF, 1090 SmallVectorImpl<ArgInfo> &InArgs, 1091 ValueAssigner &CalleeAssigner, 1092 ValueAssigner &CallerAssigner) const { 1093 const Function &F = MF.getFunction(); 1094 CallingConv::ID CalleeCC = Info.CallConv; 1095 CallingConv::ID CallerCC = F.getCallingConv(); 1096 1097 if (CallerCC == CalleeCC) 1098 return true; 1099 1100 SmallVector<CCValAssign, 16> ArgLocs1; 1101 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext()); 1102 if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1)) 1103 return false; 1104 1105 SmallVector<CCValAssign, 16> ArgLocs2; 1106 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext()); 1107 if (!determineAssignments(CallerAssigner, InArgs, CCInfo2)) 1108 return false; 1109 1110 // We need the argument locations to match up exactly. If there's more in 1111 // one than the other, then we are done. 1112 if (ArgLocs1.size() != ArgLocs2.size()) 1113 return false; 1114 1115 // Make sure that each location is passed in exactly the same way. 1116 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { 1117 const CCValAssign &Loc1 = ArgLocs1[i]; 1118 const CCValAssign &Loc2 = ArgLocs2[i]; 1119 1120 // We need both of them to be the same. So if one is a register and one 1121 // isn't, we're done. 1122 if (Loc1.isRegLoc() != Loc2.isRegLoc()) 1123 return false; 1124 1125 if (Loc1.isRegLoc()) { 1126 // If they don't have the same register location, we're done. 1127 if (Loc1.getLocReg() != Loc2.getLocReg()) 1128 return false; 1129 1130 // They matched, so we can move to the next ArgLoc. 1131 continue; 1132 } 1133 1134 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. 1135 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) 1136 return false; 1137 } 1138 1139 return true; 1140 } 1141 1142 LLT CallLowering::ValueHandler::getStackValueStoreType( 1143 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const { 1144 const MVT ValVT = VA.getValVT(); 1145 if (ValVT != MVT::iPTR) { 1146 LLT ValTy(ValVT); 1147 1148 // We lost the pointeriness going through CCValAssign, so try to restore it 1149 // based on the flags. 1150 if (Flags.isPointer()) { 1151 LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(), 1152 ValTy.getScalarSizeInBits()); 1153 if (ValVT.isVector()) 1154 return LLT::vector(ValTy.getElementCount(), PtrTy); 1155 return PtrTy; 1156 } 1157 1158 return ValTy; 1159 } 1160 1161 unsigned AddrSpace = Flags.getPointerAddrSpace(); 1162 return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace)); 1163 } 1164 1165 void CallLowering::ValueHandler::copyArgumentMemory( 1166 const ArgInfo &Arg, Register DstPtr, Register SrcPtr, 1167 const MachinePointerInfo &DstPtrInfo, Align DstAlign, 1168 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize, 1169 CCValAssign &VA) const { 1170 MachineFunction &MF = MIRBuilder.getMF(); 1171 MachineMemOperand *SrcMMO = MF.getMachineMemOperand( 1172 SrcPtrInfo, 1173 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize, 1174 SrcAlign); 1175 1176 MachineMemOperand *DstMMO = MF.getMachineMemOperand( 1177 DstPtrInfo, 1178 MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable, 1179 MemSize, DstAlign); 1180 1181 const LLT PtrTy = MRI.getType(DstPtr); 1182 const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits()); 1183 1184 auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize); 1185 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO); 1186 } 1187 1188 Register CallLowering::ValueHandler::extendRegister(Register ValReg, 1189 const CCValAssign &VA, 1190 unsigned MaxSizeBits) { 1191 LLT LocTy{VA.getLocVT()}; 1192 LLT ValTy{VA.getValVT()}; 1193 1194 if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) 1195 return ValReg; 1196 1197 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { 1198 if (MaxSizeBits <= ValTy.getSizeInBits()) 1199 return ValReg; 1200 LocTy = LLT::scalar(MaxSizeBits); 1201 } 1202 1203 const LLT ValRegTy = MRI.getType(ValReg); 1204 if (ValRegTy.isPointer()) { 1205 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so 1206 // we have to cast to do the extension. 1207 LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits()); 1208 ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0); 1209 } 1210 1211 switch (VA.getLocInfo()) { 1212 default: break; 1213 case CCValAssign::Full: 1214 case CCValAssign::BCvt: 1215 // FIXME: bitconverting between vector types may or may not be a 1216 // nop in big-endian situations. 1217 return ValReg; 1218 case CCValAssign::AExt: { 1219 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg); 1220 return MIB.getReg(0); 1221 } 1222 case CCValAssign::SExt: { 1223 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 1224 MIRBuilder.buildSExt(NewReg, ValReg); 1225 return NewReg; 1226 } 1227 case CCValAssign::ZExt: { 1228 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 1229 MIRBuilder.buildZExt(NewReg, ValReg); 1230 return NewReg; 1231 } 1232 } 1233 llvm_unreachable("unable to extend register"); 1234 } 1235 1236 void CallLowering::ValueAssigner::anchor() {} 1237 1238 Register CallLowering::IncomingValueHandler::buildExtensionHint( 1239 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) { 1240 switch (VA.getLocInfo()) { 1241 case CCValAssign::LocInfo::ZExt: { 1242 return MIRBuilder 1243 .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 1244 NarrowTy.getScalarSizeInBits()) 1245 .getReg(0); 1246 } 1247 case CCValAssign::LocInfo::SExt: { 1248 return MIRBuilder 1249 .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 1250 NarrowTy.getScalarSizeInBits()) 1251 .getReg(0); 1252 break; 1253 } 1254 default: 1255 return SrcReg; 1256 } 1257 } 1258 1259 /// Check if we can use a basic COPY instruction between the two types. 1260 /// 1261 /// We're currently building on top of the infrastructure using MVT, which loses 1262 /// pointer information in the CCValAssign. We accept copies from physical 1263 /// registers that have been reported as integers if it's to an equivalent sized 1264 /// pointer LLT. 1265 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) { 1266 if (SrcTy == DstTy) 1267 return true; 1268 1269 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) 1270 return false; 1271 1272 SrcTy = SrcTy.getScalarType(); 1273 DstTy = DstTy.getScalarType(); 1274 1275 return (SrcTy.isPointer() && DstTy.isScalar()) || 1276 (DstTy.isPointer() && SrcTy.isScalar()); 1277 } 1278 1279 void CallLowering::IncomingValueHandler::assignValueToReg( 1280 Register ValVReg, Register PhysReg, const CCValAssign &VA) { 1281 const MVT LocVT = VA.getLocVT(); 1282 const LLT LocTy(LocVT); 1283 const LLT RegTy = MRI.getType(ValVReg); 1284 1285 if (isCopyCompatibleType(RegTy, LocTy)) { 1286 MIRBuilder.buildCopy(ValVReg, PhysReg); 1287 return; 1288 } 1289 1290 auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg); 1291 auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy); 1292 MIRBuilder.buildTrunc(ValVReg, Hint); 1293 } 1294