1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements some simple delegations needed for call lowering. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 15 #include "llvm/CodeGen/Analysis.h" 16 #include "llvm/CodeGen/CallingConvLower.h" 17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 18 #include "llvm/CodeGen/GlobalISel/Utils.h" 19 #include "llvm/CodeGen/MachineFrameInfo.h" 20 #include "llvm/CodeGen/MachineOperand.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/TargetLowering.h" 23 #include "llvm/IR/DataLayout.h" 24 #include "llvm/IR/LLVMContext.h" 25 #include "llvm/IR/Module.h" 26 #include "llvm/Target/TargetMachine.h" 27 28 #define DEBUG_TYPE "call-lowering" 29 30 using namespace llvm; 31 32 void CallLowering::anchor() {} 33 34 /// Helper function which updates \p Flags when \p AttrFn returns true. 35 static void 36 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, 37 const std::function<bool(Attribute::AttrKind)> &AttrFn) { 38 if (AttrFn(Attribute::SExt)) 39 Flags.setSExt(); 40 if (AttrFn(Attribute::ZExt)) 41 Flags.setZExt(); 42 if (AttrFn(Attribute::InReg)) 43 Flags.setInReg(); 44 if (AttrFn(Attribute::StructRet)) 45 Flags.setSRet(); 46 if (AttrFn(Attribute::Nest)) 47 Flags.setNest(); 48 if (AttrFn(Attribute::ByVal)) 49 Flags.setByVal(); 50 if (AttrFn(Attribute::Preallocated)) 51 Flags.setPreallocated(); 52 if (AttrFn(Attribute::InAlloca)) 53 Flags.setInAlloca(); 54 if (AttrFn(Attribute::Returned)) 55 Flags.setReturned(); 56 if (AttrFn(Attribute::SwiftSelf)) 57 Flags.setSwiftSelf(); 58 if (AttrFn(Attribute::SwiftAsync)) 59 Flags.setSwiftAsync(); 60 if (AttrFn(Attribute::SwiftError)) 61 Flags.setSwiftError(); 62 } 63 64 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, 65 unsigned ArgIdx) const { 66 ISD::ArgFlagsTy Flags; 67 addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) { 68 return Call.paramHasAttr(ArgIdx, Attr); 69 }); 70 return Flags; 71 } 72 73 ISD::ArgFlagsTy 74 CallLowering::getAttributesForReturn(const CallBase &Call) const { 75 ISD::ArgFlagsTy Flags; 76 addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) { 77 return Call.hasRetAttr(Attr); 78 }); 79 return Flags; 80 } 81 82 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, 83 const AttributeList &Attrs, 84 unsigned OpIdx) const { 85 addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) { 86 return Attrs.hasAttributeAtIndex(OpIdx, Attr); 87 }); 88 } 89 90 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, 91 ArrayRef<Register> ResRegs, 92 ArrayRef<ArrayRef<Register>> ArgRegs, 93 Register SwiftErrorVReg, 94 std::function<unsigned()> GetCalleeReg) const { 95 CallLoweringInfo Info; 96 const DataLayout &DL = MIRBuilder.getDataLayout(); 97 MachineFunction &MF = MIRBuilder.getMF(); 98 MachineRegisterInfo &MRI = MF.getRegInfo(); 99 bool CanBeTailCalled = CB.isTailCall() && 100 isInTailCallPosition(CB, MF.getTarget()) && 101 (MF.getFunction() 102 .getFnAttribute("disable-tail-calls") 103 .getValueAsString() != "true"); 104 105 CallingConv::ID CallConv = CB.getCallingConv(); 106 Type *RetTy = CB.getType(); 107 bool IsVarArg = CB.getFunctionType()->isVarArg(); 108 109 SmallVector<BaseArgInfo, 4> SplitArgs; 110 getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL); 111 Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg); 112 113 Info.IsConvergent = CB.isConvergent(); 114 115 if (!Info.CanLowerReturn) { 116 // Callee requires sret demotion. 117 insertSRetOutgoingArgument(MIRBuilder, CB, Info); 118 119 // The sret demotion isn't compatible with tail-calls, since the sret 120 // argument points into the caller's stack frame. 121 CanBeTailCalled = false; 122 } 123 124 125 // First step is to marshall all the function's parameters into the correct 126 // physregs and memory locations. Gather the sequence of argument types that 127 // we'll pass to the assigner function. 128 unsigned i = 0; 129 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); 130 for (const auto &Arg : CB.args()) { 131 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i), 132 i < NumFixedArgs}; 133 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB); 134 135 // If we have an explicit sret argument that is an Instruction, (i.e., it 136 // might point to function-local memory), we can't meaningfully tail-call. 137 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg)) 138 CanBeTailCalled = false; 139 140 Info.OrigArgs.push_back(OrigArg); 141 ++i; 142 } 143 144 // Try looking through a bitcast from one function type to another. 145 // Commonly happens with calls to objc_msgSend(). 146 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); 147 if (const Function *F = dyn_cast<Function>(CalleeV)) 148 Info.Callee = MachineOperand::CreateGA(F, 0); 149 else 150 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); 151 152 Register ReturnHintAlignReg; 153 Align ReturnHintAlign; 154 155 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)}; 156 157 if (!Info.OrigRet.Ty->isVoidTy()) { 158 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB); 159 160 if (MaybeAlign Alignment = CB.getRetAlign()) { 161 if (*Alignment > Align(1)) { 162 ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]); 163 Info.OrigRet.Regs[0] = ReturnHintAlignReg; 164 ReturnHintAlign = *Alignment; 165 } 166 } 167 } 168 169 auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi); 170 if (Bundle && CB.isIndirectCall()) { 171 Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]); 172 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); 173 } 174 175 Info.CB = &CB; 176 Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); 177 Info.CallConv = CallConv; 178 Info.SwiftErrorVReg = SwiftErrorVReg; 179 Info.IsMustTailCall = CB.isMustTailCall(); 180 Info.IsTailCall = CanBeTailCalled; 181 Info.IsVarArg = IsVarArg; 182 if (!lowerCall(MIRBuilder, Info)) 183 return false; 184 185 if (ReturnHintAlignReg && !Info.IsTailCall) { 186 MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg, 187 ReturnHintAlign); 188 } 189 190 return true; 191 } 192 193 template <typename FuncInfoTy> 194 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, 195 const DataLayout &DL, 196 const FuncInfoTy &FuncInfo) const { 197 auto &Flags = Arg.Flags[0]; 198 const AttributeList &Attrs = FuncInfo.getAttributes(); 199 addArgFlagsFromAttributes(Flags, Attrs, OpIdx); 200 201 PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType()); 202 if (PtrTy) { 203 Flags.setPointer(); 204 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace()); 205 } 206 207 Align MemAlign = DL.getABITypeAlign(Arg.Ty); 208 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { 209 assert(OpIdx >= AttributeList::FirstArgIndex); 210 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; 211 212 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); 213 if (!ElementTy) 214 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); 215 if (!ElementTy) 216 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); 217 assert(ElementTy && "Must have byval, inalloca or preallocated type"); 218 Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); 219 220 // For ByVal, alignment should be passed from FE. BE will guess if 221 // this info is not there but there are cases it cannot get right. 222 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx)) 223 MemAlign = *ParamAlign; 224 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx))) 225 MemAlign = *ParamAlign; 226 else 227 MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL)); 228 } else if (OpIdx >= AttributeList::FirstArgIndex) { 229 if (auto ParamAlign = 230 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex)) 231 MemAlign = *ParamAlign; 232 } 233 Flags.setMemAlign(MemAlign); 234 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty)); 235 236 // Don't try to use the returned attribute if the argument is marked as 237 // swiftself, since it won't be passed in x0. 238 if (Flags.isSwiftSelf()) 239 Flags.setReturned(false); 240 } 241 242 template void 243 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 244 const DataLayout &DL, 245 const Function &FuncInfo) const; 246 247 template void 248 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 249 const DataLayout &DL, 250 const CallBase &FuncInfo) const; 251 252 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 253 SmallVectorImpl<ArgInfo> &SplitArgs, 254 const DataLayout &DL, 255 CallingConv::ID CallConv, 256 SmallVectorImpl<uint64_t> *Offsets) const { 257 LLVMContext &Ctx = OrigArg.Ty->getContext(); 258 259 SmallVector<EVT, 4> SplitVTs; 260 ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0); 261 262 if (SplitVTs.size() == 0) 263 return; 264 265 if (SplitVTs.size() == 1) { 266 // No splitting to do, but we want to replace the original type (e.g. [1 x 267 // double] -> double). 268 SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx), 269 OrigArg.OrigArgIndex, OrigArg.Flags[0], 270 OrigArg.IsFixed, OrigArg.OrigValue); 271 return; 272 } 273 274 // Create one ArgInfo for each virtual register in the original ArgInfo. 275 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch"); 276 277 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( 278 OrigArg.Ty, CallConv, false, DL); 279 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { 280 Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx); 281 SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex, 282 OrigArg.Flags[0], OrigArg.IsFixed); 283 if (NeedsRegBlock) 284 SplitArgs.back().Flags[0].setInConsecutiveRegs(); 285 } 286 287 SplitArgs.back().Flags[0].setInConsecutiveRegsLast(); 288 } 289 290 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs. 291 static MachineInstrBuilder 292 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 293 ArrayRef<Register> SrcRegs) { 294 MachineRegisterInfo &MRI = *B.getMRI(); 295 LLT LLTy = MRI.getType(DstRegs[0]); 296 LLT PartLLT = MRI.getType(SrcRegs[0]); 297 298 // Deal with v3s16 split into v2s16 299 LLT LCMTy = getCoverTy(LLTy, PartLLT); 300 if (LCMTy == LLTy) { 301 // Common case where no padding is needed. 302 assert(DstRegs.size() == 1); 303 return B.buildConcatVectors(DstRegs[0], SrcRegs); 304 } 305 306 // We need to create an unmerge to the result registers, which may require 307 // widening the original value. 308 Register UnmergeSrcReg; 309 if (LCMTy != PartLLT) { 310 assert(DstRegs.size() == 1); 311 return B.buildDeleteTrailingVectorElements( 312 DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs)); 313 } else { 314 // We don't need to widen anything if we're extracting a scalar which was 315 // promoted to a vector e.g. s8 -> v4s8 -> s8 316 assert(SrcRegs.size() == 1); 317 UnmergeSrcReg = SrcRegs[0]; 318 } 319 320 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits(); 321 322 SmallVector<Register, 8> PadDstRegs(NumDst); 323 std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin()); 324 325 // Create the excess dead defs for the unmerge. 326 for (int I = DstRegs.size(); I != NumDst; ++I) 327 PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy); 328 329 if (PadDstRegs.size() == 1) 330 return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg); 331 return B.buildUnmerge(PadDstRegs, UnmergeSrcReg); 332 } 333 334 /// Create a sequence of instructions to combine pieces split into register 335 /// typed values to the original IR value. \p OrigRegs contains the destination 336 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces 337 /// with type \p PartLLT. This is used for incoming values (physregs to vregs). 338 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs, 339 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, 340 const ISD::ArgFlagsTy Flags) { 341 MachineRegisterInfo &MRI = *B.getMRI(); 342 343 if (PartLLT == LLTy) { 344 // We should have avoided introducing a new virtual register, and just 345 // directly assigned here. 346 assert(OrigRegs[0] == Regs[0]); 347 return; 348 } 349 350 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 && 351 Regs.size() == 1) { 352 B.buildBitcast(OrigRegs[0], Regs[0]); 353 return; 354 } 355 356 // A vector PartLLT needs extending to LLTy's element size. 357 // E.g. <2 x s64> = G_SEXT <2 x s32>. 358 if (PartLLT.isVector() == LLTy.isVector() && 359 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() && 360 (!PartLLT.isVector() || 361 PartLLT.getNumElements() == LLTy.getNumElements()) && 362 OrigRegs.size() == 1 && Regs.size() == 1) { 363 Register SrcReg = Regs[0]; 364 365 LLT LocTy = MRI.getType(SrcReg); 366 367 if (Flags.isSExt()) { 368 SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits()) 369 .getReg(0); 370 } else if (Flags.isZExt()) { 371 SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits()) 372 .getReg(0); 373 } 374 375 // Sometimes pointers are passed zero extended. 376 LLT OrigTy = MRI.getType(OrigRegs[0]); 377 if (OrigTy.isPointer()) { 378 LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits()); 379 B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg)); 380 return; 381 } 382 383 B.buildTrunc(OrigRegs[0], SrcReg); 384 return; 385 } 386 387 if (!LLTy.isVector() && !PartLLT.isVector()) { 388 assert(OrigRegs.size() == 1); 389 LLT OrigTy = MRI.getType(OrigRegs[0]); 390 391 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size(); 392 if (SrcSize == OrigTy.getSizeInBits()) 393 B.buildMergeValues(OrigRegs[0], Regs); 394 else { 395 auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs); 396 B.buildTrunc(OrigRegs[0], Widened); 397 } 398 399 return; 400 } 401 402 if (PartLLT.isVector()) { 403 assert(OrigRegs.size() == 1); 404 SmallVector<Register> CastRegs(Regs.begin(), Regs.end()); 405 406 // If PartLLT is a mismatched vector in both number of elements and element 407 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to 408 // have the same elt type, i.e. v4s32. 409 if (PartLLT.getSizeInBits() > LLTy.getSizeInBits() && 410 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 && 411 Regs.size() == 1) { 412 LLT NewTy = PartLLT.changeElementType(LLTy.getElementType()) 413 .changeElementCount(PartLLT.getElementCount() * 2); 414 CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0); 415 PartLLT = NewTy; 416 } 417 418 if (LLTy.getScalarType() == PartLLT.getElementType()) { 419 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs); 420 } else { 421 unsigned I = 0; 422 LLT GCDTy = getGCDType(LLTy, PartLLT); 423 424 // We are both splitting a vector, and bitcasting its element types. Cast 425 // the source pieces into the appropriate number of pieces with the result 426 // element type. 427 for (Register SrcReg : CastRegs) 428 CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0); 429 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs); 430 } 431 432 return; 433 } 434 435 assert(LLTy.isVector() && !PartLLT.isVector()); 436 437 LLT DstEltTy = LLTy.getElementType(); 438 439 // Pointer information was discarded. We'll need to coerce some register types 440 // to avoid violating type constraints. 441 LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType(); 442 443 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); 444 445 if (DstEltTy == PartLLT) { 446 // Vector was trivially scalarized. 447 448 if (RealDstEltTy.isPointer()) { 449 for (Register Reg : Regs) 450 MRI.setType(Reg, RealDstEltTy); 451 } 452 453 B.buildBuildVector(OrigRegs[0], Regs); 454 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { 455 // Deal with vector with 64-bit elements decomposed to 32-bit 456 // registers. Need to create intermediate 64-bit elements. 457 SmallVector<Register, 8> EltMerges; 458 int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits(); 459 460 assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); 461 462 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { 463 auto Merge = 464 B.buildMergeLikeInstr(RealDstEltTy, Regs.take_front(PartsPerElt)); 465 // Fix the type in case this is really a vector of pointers. 466 MRI.setType(Merge.getReg(0), RealDstEltTy); 467 EltMerges.push_back(Merge.getReg(0)); 468 Regs = Regs.drop_front(PartsPerElt); 469 } 470 471 B.buildBuildVector(OrigRegs[0], EltMerges); 472 } else { 473 // Vector was split, and elements promoted to a wider type. 474 // FIXME: Should handle floating point promotions. 475 LLT BVType = LLT::fixed_vector(LLTy.getNumElements(), PartLLT); 476 auto BV = B.buildBuildVector(BVType, Regs); 477 B.buildTrunc(OrigRegs[0], BV); 478 } 479 } 480 481 /// Create a sequence of instructions to expand the value in \p SrcReg (of type 482 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should 483 /// contain the type of scalar value extension if necessary. 484 /// 485 /// This is used for outgoing values (vregs to physregs) 486 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, 487 Register SrcReg, LLT SrcTy, LLT PartTy, 488 unsigned ExtendOp = TargetOpcode::G_ANYEXT) { 489 // We could just insert a regular copy, but this is unreachable at the moment. 490 assert(SrcTy != PartTy && "identical part types shouldn't reach here"); 491 492 const unsigned PartSize = PartTy.getSizeInBits(); 493 494 if (PartTy.isVector() == SrcTy.isVector() && 495 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) { 496 assert(DstRegs.size() == 1); 497 B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg}); 498 return; 499 } 500 501 if (SrcTy.isVector() && !PartTy.isVector() && 502 PartSize > SrcTy.getElementType().getSizeInBits()) { 503 // Vector was scalarized, and the elements extended. 504 auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg); 505 for (int i = 0, e = DstRegs.size(); i != e; ++i) 506 B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i)); 507 return; 508 } 509 510 if (SrcTy.isVector() && PartTy.isVector() && 511 PartTy.getScalarSizeInBits() == SrcTy.getScalarSizeInBits() && 512 SrcTy.getNumElements() < PartTy.getNumElements()) { 513 // A coercion like: v2f32 -> v4f32. 514 Register DstReg = DstRegs.front(); 515 B.buildPadVectorWithUndefElements(DstReg, SrcReg); 516 return; 517 } 518 519 LLT GCDTy = getGCDType(SrcTy, PartTy); 520 if (GCDTy == PartTy) { 521 // If this already evenly divisible, we can create a simple unmerge. 522 B.buildUnmerge(DstRegs, SrcReg); 523 return; 524 } 525 526 MachineRegisterInfo &MRI = *B.getMRI(); 527 LLT DstTy = MRI.getType(DstRegs[0]); 528 LLT LCMTy = getCoverTy(SrcTy, PartTy); 529 530 if (PartTy.isVector() && LCMTy == PartTy) { 531 assert(DstRegs.size() == 1); 532 B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg); 533 return; 534 } 535 536 const unsigned DstSize = DstTy.getSizeInBits(); 537 const unsigned SrcSize = SrcTy.getSizeInBits(); 538 unsigned CoveringSize = LCMTy.getSizeInBits(); 539 540 Register UnmergeSrc = SrcReg; 541 542 if (!LCMTy.isVector() && CoveringSize != SrcSize) { 543 // For scalars, it's common to be able to use a simple extension. 544 if (SrcTy.isScalar() && DstTy.isScalar()) { 545 CoveringSize = alignTo(SrcSize, DstSize); 546 LLT CoverTy = LLT::scalar(CoveringSize); 547 UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0); 548 } else { 549 // Widen to the common type. 550 // FIXME: This should respect the extend type 551 Register Undef = B.buildUndef(SrcTy).getReg(0); 552 SmallVector<Register, 8> MergeParts(1, SrcReg); 553 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize) 554 MergeParts.push_back(Undef); 555 UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0); 556 } 557 } 558 559 if (LCMTy.isVector() && CoveringSize != SrcSize) 560 UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0); 561 562 B.buildUnmerge(DstRegs, UnmergeSrc); 563 } 564 565 bool CallLowering::determineAndHandleAssignments( 566 ValueHandler &Handler, ValueAssigner &Assigner, 567 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder, 568 CallingConv::ID CallConv, bool IsVarArg, 569 ArrayRef<Register> ThisReturnRegs) const { 570 MachineFunction &MF = MIRBuilder.getMF(); 571 const Function &F = MF.getFunction(); 572 SmallVector<CCValAssign, 16> ArgLocs; 573 574 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext()); 575 if (!determineAssignments(Assigner, Args, CCInfo)) 576 return false; 577 578 return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder, 579 ThisReturnRegs); 580 } 581 582 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) { 583 if (Flags.isSExt()) 584 return TargetOpcode::G_SEXT; 585 if (Flags.isZExt()) 586 return TargetOpcode::G_ZEXT; 587 return TargetOpcode::G_ANYEXT; 588 } 589 590 bool CallLowering::determineAssignments(ValueAssigner &Assigner, 591 SmallVectorImpl<ArgInfo> &Args, 592 CCState &CCInfo) const { 593 LLVMContext &Ctx = CCInfo.getContext(); 594 const CallingConv::ID CallConv = CCInfo.getCallingConv(); 595 596 unsigned NumArgs = Args.size(); 597 for (unsigned i = 0; i != NumArgs; ++i) { 598 EVT CurVT = EVT::getEVT(Args[i].Ty); 599 600 MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT); 601 602 // If we need to split the type over multiple regs, check it's a scenario 603 // we currently support. 604 unsigned NumParts = 605 TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT); 606 607 if (NumParts == 1) { 608 // Try to use the register type if we couldn't assign the VT. 609 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i], 610 Args[i].Flags[0], CCInfo)) 611 return false; 612 continue; 613 } 614 615 // For incoming arguments (physregs to vregs), we could have values in 616 // physregs (or memlocs) which we want to extract and copy to vregs. 617 // During this, we might have to deal with the LLT being split across 618 // multiple regs, so we have to record this information for later. 619 // 620 // If we have outgoing args, then we have the opposite case. We have a 621 // vreg with an LLT which we want to assign to a physical location, and 622 // we might have to record that the value has to be split later. 623 624 // We're handling an incoming arg which is split over multiple regs. 625 // E.g. passing an s128 on AArch64. 626 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 627 Args[i].Flags.clear(); 628 629 for (unsigned Part = 0; Part < NumParts; ++Part) { 630 ISD::ArgFlagsTy Flags = OrigFlags; 631 if (Part == 0) { 632 Flags.setSplit(); 633 } else { 634 Flags.setOrigAlign(Align(1)); 635 if (Part == NumParts - 1) 636 Flags.setSplitEnd(); 637 } 638 639 Args[i].Flags.push_back(Flags); 640 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i], 641 Args[i].Flags[Part], CCInfo)) { 642 // Still couldn't assign this smaller part type for some reason. 643 return false; 644 } 645 } 646 } 647 648 return true; 649 } 650 651 bool CallLowering::handleAssignments(ValueHandler &Handler, 652 SmallVectorImpl<ArgInfo> &Args, 653 CCState &CCInfo, 654 SmallVectorImpl<CCValAssign> &ArgLocs, 655 MachineIRBuilder &MIRBuilder, 656 ArrayRef<Register> ThisReturnRegs) const { 657 MachineFunction &MF = MIRBuilder.getMF(); 658 MachineRegisterInfo &MRI = MF.getRegInfo(); 659 const Function &F = MF.getFunction(); 660 const DataLayout &DL = F.getParent()->getDataLayout(); 661 662 const unsigned NumArgs = Args.size(); 663 664 // Stores thunks for outgoing register assignments. This is used so we delay 665 // generating register copies until mem loc assignments are done. We do this 666 // so that if the target is using the delayed stack protector feature, we can 667 // find the split point of the block accurately. E.g. if we have: 668 // G_STORE %val, %memloc 669 // $x0 = COPY %foo 670 // $x1 = COPY %bar 671 // CALL func 672 // ... then the split point for the block will correctly be at, and including, 673 // the copy to $x0. If instead the G_STORE instruction immediately precedes 674 // the CALL, then we'd prematurely choose the CALL as the split point, thus 675 // generating a split block with a CALL that uses undefined physregs. 676 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments; 677 678 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) { 679 assert(j < ArgLocs.size() && "Skipped too many arg locs"); 680 CCValAssign &VA = ArgLocs[j]; 681 assert(VA.getValNo() == i && "Location doesn't correspond to current arg"); 682 683 if (VA.needsCustom()) { 684 std::function<void()> Thunk; 685 unsigned NumArgRegs = Handler.assignCustomValue( 686 Args[i], ArrayRef(ArgLocs).slice(j), &Thunk); 687 if (Thunk) 688 DelayedOutgoingRegAssignments.emplace_back(Thunk); 689 if (!NumArgRegs) 690 return false; 691 j += NumArgRegs; 692 continue; 693 } 694 695 const MVT ValVT = VA.getValVT(); 696 const MVT LocVT = VA.getLocVT(); 697 698 const LLT LocTy(LocVT); 699 const LLT ValTy(ValVT); 700 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy; 701 const EVT OrigVT = EVT::getEVT(Args[i].Ty); 702 const LLT OrigTy = getLLTForType(*Args[i].Ty, DL); 703 704 // Expected to be multiple regs for a single incoming arg. 705 // There should be Regs.size() ArgLocs per argument. 706 // This should be the same as getNumRegistersForCallingConv 707 const unsigned NumParts = Args[i].Flags.size(); 708 709 // Now split the registers into the assigned types. 710 Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end()); 711 712 if (NumParts != 1 || NewLLT != OrigTy) { 713 // If we can't directly assign the register, we need one or more 714 // intermediate values. 715 Args[i].Regs.resize(NumParts); 716 717 // For each split register, create and assign a vreg that will store 718 // the incoming component of the larger value. These will later be 719 // merged to form the final vreg. 720 for (unsigned Part = 0; Part < NumParts; ++Part) 721 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT); 722 } 723 724 assert((j + (NumParts - 1)) < ArgLocs.size() && 725 "Too many regs for number of args"); 726 727 // Coerce into outgoing value types before register assignment. 728 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) { 729 assert(Args[i].OrigRegs.size() == 1); 730 buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy, 731 ValTy, extendOpFromFlags(Args[i].Flags[0])); 732 } 733 734 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL); 735 for (unsigned Part = 0; Part < NumParts; ++Part) { 736 Register ArgReg = Args[i].Regs[Part]; 737 // There should be Regs.size() ArgLocs per argument. 738 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part; 739 CCValAssign &VA = ArgLocs[j + Idx]; 740 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part]; 741 742 if (VA.isMemLoc() && !Flags.isByVal()) { 743 // Individual pieces may have been spilled to the stack and others 744 // passed in registers. 745 746 // TODO: The memory size may be larger than the value we need to 747 // store. We may need to adjust the offset for big endian targets. 748 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags); 749 750 MachinePointerInfo MPO; 751 Register StackAddr = Handler.getStackAddress( 752 MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags); 753 754 Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA); 755 continue; 756 } 757 758 if (VA.isMemLoc() && Flags.isByVal()) { 759 assert(Args[i].Regs.size() == 1 && 760 "didn't expect split byval pointer"); 761 762 if (Handler.isIncomingArgumentHandler()) { 763 // We just need to copy the frame index value to the pointer. 764 MachinePointerInfo MPO; 765 Register StackAddr = Handler.getStackAddress( 766 Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags); 767 MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr); 768 } else { 769 // For outgoing byval arguments, insert the implicit copy byval 770 // implies, such that writes in the callee do not modify the caller's 771 // value. 772 uint64_t MemSize = Flags.getByValSize(); 773 int64_t Offset = VA.getLocMemOffset(); 774 775 MachinePointerInfo DstMPO; 776 Register StackAddr = 777 Handler.getStackAddress(MemSize, Offset, DstMPO, Flags); 778 779 MachinePointerInfo SrcMPO(Args[i].OrigValue); 780 if (!Args[i].OrigValue) { 781 // We still need to accurately track the stack address space if we 782 // don't know the underlying value. 783 const LLT PtrTy = MRI.getType(StackAddr); 784 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace()); 785 } 786 787 Align DstAlign = std::max(Flags.getNonZeroByValAlign(), 788 inferAlignFromPtrInfo(MF, DstMPO)); 789 790 Align SrcAlign = std::max(Flags.getNonZeroByValAlign(), 791 inferAlignFromPtrInfo(MF, SrcMPO)); 792 793 Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0], 794 DstMPO, DstAlign, SrcMPO, SrcAlign, 795 MemSize, VA); 796 } 797 continue; 798 } 799 800 assert(!VA.needsCustom() && "custom loc should have been handled already"); 801 802 if (i == 0 && !ThisReturnRegs.empty() && 803 Handler.isIncomingArgumentHandler() && 804 isTypeIsValidForThisReturn(ValVT)) { 805 Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA); 806 continue; 807 } 808 809 if (Handler.isIncomingArgumentHandler()) 810 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 811 else { 812 DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() { 813 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 814 }); 815 } 816 } 817 818 // Now that all pieces have been assigned, re-pack the register typed values 819 // into the original value typed registers. 820 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) { 821 // Merge the split registers into the expected larger result vregs of 822 // the original call. 823 buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy, 824 LocTy, Args[i].Flags[0]); 825 } 826 827 j += NumParts - 1; 828 } 829 for (auto &Fn : DelayedOutgoingRegAssignments) 830 Fn(); 831 832 return true; 833 } 834 835 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, 836 ArrayRef<Register> VRegs, Register DemoteReg, 837 int FI) const { 838 MachineFunction &MF = MIRBuilder.getMF(); 839 MachineRegisterInfo &MRI = MF.getRegInfo(); 840 const DataLayout &DL = MF.getDataLayout(); 841 842 SmallVector<EVT, 4> SplitVTs; 843 SmallVector<uint64_t, 4> Offsets; 844 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 845 846 assert(VRegs.size() == SplitVTs.size()); 847 848 unsigned NumValues = SplitVTs.size(); 849 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 850 Type *RetPtrTy = 851 PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace()); 852 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL); 853 854 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); 855 856 for (unsigned I = 0; I < NumValues; ++I) { 857 Register Addr; 858 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 859 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 860 MRI.getType(VRegs[I]), 861 commonAlignment(BaseAlign, Offsets[I])); 862 MIRBuilder.buildLoad(VRegs[I], Addr, *MMO); 863 } 864 } 865 866 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, 867 ArrayRef<Register> VRegs, 868 Register DemoteReg) const { 869 MachineFunction &MF = MIRBuilder.getMF(); 870 MachineRegisterInfo &MRI = MF.getRegInfo(); 871 const DataLayout &DL = MF.getDataLayout(); 872 873 SmallVector<EVT, 4> SplitVTs; 874 SmallVector<uint64_t, 4> Offsets; 875 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); 876 877 assert(VRegs.size() == SplitVTs.size()); 878 879 unsigned NumValues = SplitVTs.size(); 880 Align BaseAlign = DL.getPrefTypeAlign(RetTy); 881 unsigned AS = DL.getAllocaAddrSpace(); 882 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL); 883 884 MachinePointerInfo PtrInfo(AS); 885 886 for (unsigned I = 0; I < NumValues; ++I) { 887 Register Addr; 888 MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]); 889 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 890 MRI.getType(VRegs[I]), 891 commonAlignment(BaseAlign, Offsets[I])); 892 MIRBuilder.buildStore(VRegs[I], Addr, *MMO); 893 } 894 } 895 896 void CallLowering::insertSRetIncomingArgument( 897 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg, 898 MachineRegisterInfo &MRI, const DataLayout &DL) const { 899 unsigned AS = DL.getAllocaAddrSpace(); 900 DemoteReg = MRI.createGenericVirtualRegister( 901 LLT::pointer(AS, DL.getPointerSizeInBits(AS))); 902 903 Type *PtrTy = PointerType::get(F.getReturnType(), AS); 904 905 SmallVector<EVT, 1> ValueVTs; 906 ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs); 907 908 // NOTE: Assume that a pointer won't get split into more than one VT. 909 assert(ValueVTs.size() == 1); 910 911 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()), 912 ArgInfo::NoArgIndex); 913 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F); 914 DemoteArg.Flags[0].setSRet(); 915 SplitArgs.insert(SplitArgs.begin(), DemoteArg); 916 } 917 918 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder, 919 const CallBase &CB, 920 CallLoweringInfo &Info) const { 921 const DataLayout &DL = MIRBuilder.getDataLayout(); 922 Type *RetTy = CB.getType(); 923 unsigned AS = DL.getAllocaAddrSpace(); 924 LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); 925 926 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject( 927 DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false); 928 929 Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0); 930 ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS), 931 ArgInfo::NoArgIndex); 932 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB); 933 DemoteArg.Flags[0].setSRet(); 934 935 Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg); 936 Info.DemoteStackIndex = FI; 937 Info.DemoteRegister = DemoteReg; 938 } 939 940 bool CallLowering::checkReturn(CCState &CCInfo, 941 SmallVectorImpl<BaseArgInfo> &Outs, 942 CCAssignFn *Fn) const { 943 for (unsigned I = 0, E = Outs.size(); I < E; ++I) { 944 MVT VT = MVT::getVT(Outs[I].Ty); 945 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo)) 946 return false; 947 } 948 return true; 949 } 950 951 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy, 952 AttributeList Attrs, 953 SmallVectorImpl<BaseArgInfo> &Outs, 954 const DataLayout &DL) const { 955 LLVMContext &Context = RetTy->getContext(); 956 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 957 958 SmallVector<EVT, 4> SplitVTs; 959 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs); 960 addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex); 961 962 for (EVT VT : SplitVTs) { 963 unsigned NumParts = 964 TLI->getNumRegistersForCallingConv(Context, CallConv, VT); 965 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT); 966 Type *PartTy = EVT(RegVT).getTypeForEVT(Context); 967 968 for (unsigned I = 0; I < NumParts; ++I) { 969 Outs.emplace_back(PartTy, Flags); 970 } 971 } 972 } 973 974 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const { 975 const auto &F = MF.getFunction(); 976 Type *ReturnType = F.getReturnType(); 977 CallingConv::ID CallConv = F.getCallingConv(); 978 979 SmallVector<BaseArgInfo, 4> SplitArgs; 980 getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs, 981 MF.getDataLayout()); 982 return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg()); 983 } 984 985 bool CallLowering::parametersInCSRMatch( 986 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, 987 const SmallVectorImpl<CCValAssign> &OutLocs, 988 const SmallVectorImpl<ArgInfo> &OutArgs) const { 989 for (unsigned i = 0; i < OutLocs.size(); ++i) { 990 const auto &ArgLoc = OutLocs[i]; 991 // If it's not a register, it's fine. 992 if (!ArgLoc.isRegLoc()) 993 continue; 994 995 MCRegister PhysReg = ArgLoc.getLocReg(); 996 997 // Only look at callee-saved registers. 998 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg)) 999 continue; 1000 1001 LLVM_DEBUG( 1002 dbgs() 1003 << "... Call has an argument passed in a callee-saved register.\n"); 1004 1005 // Check if it was copied from. 1006 const ArgInfo &OutInfo = OutArgs[i]; 1007 1008 if (OutInfo.Regs.size() > 1) { 1009 LLVM_DEBUG( 1010 dbgs() << "... Cannot handle arguments in multiple registers.\n"); 1011 return false; 1012 } 1013 1014 // Check if we copy the register, walking through copies from virtual 1015 // registers. Note that getDefIgnoringCopies does not ignore copies from 1016 // physical registers. 1017 MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI); 1018 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) { 1019 LLVM_DEBUG( 1020 dbgs() 1021 << "... Parameter was not copied into a VReg, cannot tail call.\n"); 1022 return false; 1023 } 1024 1025 // Got a copy. Verify that it's the same as the register we want. 1026 Register CopyRHS = RegDef->getOperand(1).getReg(); 1027 if (CopyRHS != PhysReg) { 1028 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into " 1029 "VReg, cannot tail call.\n"); 1030 return false; 1031 } 1032 } 1033 1034 return true; 1035 } 1036 1037 bool CallLowering::resultsCompatible(CallLoweringInfo &Info, 1038 MachineFunction &MF, 1039 SmallVectorImpl<ArgInfo> &InArgs, 1040 ValueAssigner &CalleeAssigner, 1041 ValueAssigner &CallerAssigner) const { 1042 const Function &F = MF.getFunction(); 1043 CallingConv::ID CalleeCC = Info.CallConv; 1044 CallingConv::ID CallerCC = F.getCallingConv(); 1045 1046 if (CallerCC == CalleeCC) 1047 return true; 1048 1049 SmallVector<CCValAssign, 16> ArgLocs1; 1050 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext()); 1051 if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1)) 1052 return false; 1053 1054 SmallVector<CCValAssign, 16> ArgLocs2; 1055 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext()); 1056 if (!determineAssignments(CallerAssigner, InArgs, CCInfo2)) 1057 return false; 1058 1059 // We need the argument locations to match up exactly. If there's more in 1060 // one than the other, then we are done. 1061 if (ArgLocs1.size() != ArgLocs2.size()) 1062 return false; 1063 1064 // Make sure that each location is passed in exactly the same way. 1065 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { 1066 const CCValAssign &Loc1 = ArgLocs1[i]; 1067 const CCValAssign &Loc2 = ArgLocs2[i]; 1068 1069 // We need both of them to be the same. So if one is a register and one 1070 // isn't, we're done. 1071 if (Loc1.isRegLoc() != Loc2.isRegLoc()) 1072 return false; 1073 1074 if (Loc1.isRegLoc()) { 1075 // If they don't have the same register location, we're done. 1076 if (Loc1.getLocReg() != Loc2.getLocReg()) 1077 return false; 1078 1079 // They matched, so we can move to the next ArgLoc. 1080 continue; 1081 } 1082 1083 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. 1084 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) 1085 return false; 1086 } 1087 1088 return true; 1089 } 1090 1091 LLT CallLowering::ValueHandler::getStackValueStoreType( 1092 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const { 1093 const MVT ValVT = VA.getValVT(); 1094 if (ValVT != MVT::iPTR) { 1095 LLT ValTy(ValVT); 1096 1097 // We lost the pointeriness going through CCValAssign, so try to restore it 1098 // based on the flags. 1099 if (Flags.isPointer()) { 1100 LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(), 1101 ValTy.getScalarSizeInBits()); 1102 if (ValVT.isVector()) 1103 return LLT::vector(ValTy.getElementCount(), PtrTy); 1104 return PtrTy; 1105 } 1106 1107 return ValTy; 1108 } 1109 1110 unsigned AddrSpace = Flags.getPointerAddrSpace(); 1111 return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace)); 1112 } 1113 1114 void CallLowering::ValueHandler::copyArgumentMemory( 1115 const ArgInfo &Arg, Register DstPtr, Register SrcPtr, 1116 const MachinePointerInfo &DstPtrInfo, Align DstAlign, 1117 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize, 1118 CCValAssign &VA) const { 1119 MachineFunction &MF = MIRBuilder.getMF(); 1120 MachineMemOperand *SrcMMO = MF.getMachineMemOperand( 1121 SrcPtrInfo, 1122 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize, 1123 SrcAlign); 1124 1125 MachineMemOperand *DstMMO = MF.getMachineMemOperand( 1126 DstPtrInfo, 1127 MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable, 1128 MemSize, DstAlign); 1129 1130 const LLT PtrTy = MRI.getType(DstPtr); 1131 const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits()); 1132 1133 auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize); 1134 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO); 1135 } 1136 1137 Register CallLowering::ValueHandler::extendRegister(Register ValReg, 1138 const CCValAssign &VA, 1139 unsigned MaxSizeBits) { 1140 LLT LocTy{VA.getLocVT()}; 1141 LLT ValTy{VA.getValVT()}; 1142 1143 if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) 1144 return ValReg; 1145 1146 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { 1147 if (MaxSizeBits <= ValTy.getSizeInBits()) 1148 return ValReg; 1149 LocTy = LLT::scalar(MaxSizeBits); 1150 } 1151 1152 const LLT ValRegTy = MRI.getType(ValReg); 1153 if (ValRegTy.isPointer()) { 1154 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so 1155 // we have to cast to do the extension. 1156 LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits()); 1157 ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0); 1158 } 1159 1160 switch (VA.getLocInfo()) { 1161 default: break; 1162 case CCValAssign::Full: 1163 case CCValAssign::BCvt: 1164 // FIXME: bitconverting between vector types may or may not be a 1165 // nop in big-endian situations. 1166 return ValReg; 1167 case CCValAssign::AExt: { 1168 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg); 1169 return MIB.getReg(0); 1170 } 1171 case CCValAssign::SExt: { 1172 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 1173 MIRBuilder.buildSExt(NewReg, ValReg); 1174 return NewReg; 1175 } 1176 case CCValAssign::ZExt: { 1177 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 1178 MIRBuilder.buildZExt(NewReg, ValReg); 1179 return NewReg; 1180 } 1181 } 1182 llvm_unreachable("unable to extend register"); 1183 } 1184 1185 void CallLowering::ValueAssigner::anchor() {} 1186 1187 Register CallLowering::IncomingValueHandler::buildExtensionHint( 1188 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) { 1189 switch (VA.getLocInfo()) { 1190 case CCValAssign::LocInfo::ZExt: { 1191 return MIRBuilder 1192 .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 1193 NarrowTy.getScalarSizeInBits()) 1194 .getReg(0); 1195 } 1196 case CCValAssign::LocInfo::SExt: { 1197 return MIRBuilder 1198 .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg, 1199 NarrowTy.getScalarSizeInBits()) 1200 .getReg(0); 1201 break; 1202 } 1203 default: 1204 return SrcReg; 1205 } 1206 } 1207 1208 /// Check if we can use a basic COPY instruction between the two types. 1209 /// 1210 /// We're currently building on top of the infrastructure using MVT, which loses 1211 /// pointer information in the CCValAssign. We accept copies from physical 1212 /// registers that have been reported as integers if it's to an equivalent sized 1213 /// pointer LLT. 1214 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) { 1215 if (SrcTy == DstTy) 1216 return true; 1217 1218 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) 1219 return false; 1220 1221 SrcTy = SrcTy.getScalarType(); 1222 DstTy = DstTy.getScalarType(); 1223 1224 return (SrcTy.isPointer() && DstTy.isScalar()) || 1225 (DstTy.isPointer() && SrcTy.isScalar()); 1226 } 1227 1228 void CallLowering::IncomingValueHandler::assignValueToReg( 1229 Register ValVReg, Register PhysReg, const CCValAssign &VA) { 1230 const MVT LocVT = VA.getLocVT(); 1231 const LLT LocTy(LocVT); 1232 const LLT RegTy = MRI.getType(ValVReg); 1233 1234 if (isCopyCompatibleType(RegTy, LocTy)) { 1235 MIRBuilder.buildCopy(ValVReg, PhysReg); 1236 return; 1237 } 1238 1239 auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg); 1240 auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy); 1241 MIRBuilder.buildTrunc(ValVReg, Hint); 1242 } 1243