1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86MachineFunctionInfo.h" 20 #include "X86RegisterInfo.h" 21 #include "X86Subtarget.h" 22 #include "llvm/ADT/ArrayRef.h" 23 #include "llvm/ADT/SmallVector.h" 24 #include "llvm/CodeGen/Analysis.h" 25 #include "llvm/CodeGen/CallingConvLower.h" 26 #include "llvm/CodeGen/FunctionLoweringInfo.h" 27 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 28 #include "llvm/CodeGen/GlobalISel/Utils.h" 29 #include "llvm/CodeGen/LowLevelTypeUtils.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineMemOperand.h" 35 #include "llvm/CodeGen/MachineOperand.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/TargetInstrInfo.h" 38 #include "llvm/CodeGen/TargetSubtargetInfo.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/CodeGenTypes/LowLevelType.h" 41 #include "llvm/CodeGenTypes/MachineValueType.h" 42 #include "llvm/IR/Attributes.h" 43 #include "llvm/IR/DataLayout.h" 44 #include "llvm/IR/Function.h" 45 #include "llvm/IR/Value.h" 46 #include <cassert> 47 #include <cstdint> 48 49 using namespace llvm; 50 51 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 52 : CallLowering(&TLI) {} 53 54 namespace { 55 56 struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner { 57 private: 58 uint64_t StackSize = 0; 59 unsigned NumXMMRegs = 0; 60 61 public: 62 uint64_t getStackSize() { return StackSize; } 63 unsigned getNumXmmRegs() { return NumXMMRegs; } 64 65 X86OutgoingValueAssigner(CCAssignFn *AssignFn_) 66 : CallLowering::OutgoingValueAssigner(AssignFn_) {} 67 68 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, 69 CCValAssign::LocInfo LocInfo, 70 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 71 CCState &State) override { 72 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 73 StackSize = State.getStackSize(); 74 75 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 76 X86::XMM3, X86::XMM4, X86::XMM5, 77 X86::XMM6, X86::XMM7}; 78 if (!Info.IsFixed) 79 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 80 81 return Res; 82 } 83 }; 84 85 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 86 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 87 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) 88 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), 89 DL(MIRBuilder.getMF().getDataLayout()), 90 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 91 92 Register getStackAddress(uint64_t Size, int64_t Offset, 93 MachinePointerInfo &MPO, 94 ISD::ArgFlagsTy Flags) override { 95 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 96 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 97 auto SPReg = 98 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 99 100 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 101 102 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 103 104 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 105 return AddrReg.getReg(0); 106 } 107 108 void assignValueToReg(Register ValVReg, Register PhysReg, 109 const CCValAssign &VA) override { 110 MIB.addUse(PhysReg, RegState::Implicit); 111 Register ExtReg = extendRegister(ValVReg, VA); 112 MIRBuilder.buildCopy(PhysReg, ExtReg); 113 } 114 115 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 116 const MachinePointerInfo &MPO, 117 const CCValAssign &VA) override { 118 MachineFunction &MF = MIRBuilder.getMF(); 119 Register ExtReg = extendRegister(ValVReg, VA); 120 121 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy, 122 inferAlignFromPtrInfo(MF, MPO)); 123 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 124 } 125 126 protected: 127 MachineInstrBuilder &MIB; 128 const DataLayout &DL; 129 const X86Subtarget &STI; 130 }; 131 132 } // end anonymous namespace 133 134 bool X86CallLowering::canLowerReturn( 135 MachineFunction &MF, CallingConv::ID CallConv, 136 SmallVectorImpl<CallLowering::BaseArgInfo> &Outs, bool IsVarArg) const { 137 LLVMContext &Context = MF.getFunction().getContext(); 138 SmallVector<CCValAssign, 16> RVLocs; 139 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 140 return checkReturn(CCInfo, Outs, RetCC_X86); 141 } 142 143 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 144 const Value *Val, ArrayRef<Register> VRegs, 145 FunctionLoweringInfo &FLI) const { 146 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 147 "Return value without a vreg"); 148 MachineFunction &MF = MIRBuilder.getMF(); 149 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 150 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 151 const auto &STI = MF.getSubtarget<X86Subtarget>(); 152 Register RetReg = STI.is64Bit() ? X86::RAX : X86::EAX; 153 154 if (!FLI.CanLowerReturn) { 155 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister); 156 MIRBuilder.buildCopy(RetReg, FLI.DemoteRegister); 157 MIB.addReg(RetReg); 158 } else if (Register Reg = FuncInfo->getSRetReturnReg()) { 159 MIRBuilder.buildCopy(RetReg, Reg); 160 MIB.addReg(RetReg); 161 } else if (!VRegs.empty()) { 162 const Function &F = MF.getFunction(); 163 MachineRegisterInfo &MRI = MF.getRegInfo(); 164 const DataLayout &DL = MF.getDataLayout(); 165 166 ArgInfo OrigRetInfo(VRegs, Val->getType(), 0); 167 setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F); 168 169 SmallVector<ArgInfo, 4> SplitRetInfos; 170 splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, F.getCallingConv()); 171 172 X86OutgoingValueAssigner Assigner(RetCC_X86); 173 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 174 if (!determineAndHandleAssignments(Handler, Assigner, SplitRetInfos, 175 MIRBuilder, F.getCallingConv(), 176 F.isVarArg())) 177 return false; 178 } 179 180 MIRBuilder.insertInstr(MIB); 181 return true; 182 } 183 184 namespace { 185 186 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 187 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 188 MachineRegisterInfo &MRI) 189 : IncomingValueHandler(MIRBuilder, MRI), 190 DL(MIRBuilder.getMF().getDataLayout()) {} 191 192 Register getStackAddress(uint64_t Size, int64_t Offset, 193 MachinePointerInfo &MPO, 194 ISD::ArgFlagsTy Flags) override { 195 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 196 197 // Byval is assumed to be writable memory, but other stack passed arguments 198 // are not. 199 const bool IsImmutable = !Flags.isByVal(); 200 201 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable); 202 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 203 204 return MIRBuilder 205 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 206 .getReg(0); 207 } 208 209 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 210 const MachinePointerInfo &MPO, 211 const CCValAssign &VA) override { 212 MachineFunction &MF = MIRBuilder.getMF(); 213 auto *MMO = MF.getMachineMemOperand( 214 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy, 215 inferAlignFromPtrInfo(MF, MPO)); 216 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 217 } 218 219 void assignValueToReg(Register ValVReg, Register PhysReg, 220 const CCValAssign &VA) override { 221 markPhysRegUsed(PhysReg); 222 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA); 223 } 224 225 /// How the physical register gets marked varies between formal 226 /// parameters (it's a basic-block live-in), and a call instruction 227 /// (it's an implicit-def of the BL). 228 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 229 230 protected: 231 const DataLayout &DL; 232 }; 233 234 struct FormalArgHandler : public X86IncomingValueHandler { 235 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) 236 : X86IncomingValueHandler(MIRBuilder, MRI) {} 237 238 void markPhysRegUsed(unsigned PhysReg) override { 239 MIRBuilder.getMRI()->addLiveIn(PhysReg); 240 MIRBuilder.getMBB().addLiveIn(PhysReg); 241 } 242 }; 243 244 struct CallReturnHandler : public X86IncomingValueHandler { 245 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 246 MachineInstrBuilder &MIB) 247 : X86IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {} 248 249 void markPhysRegUsed(unsigned PhysReg) override { 250 MIB.addDef(PhysReg, RegState::Implicit); 251 } 252 253 protected: 254 MachineInstrBuilder &MIB; 255 }; 256 257 } // end anonymous namespace 258 259 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 260 const Function &F, 261 ArrayRef<ArrayRef<Register>> VRegs, 262 FunctionLoweringInfo &FLI) const { 263 MachineFunction &MF = MIRBuilder.getMF(); 264 MachineRegisterInfo &MRI = MF.getRegInfo(); 265 auto DL = MF.getDataLayout(); 266 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 267 268 SmallVector<ArgInfo, 8> SplitArgs; 269 270 if (!FLI.CanLowerReturn) 271 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL); 272 273 // TODO: handle variadic function 274 if (F.isVarArg()) 275 return false; 276 277 unsigned Idx = 0; 278 for (const auto &Arg : F.args()) { 279 // TODO: handle not simple cases. 280 if (Arg.hasAttribute(Attribute::ByVal) || 281 Arg.hasAttribute(Attribute::InReg) || 282 Arg.hasAttribute(Attribute::SwiftSelf) || 283 Arg.hasAttribute(Attribute::SwiftError) || 284 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 285 return false; 286 287 if (Arg.hasAttribute(Attribute::StructRet)) { 288 assert(VRegs[Idx].size() == 1 && 289 "Unexpected amount of registers for sret argument."); 290 FuncInfo->setSRetReturnReg(VRegs[Idx][0]); 291 } 292 293 ArgInfo OrigArg(VRegs[Idx], Arg.getType(), Idx); 294 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 295 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv()); 296 Idx++; 297 } 298 299 if (SplitArgs.empty()) 300 return true; 301 302 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 303 if (!MBB.empty()) 304 MIRBuilder.setInstr(*MBB.begin()); 305 306 X86OutgoingValueAssigner Assigner(CC_X86); 307 FormalArgHandler Handler(MIRBuilder, MRI); 308 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 309 F.getCallingConv(), F.isVarArg())) 310 return false; 311 312 // Move back to the end of the basic block. 313 MIRBuilder.setMBB(MBB); 314 315 return true; 316 } 317 318 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 319 CallLoweringInfo &Info) const { 320 MachineFunction &MF = MIRBuilder.getMF(); 321 const Function &F = MF.getFunction(); 322 MachineRegisterInfo &MRI = MF.getRegInfo(); 323 const DataLayout &DL = F.getDataLayout(); 324 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 325 const TargetInstrInfo &TII = *STI.getInstrInfo(); 326 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 327 328 // Handle only Linux C, X86_64_SysV calling conventions for now. 329 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 330 Info.CallConv == CallingConv::X86_64_SysV)) 331 return false; 332 333 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 334 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 335 336 // Create a temporarily-floating call instruction so we can add the implicit 337 // uses of arg registers. 338 bool Is64Bit = STI.is64Bit(); 339 unsigned CallOpc = Info.Callee.isReg() 340 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 341 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 342 343 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 344 .add(Info.Callee) 345 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 346 347 SmallVector<ArgInfo, 8> SplitArgs; 348 for (const auto &OrigArg : Info.OrigArgs) { 349 350 // TODO: handle not simple cases. 351 if (OrigArg.Flags[0].isByVal()) 352 return false; 353 354 if (OrigArg.Regs.size() > 1) 355 return false; 356 357 splitToValueTypes(OrigArg, SplitArgs, DL, Info.CallConv); 358 } 359 // Do the actual argument marshalling. 360 X86OutgoingValueAssigner Assigner(CC_X86); 361 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 362 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 363 Info.CallConv, Info.IsVarArg)) 364 return false; 365 366 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 367 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 368 // From AMD64 ABI document: 369 // For calls that may call functions that use varargs or stdargs 370 // (prototype-less calls or calls to functions containing ellipsis (...) in 371 // the declaration) %al is used as hidden argument to specify the number 372 // of SSE registers used. The contents of %al do not need to match exactly 373 // the number of registers, but must be an ubound on the number of SSE 374 // registers used and is in the range 0 - 8 inclusive. 375 376 MIRBuilder.buildInstr(X86::MOV8ri) 377 .addDef(X86::AL) 378 .addImm(Assigner.getNumXmmRegs()); 379 MIB.addUse(X86::AL, RegState::Implicit); 380 } 381 382 // Now we can add the actual call instruction to the correct basic block. 383 MIRBuilder.insertInstr(MIB); 384 385 // If Callee is a reg, since it is used by a target specific 386 // instruction, it must have a register class matching the 387 // constraint of that instruction. 388 if (Info.Callee.isReg()) 389 MIB->getOperand(0).setReg(constrainOperandRegClass( 390 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 391 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 392 0)); 393 394 // Finally we can copy the returned value back into its virtual-register. In 395 // symmetry with the arguments, the physical register must be an 396 // implicit-define of the call instruction. 397 398 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) { 399 if (Info.OrigRet.Regs.size() > 1) 400 return false; 401 402 SplitArgs.clear(); 403 SmallVector<Register, 8> NewRegs; 404 405 splitToValueTypes(Info.OrigRet, SplitArgs, DL, Info.CallConv); 406 407 X86OutgoingValueAssigner Assigner(RetCC_X86); 408 CallReturnHandler Handler(MIRBuilder, MRI, MIB); 409 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 410 Info.CallConv, Info.IsVarArg)) 411 return false; 412 413 if (!NewRegs.empty()) 414 MIRBuilder.buildMergeLikeInstr(Info.OrigRet.Regs[0], NewRegs); 415 } 416 417 CallSeqStart.addImm(Assigner.getStackSize()) 418 .addImm(0 /* see getFrameTotalSize */) 419 .addImm(0 /* see getFrameAdjustment */); 420 421 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 422 MIRBuilder.buildInstr(AdjStackUp) 423 .addImm(Assigner.getStackSize()) 424 .addImm(0 /* NumBytesForCalleeToPop */); 425 426 if (!Info.CanLowerReturn) 427 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs, 428 Info.DemoteRegister, Info.DemoteStackIndex); 429 430 return true; 431 } 432