1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/FunctionLoweringInfo.h" 26 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 27 #include "llvm/CodeGen/GlobalISel/Utils.h" 28 #include "llvm/CodeGen/LowLevelType.h" 29 #include "llvm/CodeGen/LowLevelTypeUtils.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineMemOperand.h" 35 #include "llvm/CodeGen/MachineOperand.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/MachineValueType.h" 38 #include "llvm/CodeGen/TargetInstrInfo.h" 39 #include "llvm/CodeGen/TargetSubtargetInfo.h" 40 #include "llvm/CodeGen/ValueTypes.h" 41 #include "llvm/IR/Attributes.h" 42 #include "llvm/IR/DataLayout.h" 43 #include "llvm/IR/Function.h" 44 #include "llvm/IR/Value.h" 45 #include "llvm/MC/MCRegisterInfo.h" 46 #include <cassert> 47 #include <cstdint> 48 49 using namespace llvm; 50 51 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 52 : CallLowering(&TLI) {} 53 54 namespace { 55 56 struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner { 57 private: 58 uint64_t StackSize = 0; 59 unsigned NumXMMRegs = 0; 60 61 public: 62 uint64_t getStackSize() { return StackSize; } 63 unsigned getNumXmmRegs() { return NumXMMRegs; } 64 65 X86OutgoingValueAssigner(CCAssignFn *AssignFn_) 66 : CallLowering::OutgoingValueAssigner(AssignFn_) {} 67 68 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, 69 CCValAssign::LocInfo LocInfo, 70 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 71 CCState &State) override { 72 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 73 StackSize = State.getStackSize(); 74 75 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 76 X86::XMM3, X86::XMM4, X86::XMM5, 77 X86::XMM6, X86::XMM7}; 78 if (!Info.IsFixed) 79 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 80 81 return Res; 82 } 83 }; 84 85 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 86 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 87 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) 88 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), 89 DL(MIRBuilder.getMF().getDataLayout()), 90 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 91 92 Register getStackAddress(uint64_t Size, int64_t Offset, 93 MachinePointerInfo &MPO, 94 ISD::ArgFlagsTy Flags) override { 95 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 96 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 97 auto SPReg = 98 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 99 100 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 101 102 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 103 104 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 105 return AddrReg.getReg(0); 106 } 107 108 void assignValueToReg(Register ValVReg, Register PhysReg, 109 CCValAssign VA) override { 110 MIB.addUse(PhysReg, RegState::Implicit); 111 Register ExtReg = extendRegister(ValVReg, VA); 112 MIRBuilder.buildCopy(PhysReg, ExtReg); 113 } 114 115 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 116 MachinePointerInfo &MPO, CCValAssign &VA) override { 117 MachineFunction &MF = MIRBuilder.getMF(); 118 Register ExtReg = extendRegister(ValVReg, VA); 119 120 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy, 121 inferAlignFromPtrInfo(MF, MPO)); 122 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 123 } 124 125 protected: 126 MachineInstrBuilder &MIB; 127 const DataLayout &DL; 128 const X86Subtarget &STI; 129 }; 130 131 } // end anonymous namespace 132 133 bool X86CallLowering::canLowerReturn( 134 MachineFunction &MF, CallingConv::ID CallConv, 135 SmallVectorImpl<CallLowering::BaseArgInfo> &Outs, bool IsVarArg) const { 136 LLVMContext &Context = MF.getFunction().getContext(); 137 SmallVector<CCValAssign, 16> RVLocs; 138 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 139 return checkReturn(CCInfo, Outs, RetCC_X86); 140 } 141 142 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 143 const Value *Val, ArrayRef<Register> VRegs, 144 FunctionLoweringInfo &FLI) const { 145 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 146 "Return value without a vreg"); 147 MachineFunction &MF = MIRBuilder.getMF(); 148 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 149 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 150 bool Is64Bit = STI.is64Bit(); 151 152 if (!FLI.CanLowerReturn) { 153 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister); 154 MIRBuilder.buildCopy(Is64Bit ? X86::RAX : X86::EAX, FLI.DemoteRegister); 155 } else if (!VRegs.empty()) { 156 const Function &F = MF.getFunction(); 157 MachineRegisterInfo &MRI = MF.getRegInfo(); 158 const DataLayout &DL = MF.getDataLayout(); 159 160 ArgInfo OrigRetInfo(VRegs, Val->getType(), 0); 161 setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F); 162 163 SmallVector<ArgInfo, 4> SplitRetInfos; 164 splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, F.getCallingConv()); 165 166 X86OutgoingValueAssigner Assigner(RetCC_X86); 167 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 168 if (!determineAndHandleAssignments(Handler, Assigner, SplitRetInfos, 169 MIRBuilder, F.getCallingConv(), 170 F.isVarArg())) 171 return false; 172 } 173 174 MIRBuilder.insertInstr(MIB); 175 return true; 176 } 177 178 namespace { 179 180 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 181 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 182 MachineRegisterInfo &MRI) 183 : IncomingValueHandler(MIRBuilder, MRI), 184 DL(MIRBuilder.getMF().getDataLayout()) {} 185 186 Register getStackAddress(uint64_t Size, int64_t Offset, 187 MachinePointerInfo &MPO, 188 ISD::ArgFlagsTy Flags) override { 189 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 190 191 // Byval is assumed to be writable memory, but other stack passed arguments 192 // are not. 193 const bool IsImmutable = !Flags.isByVal(); 194 195 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable); 196 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 197 198 return MIRBuilder 199 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 200 .getReg(0); 201 } 202 203 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 204 MachinePointerInfo &MPO, CCValAssign &VA) override { 205 MachineFunction &MF = MIRBuilder.getMF(); 206 auto *MMO = MF.getMachineMemOperand( 207 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy, 208 inferAlignFromPtrInfo(MF, MPO)); 209 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 210 } 211 212 void assignValueToReg(Register ValVReg, Register PhysReg, 213 CCValAssign VA) override { 214 markPhysRegUsed(PhysReg); 215 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA); 216 } 217 218 /// How the physical register gets marked varies between formal 219 /// parameters (it's a basic-block live-in), and a call instruction 220 /// (it's an implicit-def of the BL). 221 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 222 223 protected: 224 const DataLayout &DL; 225 }; 226 227 struct FormalArgHandler : public X86IncomingValueHandler { 228 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) 229 : X86IncomingValueHandler(MIRBuilder, MRI) {} 230 231 void markPhysRegUsed(unsigned PhysReg) override { 232 MIRBuilder.getMRI()->addLiveIn(PhysReg); 233 MIRBuilder.getMBB().addLiveIn(PhysReg); 234 } 235 }; 236 237 struct CallReturnHandler : public X86IncomingValueHandler { 238 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 239 MachineInstrBuilder &MIB) 240 : X86IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {} 241 242 void markPhysRegUsed(unsigned PhysReg) override { 243 MIB.addDef(PhysReg, RegState::Implicit); 244 } 245 246 protected: 247 MachineInstrBuilder &MIB; 248 }; 249 250 } // end anonymous namespace 251 252 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 253 const Function &F, 254 ArrayRef<ArrayRef<Register>> VRegs, 255 FunctionLoweringInfo &FLI) const { 256 MachineFunction &MF = MIRBuilder.getMF(); 257 MachineRegisterInfo &MRI = MF.getRegInfo(); 258 auto DL = MF.getDataLayout(); 259 260 SmallVector<ArgInfo, 8> SplitArgs; 261 262 if (!FLI.CanLowerReturn) 263 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL); 264 265 // TODO: handle variadic function 266 if (F.isVarArg()) 267 return false; 268 269 unsigned Idx = 0; 270 for (const auto &Arg : F.args()) { 271 // TODO: handle not simple cases. 272 if (Arg.hasAttribute(Attribute::ByVal) || 273 Arg.hasAttribute(Attribute::InReg) || 274 Arg.hasAttribute(Attribute::StructRet) || 275 Arg.hasAttribute(Attribute::SwiftSelf) || 276 Arg.hasAttribute(Attribute::SwiftError) || 277 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 278 return false; 279 280 ArgInfo OrigArg(VRegs[Idx], Arg.getType(), Idx); 281 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 282 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv()); 283 Idx++; 284 } 285 286 if (SplitArgs.empty()) 287 return true; 288 289 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 290 if (!MBB.empty()) 291 MIRBuilder.setInstr(*MBB.begin()); 292 293 X86OutgoingValueAssigner Assigner(CC_X86); 294 FormalArgHandler Handler(MIRBuilder, MRI); 295 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 296 F.getCallingConv(), F.isVarArg())) 297 return false; 298 299 // Move back to the end of the basic block. 300 MIRBuilder.setMBB(MBB); 301 302 return true; 303 } 304 305 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 306 CallLoweringInfo &Info) const { 307 MachineFunction &MF = MIRBuilder.getMF(); 308 const Function &F = MF.getFunction(); 309 MachineRegisterInfo &MRI = MF.getRegInfo(); 310 const DataLayout &DL = F.getParent()->getDataLayout(); 311 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 312 const TargetInstrInfo &TII = *STI.getInstrInfo(); 313 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 314 315 // Handle only Linux C, X86_64_SysV calling conventions for now. 316 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 317 Info.CallConv == CallingConv::X86_64_SysV)) 318 return false; 319 320 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 321 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 322 323 // Create a temporarily-floating call instruction so we can add the implicit 324 // uses of arg registers. 325 bool Is64Bit = STI.is64Bit(); 326 unsigned CallOpc = Info.Callee.isReg() 327 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 328 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 329 330 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 331 .add(Info.Callee) 332 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 333 334 SmallVector<ArgInfo, 8> SplitArgs; 335 for (const auto &OrigArg : Info.OrigArgs) { 336 337 // TODO: handle not simple cases. 338 if (OrigArg.Flags[0].isByVal()) 339 return false; 340 341 if (OrigArg.Regs.size() > 1) 342 return false; 343 344 splitToValueTypes(OrigArg, SplitArgs, DL, Info.CallConv); 345 } 346 // Do the actual argument marshalling. 347 X86OutgoingValueAssigner Assigner(CC_X86); 348 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 349 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 350 Info.CallConv, Info.IsVarArg)) 351 return false; 352 353 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 354 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 355 // From AMD64 ABI document: 356 // For calls that may call functions that use varargs or stdargs 357 // (prototype-less calls or calls to functions containing ellipsis (...) in 358 // the declaration) %al is used as hidden argument to specify the number 359 // of SSE registers used. The contents of %al do not need to match exactly 360 // the number of registers, but must be an ubound on the number of SSE 361 // registers used and is in the range 0 - 8 inclusive. 362 363 MIRBuilder.buildInstr(X86::MOV8ri) 364 .addDef(X86::AL) 365 .addImm(Assigner.getNumXmmRegs()); 366 MIB.addUse(X86::AL, RegState::Implicit); 367 } 368 369 // Now we can add the actual call instruction to the correct basic block. 370 MIRBuilder.insertInstr(MIB); 371 372 // If Callee is a reg, since it is used by a target specific 373 // instruction, it must have a register class matching the 374 // constraint of that instruction. 375 if (Info.Callee.isReg()) 376 MIB->getOperand(0).setReg(constrainOperandRegClass( 377 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 378 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 379 0)); 380 381 // Finally we can copy the returned value back into its virtual-register. In 382 // symmetry with the arguments, the physical register must be an 383 // implicit-define of the call instruction. 384 385 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) { 386 if (Info.OrigRet.Regs.size() > 1) 387 return false; 388 389 SplitArgs.clear(); 390 SmallVector<Register, 8> NewRegs; 391 392 splitToValueTypes(Info.OrigRet, SplitArgs, DL, Info.CallConv); 393 394 X86OutgoingValueAssigner Assigner(RetCC_X86); 395 CallReturnHandler Handler(MIRBuilder, MRI, MIB); 396 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 397 Info.CallConv, Info.IsVarArg)) 398 return false; 399 400 if (!NewRegs.empty()) 401 MIRBuilder.buildMergeLikeInstr(Info.OrigRet.Regs[0], NewRegs); 402 } 403 404 CallSeqStart.addImm(Assigner.getStackSize()) 405 .addImm(0 /* see getFrameTotalSize */) 406 .addImm(0 /* see getFrameAdjustment */); 407 408 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 409 MIRBuilder.buildInstr(AdjStackUp) 410 .addImm(Assigner.getStackSize()) 411 .addImm(0 /* NumBytesForCalleeToPop */); 412 413 if (!Info.CanLowerReturn) 414 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs, 415 Info.DemoteRegister, Info.DemoteStackIndex); 416 417 return true; 418 } 419