diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3330,13 +3330,214 @@ } #endif +/// This is a helper class for lowering variable arguments parameters. +class VarArgsLoweringHelper { +public: + VarArgsLoweringHelper(X86MachineFunctionInfo *FuncInfo, const SDLoc &Loc, + SelectionDAG &DAG, const X86Subtarget &Subtarget, + CallingConv::ID CallConv, CCState &CCInfo) + : FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget), + MachineFunction(DAG.getMachineFunction()), + Function(MachineFunction.getFunction()), + FrameInfo(MachineFunction.getFrameInfo()), + FrameLowering(*Subtarget.getFrameLowering()), + TargLowering(DAG.getTargetLoweringInfo()), CallConv(CallConv), + CCInfo(CCInfo) {} + + // Lower variable arguments parameters. + void lowerVarArgsParameters(SDValue &Chain, unsigned StackSize); + +private: + void createVarArgAreaAndStoreRegisters(SDValue &Chain, unsigned StackSize); + + void forwardMustTailParameters(SDValue &Chain); + + bool is64Bit() { return Subtarget.is64Bit(); } + bool isWin64() { return Subtarget.isCallingConvWin64(CallConv); } + + X86MachineFunctionInfo *FuncInfo; + const SDLoc &DL; + SelectionDAG &DAG; + const X86Subtarget &Subtarget; + MachineFunction &MachineFunction; + const Function &Function; + MachineFrameInfo &FrameInfo; + const TargetFrameLowering &FrameLowering; + const TargetLowering &TargLowering; + CallingConv::ID CallConv; + CCState &CCInfo; +}; + +void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters( + SDValue &Chain, unsigned StackSize) { + // If the function takes variable number of arguments, make a frame index for + // the start of the first vararg value... for expansion of llvm.va_start. We + // can skip this if there are no va_start calls. + if (is64Bit() || (CallConv != CallingConv::X86_FastCall && + CallConv != CallingConv::X86_ThisCall)) { + FuncInfo->setVarArgsFrameIndex( + FrameInfo.CreateFixedObject(1, StackSize, true)); + } + + // Figure out if XMM registers are in use. + assert(!(Subtarget.useSoftFloat() && + Function.hasFnAttribute(Attribute::NoImplicitFloat)) && + "SSE register cannot be used when SSE is disabled!"); + + // 64-bit calling conventions support varargs and register parameters, so we + // have to do extra work to spill them in the prologue. + if (is64Bit()) { + // Find the first unallocated argument registers. + ArrayRef ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget); + ArrayRef ArgXMMs = + get64BitArgumentXMMs(MachineFunction, CallConv, Subtarget); + unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs); + unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs); + + assert(!(NumXMMRegs && !Subtarget.hasSSE1()) && + "SSE register cannot be used when SSE is disabled!"); + + if (isWin64()) { + // Get to the caller-allocated home save location. Add 8 to account + // for the return address. + int HomeOffset = FrameLowering.getOffsetOfLocalArea() + 8; + FuncInfo->setRegSaveFrameIndex( + FrameInfo.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false)); + // Fixup to set vararg frame on shadow area (4 x i64). + if (NumIntRegs < 4) + FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex()); + } else { + // For X86-64, if there are vararg parameters that are passed via + // registers, then we must store them to their spots on the stack so + // they may be loaded by dereferencing the result of va_next. + FuncInfo->setVarArgsGPOffset(NumIntRegs * 8); + FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16); + FuncInfo->setRegSaveFrameIndex(FrameInfo.CreateStackObject( + ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false)); + } + + SmallVector + LiveGPRs; // list of SDValue for GPR registers keeping live input value + SmallVector LiveXMMRegs; // list of SDValue for XMM registers + // keeping live input value + SDValue ALVal; // if applicable keeps SDValue for %al register + + // Gather all the live in physical registers. + for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) { + unsigned GPR = MachineFunction.addLiveIn(Reg, &X86::GR64RegClass); + LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64)); + } + const auto &AvailableXmms = ArgXMMs.slice(NumXMMRegs); + if (!AvailableXmms.empty()) { + unsigned AL = MachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass); + ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8); + for (MCPhysReg Reg : AvailableXmms) { + unsigned XMMReg = MachineFunction.addLiveIn(Reg, &X86::VR128RegClass); + LiveXMMRegs.push_back( + DAG.getCopyFromReg(Chain, DL, XMMReg, MVT::v4f32)); + } + } + + // Store the integer parameter registers. + SmallVector MemOps; + SDValue RSFIN = + DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), + TargLowering.getPointerTy(DAG.getDataLayout())); + unsigned Offset = FuncInfo->getVarArgsGPOffset(); + for (SDValue Val : LiveGPRs) { + SDValue FIN = DAG.getNode(ISD::ADD, DL, + TargLowering.getPointerTy(DAG.getDataLayout()), + RSFIN, DAG.getIntPtrConstant(Offset, DL)); + SDValue Store = + DAG.getStore(Val.getValue(1), DL, Val, FIN, + MachinePointerInfo::getFixedStack( + DAG.getMachineFunction(), + FuncInfo->getRegSaveFrameIndex(), Offset)); + MemOps.push_back(Store); + Offset += 8; + } + + // Now store the XMM (fp + vector) parameter registers. + if (!LiveXMMRegs.empty()) { + SmallVector SaveXMMOps; + SaveXMMOps.push_back(Chain); + SaveXMMOps.push_back(ALVal); + SaveXMMOps.push_back( + DAG.getIntPtrConstant(FuncInfo->getRegSaveFrameIndex(), DL)); + SaveXMMOps.push_back( + DAG.getIntPtrConstant(FuncInfo->getVarArgsFPOffset(), DL)); + SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(), + LiveXMMRegs.end()); + MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, DL, + MVT::Other, SaveXMMOps)); + } + + if (!MemOps.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); + } +} + +void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) { + // Find the largest legal vector type. + MVT VecVT = MVT::Other; + // FIXME: Only some x86_32 calling conventions support AVX512. + if (Subtarget.useAVX512Regs() && + (is64Bit() || (CallConv == CallingConv::X86_VectorCall || + CallConv == CallingConv::Intel_OCL_BI))) + VecVT = MVT::v16f32; + else if (Subtarget.hasAVX()) + VecVT = MVT::v8f32; + else if (Subtarget.hasSSE2()) + VecVT = MVT::v4f32; + + // We forward some GPRs and some vector types. + SmallVector RegParmTypes; + MVT IntVT = is64Bit() ? MVT::i64 : MVT::i32; + RegParmTypes.push_back(IntVT); + if (VecVT != MVT::Other) + RegParmTypes.push_back(VecVT); + + // Compute the set of forwarded registers. The rest are scratch. + SmallVectorImpl &Forwards = + FuncInfo->getForwardedMustTailRegParms(); + CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86); + + // Forward AL for SysV x86_64 targets, since it is used for varargs. + if (is64Bit() && !isWin64() && !CCInfo.isAllocated(X86::AL)) { + unsigned ALVReg = MachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass); + Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); + } + + // Copy all forwards from physical to virtual registers. + for (ForwardedRegister &FR : Forwards) { + // FIXME: Can we use a less constrained schedule? + SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT); + FR.VReg = MachineFunction.getRegInfo().createVirtualRegister( + TargLowering.getRegClassFor(FR.VT)); + Chain = DAG.getCopyToReg(Chain, DL, FR.VReg, RegVal); + } +} + +void VarArgsLoweringHelper::lowerVarArgsParameters(SDValue &Chain, + unsigned StackSize) { + // Set FrameIndex to the 0xAAAAAAA value to mark unset state. + // If necessary, it would be set into the correct value later. + FuncInfo->setVarArgsFrameIndex(0xAAAAAAA); + FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); + + if (FrameInfo.hasVAStart()) + createVarArgAreaAndStoreRegisters(Chain, StackSize); + + if (FrameInfo.hasMustTailInVarArgFunc()) + forwardMustTailParameters(Chain); +} + SDValue X86TargetLowering::LowerFormalArguments( - SDValue Chain, CallingConv::ID CallConv, bool isVarArg, + SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo(); - const TargetFrameLowering &TFI = *Subtarget.getFrameLowering(); const Function &F = MF.getFunction(); if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() && @@ -3348,12 +3549,12 @@ bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); assert( - !(isVarArg && canGuaranteeTCO(CallConv)) && + !(IsVarArg && canGuaranteeTCO(CallConv)) && "Var args not supported with calling conv' regcall, fastcc, ghc or hipe"); // Assign locations to all of the incoming arguments. SmallVector ArgLocs; - CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); + CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); // Allocate shadow area for Win64. if (IsWin64) @@ -3500,147 +3701,12 @@ MF.getTarget().Options.GuaranteedTailCallOpt)) StackSize = GetAlignedArgumentStackSize(StackSize, DAG); - // If the function takes variable number of arguments, make a frame index for - // the start of the first vararg value... for expansion of llvm.va_start. We - // can skip this if there are no va_start calls. - if (MFI.hasVAStart() && - (Is64Bit || (CallConv != CallingConv::X86_FastCall && - CallConv != CallingConv::X86_ThisCall))) { - FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true)); - } - - // Figure out if XMM registers are in use. - assert(!(Subtarget.useSoftFloat() && - F.hasFnAttribute(Attribute::NoImplicitFloat)) && - "SSE register cannot be used when SSE is disabled!"); - - // 64-bit calling conventions support varargs and register parameters, so we - // have to do extra work to spill them in the prologue. - if (Is64Bit && isVarArg && MFI.hasVAStart()) { - // Find the first unallocated argument registers. - ArrayRef ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget); - ArrayRef ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget); - unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs); - unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs); - assert(!(NumXMMRegs && !Subtarget.hasSSE1()) && - "SSE register cannot be used when SSE is disabled!"); - - // Gather all the live in physical registers. - SmallVector LiveGPRs; - SmallVector LiveXMMRegs; - SDValue ALVal; - for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) { - unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass); - LiveGPRs.push_back( - DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64)); - } - if (!ArgXMMs.empty()) { - unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass); - ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8); - for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) { - unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass); - LiveXMMRegs.push_back( - DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32)); - } - } - - if (IsWin64) { - // Get to the caller-allocated home save location. Add 8 to account - // for the return address. - int HomeOffset = TFI.getOffsetOfLocalArea() + 8; - FuncInfo->setRegSaveFrameIndex( - MFI.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false)); - // Fixup to set vararg frame on shadow area (4 x i64). - if (NumIntRegs < 4) - FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex()); - } else { - // For X86-64, if there are vararg parameters that are passed via - // registers, then we must store them to their spots on the stack so - // they may be loaded by dereferencing the result of va_next. - FuncInfo->setVarArgsGPOffset(NumIntRegs * 8); - FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16); - FuncInfo->setRegSaveFrameIndex(MFI.CreateStackObject( - ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false)); - } - - // Store the integer parameter registers. - SmallVector MemOps; - SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), - getPointerTy(DAG.getDataLayout())); - unsigned Offset = FuncInfo->getVarArgsGPOffset(); - for (SDValue Val : LiveGPRs) { - SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), - RSFIN, DAG.getIntPtrConstant(Offset, dl)); - SDValue Store = - DAG.getStore(Val.getValue(1), dl, Val, FIN, - MachinePointerInfo::getFixedStack( - DAG.getMachineFunction(), - FuncInfo->getRegSaveFrameIndex(), Offset)); - MemOps.push_back(Store); - Offset += 8; - } - - if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) { - // Now store the XMM (fp + vector) parameter registers. - SmallVector SaveXMMOps; - SaveXMMOps.push_back(Chain); - SaveXMMOps.push_back(ALVal); - SaveXMMOps.push_back(DAG.getIntPtrConstant( - FuncInfo->getRegSaveFrameIndex(), dl)); - SaveXMMOps.push_back(DAG.getIntPtrConstant( - FuncInfo->getVarArgsFPOffset(), dl)); - SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(), - LiveXMMRegs.end()); - MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, - MVT::Other, SaveXMMOps)); - } - - if (!MemOps.empty()) - Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); - } - - if (isVarArg && MFI.hasMustTailInVarArgFunc()) { - // Find the largest legal vector type. - MVT VecVT = MVT::Other; - // FIXME: Only some x86_32 calling conventions support AVX512. - if (Subtarget.useAVX512Regs() && - (Is64Bit || (CallConv == CallingConv::X86_VectorCall || - CallConv == CallingConv::Intel_OCL_BI))) - VecVT = MVT::v16f32; - else if (Subtarget.hasAVX()) - VecVT = MVT::v8f32; - else if (Subtarget.hasSSE2()) - VecVT = MVT::v4f32; - - // We forward some GPRs and some vector types. - SmallVector RegParmTypes; - MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32; - RegParmTypes.push_back(IntVT); - if (VecVT != MVT::Other) - RegParmTypes.push_back(VecVT); - - // Compute the set of forwarded registers. The rest are scratch. - SmallVectorImpl &Forwards = - FuncInfo->getForwardedMustTailRegParms(); - CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86); - - // Forward AL for SysV x86_64 targets, since it is used for varargs. - if (Is64Bit && !IsWin64 && !CCInfo.isAllocated(X86::AL)) { - unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass); - Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); - } - - // Copy all forwards from physical to virtual registers. - for (ForwardedRegister &FR : Forwards) { - // FIXME: Can we use a less constrained schedule? - SDValue RegVal = DAG.getCopyFromReg(Chain, dl, FR.VReg, FR.VT); - FR.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(FR.VT)); - Chain = DAG.getCopyToReg(Chain, dl, FR.VReg, RegVal); - } - } + if (IsVarArg) + VarArgsLoweringHelper(FuncInfo, dl, DAG, Subtarget, CallConv, CCInfo) + .lowerVarArgsParameters(Chain, StackSize); // Some CCs need callee pop. - if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, + if (X86::isCalleePop(CallConv, Is64Bit, IsVarArg, MF.getTarget().Options.GuaranteedTailCallOpt)) { FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything. } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) { @@ -3659,10 +3725,6 @@ if (!Is64Bit) { // RegSaveFrameIndex is X86-64 only. FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); - if (CallConv == CallingConv::X86_FastCall || - CallConv == CallingConv::X86_ThisCall) - // fastcc functions can't have varargs. - FuncInfo->setVarArgsFrameIndex(0xAAAAAAA); } FuncInfo->setArgumentStackSize(StackSize); @@ -3789,7 +3851,6 @@ Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); MachineFunction::CallSiteInfo CSInfo; - if (CallConv == CallingConv::X86_INTR) report_fatal_error("X86 interrupts may not be called directly"); @@ -4051,7 +4112,6 @@ unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs); assert((Subtarget.hasSSE1() || !NumXMMRegs) && "SSE registers cannot be used when SSE is disabled"); - RegsToPass.push_back(std::make_pair(unsigned(X86::AL), DAG.getConstant(NumXMMRegs, dl, MVT::i8)));