Implement the x86 lowering for inalloca

[rnk]

Summary

This implements inalloca lowering in SelectionDAGBuilder, which is
target independent. The core idea is to move the frame adjustment from
the CALLSEQ_START node to a dominating DYNAMIC_STACKALLOC and let the
frontend emit stackrestore calls to clear the stack when necessary.

With the current rules for inalloca, the last inalloca argument must
dominate the rest. For simplicity, I emit code for all inalloca
arguments for a call at the site of the last inalloca alloca.

I added a new target hook, AnalyzeCallArgs, which is split from
LowerCallTo. It exposes the argument frame size and the locations of
all the arguments to the SD builder. With that, I can create an
appopriately sized DYNAMIC_STACKALLOC and set the virtual result
registers for the allocas to the appropriate stack offsets.

Depends On: D2449

[sunfish]

Index: include/llvm/Target/TargetCallingConv.h

• include/llvm/Target/TargetCallingConv.h

+++ include/llvm/Target/TargetCallingConv.h
@@ -42,6 +42,8 @@

static const uint64_t ByValAlignOffs = 7;
static const uint64_t Split          = 1ULL<<11;
static const uint64_t SplitOffs      = 11;

+ static const uint64_t InAlloca = 1ULL<<12; ///< Passed in alloca
+ static const uint64_t InAllocaOffs = 12;

Change the comment to say "Passed with inalloca" since inalloca means a bunch of things beyond just using an alloca.

Index: include/llvm/Target/TargetLowering.h

• include/llvm/Target/TargetLowering.h

+++ include/llvm/Target/TargetLowering.h
@@ -24,6 +24,7 @@
#define LLVM_TARGET_TARGETLOWERING_H

#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/CallingConvLower.h"

Is this #include needed? It looks like it's for CallLoweringInfo and CCState, which could
be forward-declared.

Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

• lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

+++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

[...]

+ Dynamically allocate space for the stack frame.
+ TODO: Can we support args aligned to greater than the stack alignment?
+ unsigned Align = 0;

The (unchecked) assumption that default stack alignment is enough seems dangerous. How hard would it be to implement this? Is it just a matter of computing the greatest alignment from the arguments, or does it require additional codegen support?

[...]

+ if (Args[i].isInAlloca) {
+ Flags.setByVal(); // TODO: Remove when isInAlloca() works.

Does isInAlloca not work yet?

Index: lib/Target/X86/X86ISelLowering.cpp

• lib/Target/X86/X86ISelLowering.cpp

+++ lib/Target/X86/X86ISelLowering.cpp
@@ -2501,6 +2501,21 @@

return Chain;

}

+void X86TargetLowering::AnalyzeCallArgs(TargetLowering::CallLoweringInfo &CLI,
+ CCState &CCInfo) const {
+ SelectionDAG &DAG = CLI.DAG;
+ SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
+ CallingConv::ID CallConv = CLI.CallConv;
+ bool isVarArg = CLI.IsVarArg;
+ MachineFunction &MF = DAG.getMachineFunction();

DAG, isVarArg, and MF are unused here.

[rnk]

Thanks!

[Eugene.Zelenko]

Some regression were committed as part of D2637. Is this patch obsolete?

[rnk]

We changed the design to make this patch obsolete. D2637 / rL200596 implement the current design.

include/llvm/Target/TargetCallingConv.h

private:
     static const uint64_t Nest           = 1ULL<<5;  ///< Nested fn static chain
     static const uint64_t NestOffs       = 5;
     static const uint64_t Returned       = 1ULL<<6;  ///< Always returned
     static const uint64_t ReturnedOffs   = 6;
     static const uint64_t ByValAlign     = 0xFULL<<7; ///< Struct alignment
     static const uint64_t ByValAlignOffs = 7;
     static const uint64_t Split          = 1ULL<<11;
     static const uint64_t SplitOffs      = 11;
+    static const uint64_t InAlloca       = 1ULL<<12; ///< Passed in alloca
+    static const uint64_t InAllocaOffs   = 12;
     static const uint64_t OrigAlign      = 0x1FULL<<27;
     static const uint64_t OrigAlignOffs  = 27;
     static const uint64_t ByValSize      = 0xffffffffULL<<32; ///< Struct size
     static const uint64_t ByValSizeOffs  = 32;
 
     static const uint64_t One            = 1ULL; ///< 1 of this type, for shifts
 
     uint64_t Flags;
public:
     void setInReg()    { Flags |= One << InRegOffs; }
 
     bool isSRet()      const { return Flags & SRet; }
     void setSRet()     { Flags |= One << SRetOffs; }
 
     bool isByVal()     const { return Flags & ByVal; }
     void setByVal()    { Flags |= One << ByValOffs; }
 
+    bool isInAlloca()  const { return Flags & InAlloca; }
+    void setInAlloca() { Flags |= One << InAllocaOffs; }
+
     bool isNest()      const { return Flags & Nest; }
     void setNest()     { Flags |= One << NestOffs; }
 
     bool isReturned()  const { return Flags & Returned; }
     void setReturned() { Flags |= One << ReturnedOffs; }
 
     unsigned getByValAlign() const {
       return (unsigned)

include/llvm/Target/TargetLowering.h

 /// immediates.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TARGET_TARGETLOWERING_H
 #define LLVM_TARGET_TARGETLOWERING_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/DAGCombine.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Target/TargetCallingConv.h"
EVT getValueType(Type *Ty, bool AllowUnknown = false) const {
     return EVT::getEVT(Ty, AllowUnknown);
   }
 
   /// Return the MVT corresponding to this LLVM type. See getValueType.
   MVT getSimpleValueType(Type *Ty, bool AllowUnknown = false) const {
     return getValueType(Ty, AllowUnknown).getSimpleVT();
   }
 
-  /// Return the desired alignment for ByVal aggregate function arguments in the
-  /// caller parameter area.  This is the actual alignment, not its logarithm.
+  /// Return the desired alignment for ByVal or InAlloca aggregate function
+  /// arguments in the caller parameter area.  This is the actual alignment, not
+  /// its logarithm.
   virtual unsigned getByValTypeAlignment(Type *Ty) const;
 
   /// Return the type of registers that this ValueType will eventually require.
   MVT getRegisterType(MVT VT) const {
     assert((unsigned)VT.SimpleTy < array_lengthof(RegisterTypeForVT));
     return RegisterTypeForVT[VT.SimpleTy];
   }
 
struct ArgListEntry {
     SDValue Node;
     Type* Ty;
     bool isSExt     : 1;
     bool isZExt     : 1;
     bool isInReg    : 1;
     bool isSRet     : 1;
     bool isNest     : 1;
     bool isByVal    : 1;
+    bool isInAlloca : 1;
     bool isReturned : 1;
     uint16_t Alignment;
 
     ArgListEntry() : isSExt(false), isZExt(false), isInReg(false),
-      isSRet(false), isNest(false), isByVal(false), isReturned(false),
-      Alignment(0) { }
+      isSRet(false), isNest(false), isByVal(false), isInAlloca(false),
+      isReturned(false), Alignment(0) { }
 
     void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx);
   };
   typedef std::vector<ArgListEntry> ArgListTy;
 
   /// This structure contains all information that is necessary for lowering
   /// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
   /// needs to lower a call, and targets will see this struct in their LowerCall
public:
   /// array. The implementation should fill in the InVals array with legal-type
   /// return values from the call, and return the resulting token chain value.
   virtual SDValue
     LowerCall(CallLoweringInfo &/*CLI*/,
               SmallVectorImpl<SDValue> &/*InVals*/) const {
     llvm_unreachable("Not Implemented");
   }
 
+  void AnalyzeCallOutArgs(CallLoweringInfo &CLI, bool EmitNodes) const;
+
+  /// Analyze the call to find the argument locations without adding any
+  /// SDNodes to the DAG.
+  virtual void AnalyzeCallArgs(CallLoweringInfo & /*CLI*/,
+                               CCState & /*CCInfo*/) const {
+    llvm_unreachable("Not Implemented");
+  }
+
   /// Target-specific cleanup for formal ByVal parameters.
   virtual void HandleByVal(CCState *, unsigned &, unsigned) const {}
 
   /// This hook should be implemented to check whether the return values
   /// described by the Outs array can fit into the return registers.  If false
   /// is returned, an sret-demotion is performed.
   virtual bool CanLowerReturn(CallingConv::ID /*CallConv*/,
                               MachineFunction &/*MF*/, bool /*isVarArg*/,

lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp

CanLowerReturn = TLI->CanLowerReturn(Fn->getCallingConv(), *MF,
                                        Fn->isVarArg(),
                                        Outs, Fn->getContext());
 
   // Initialize the mapping of values to registers.  This is only set up for
   // instruction values that are used outside of the block that defines
   // them.
   Function::const_iterator BB = Fn->begin(), EB = Fn->end();
   for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I)
-    if (const AllocaInst *AI = dyn_cast<AllocaInst>(I))
+    if (const AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
+      if (AI->isUsedWithInAlloca())
+        continue;
       if (const ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) {
         Type *Ty = AI->getAllocatedType();
         uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
         unsigned Align =
           std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
                    AI->getAlignment());
 
         TySize *= CUI->getZExtValue();   // Get total allocated size.
         if (TySize == 0) TySize = 1; // Don't create zero-sized stack objects.
 
         StaticAllocaMap[AI] =
           MF->getFrameInfo()->CreateStackObject(TySize, Align, false, AI);
       }
+    }
 
   for (; BB != EB; ++BB)
     for (BasicBlock::const_iterator I = BB->begin(), E = BB->end();
          I != E; ++I) {
       // Mark values used outside their block as exported, by allocating
       // a virtual register for them.
       if (isUsedOutsideOfDefiningBlock(I))
         if (!isa<AllocaInst>(I) ||

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetLowering.h"
 #include <vector>
 
 namespace llvm {
 
 class AddrSpaceCastInst;
 class AliasAnalysis;
 class AllocaInst;
 class BasicBlock;

 class SExtInst;
 class SelectInst;
 class ShuffleVectorInst;
 class SIToFPInst;
 class StoreInst;
 class SwitchInst;
 class DataLayout;
 class TargetLibraryInfo;
-class TargetLowering;
 class TruncInst;
 class UIToFPInst;
 class UnreachableInst;
 class VAArgInst;
 class ZExtInst;
 
 //===----------------------------------------------------------------------===//
 /// SelectionDAGBuilder - This is the common target-independent lowering
public:
   void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB,
                                     MachineBasicBlock *FBB,
                                     MachineBasicBlock *CurBB,
                                     MachineBasicBlock *SwitchBB);
   bool ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases);
   bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB);
   void CopyToExportRegsIfNeeded(const Value *V);
   void ExportFromCurrentBlock(const Value *V);
+
+  /// \brief Demote an unlowerable return type to an implicit sret parameter.
+  /// Updates CLI.Args and CLI.RetTy.
+  ///
+  /// \param [opt,out] DemoteStackSlot - If non-null, a stack object is created
+  ///                   and returned in this outparam.
+  bool MaybeLowerResultAsSRet(TargetLowering::CallLoweringInfo &CLI,
+                              SDValue *DemoteStackSlot);
+
+  /// \brief Fill out CLI.Args with argument information.
+  ///
+  /// \param EmitNodes - If true, register copies for arguments are emitted.
+  void AddArgsToCLI(TargetLowering::CallLoweringInfo &CLI, bool EmitNodes);
+
   void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall,
                    MachineBasicBlock *LandingPad = NULL);
 
   std::pair<SDValue, SDValue> LowerCallOperands(const CallInst &CI,
                                                 unsigned ArgIdx,
                                                 unsigned NumArgs,
                                                 SDValue Callee,
                                                 bool useVoidTy = false);
private:
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
 
   void visitVAStart(const CallInst &I);
   void visitVAArg(const VAArgInst &I);
   void visitVAEnd(const CallInst &I);
   void visitVACopy(const CallInst &I);
   void visitStackmap(const CallInst &I);
   void visitPatchpoint(const CallInst &I);
+  void visitInAlloca(const AllocaInst &I);
 
   void visitUserOp1(const Instruction &I) {
     llvm_unreachable("UserOp1 should not exist at instruction selection time!");
   }
   void visitUserOp2(const Instruction &I) {
     llvm_unreachable("UserOp2 should not exist at instruction selection time!");
   }
 

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

if (isa<TerminatorInst>(&I))
     HandlePHINodesInSuccessorBlocks(I.getParent());
 
   ++SDNodeOrder;
 
   CurInst = &I;
 
   visit(I.getOpcode(), I);
 
-  if (!isa<TerminatorInst>(&I) && !HasTailCall)
+  // TODO: Try to localize inalloca values and remove this special case.
+  if (!isa<TerminatorInst>(&I) && !HasTailCall &&
+      (!isa<AllocaInst>(&I) || !cast<AllocaInst>(&I)->isUsedWithInAlloca()))
     CopyToExportRegsIfNeeded(&I);
 
   CurInst = NULL;
 }
 
 void SelectionDAGBuilder::visitPHI(const PHINode &) {
   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
 }

 }
 
 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   // If this is a fixed sized alloca in the entry block of the function,
   // allocate it statically on the stack.
   if (FuncInfo.StaticAllocaMap.count(&I))
     return;   // getValue will auto-populate this.
 
+  if (I.isUsedWithInAlloca()) {
+    visitInAlloca(I);
+    return;
+  }
+
   Type *Ty = I.getAllocatedType();
   const TargetLowering *TLI = TM.getTargetLowering();
   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
   unsigned Align =
     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
              I.getAlignment());
 
   SDValue AllocSize = getValue(I.getArraySize());
void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   setValue(&I, DSA);
   DAG.setRoot(DSA.getValue(1));
 
   // Inform the Frame Information that we have just allocated a variable-sized
   // object.
   FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject(Align ? Align : 1, &I);
 }
 
+void SelectionDAGBuilder::visitInAlloca(const AllocaInst &AI) {
+  // Find the call site that uses us, and figure out if we are last inalloca
+  // argument to the callsite, which implies that we are the first alloca to
+  // execute.
+  ImmutableCallSite CS = AI.getInAllocaCallSite();
+  assert(CS);
+  for (int I = CS.arg_size() - 1; I >= 0; --I) {
+    const Value *Arg = CS.getArgument(I);
+    if (Arg == &AI)
+      break;  // We were the last inalloca alloca, do the adjustment here.
+    if (CS.paramHasAttr(1 + I, Attribute::InAlloca)) {
+      return;  // We weren't the last alloca.  We'll be lowered elsewhere.
+    }
+  }
+
+  // Analyze the call site to figure out how much stack memory to allocate.
+  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
+  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+  Type *RetTy = FTy->getReturnType();
+  TargetLowering::ArgListTy Args;
+  SmallVector<CCValAssign, 16> ArgLocs;
+  MachineFunction &MF = DAG.getMachineFunction();
+  TargetLowering::CallLoweringInfo CLI(
+      /*Chain=*/SDValue(), RetTy, FTy, /*isTailCall=*/false,
+      /*Callee=*/SDValue(), Args, DAG, SDLoc(), CS);
+  bool AddedSRet = !MaybeLowerResultAsSRet(CLI, 0);
+  AddArgsToCLI(CLI, /*EmitNodes=*/false);
+
+  const TargetLowering *TLI = TM.getTargetLowering();
+  CCState CCInfo(CS.getCallingConv(), FTy->isVarArg(), MF, TM, ArgLocs,
+                 *DAG.getContext());
+  TLI->AnalyzeCallArgs(CLI, CCInfo);
+
+  // Dynamically allocate space for the stack frame.
+  // TODO: Can we support args aligned to greater than the stack alignment?
+  unsigned Align = 0;
+  SDValue AllocSize = DAG.getIntPtrConstant(CCInfo.getNextStackOffset());
+  SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
+  EVT IntPtr = TLI->getPointerTy();
+  SDVTList VTs = DAG.getVTList(IntPtr, MVT::Other);
+  SDValue ArgMem =
+      DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops, 3);
+  DAG.setRoot(ArgMem.getValue(1));
+
+  // We need to tell frame information that we allocated a "variable" sized
+  // object in order to force the use of a base pointer.
+  FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject(Align ? Align : 1,
+                                                         &AI);
+
+  // Compute the address of the inalloca args in the frame and create virtual
+  // registers for them.
+  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
+    unsigned ValNo = ArgLocs[I].getValNo();
+    unsigned ArgNo = CLI.Outs[ValNo].OrigArgIndex - AddedSRet;
+    if (CS.paramHasAttr(1 + ArgNo, Attribute::InAlloca)) {
+      if (!ArgLocs[I].isMemLoc())
+        report_fatal_error("inalloca used for register parameter");
+
+      SDValue AllocaAddr =
+          DAG.getNode(ISD::ADD, getCurSDLoc(), IntPtr, ArgMem,
+                      DAG.getConstant(ArgLocs[I].getLocMemOffset(), IntPtr));
+
+      // Define exported virtual registers for all inalloca values up front.
+      // TODO: Try to localize them so we get better isel.
+      const Value *Alloca = CS.getArgument(ArgNo);
+      assert(isa<AllocaInst>(Alloca) && "inalloca with non-alloca?");
+      setValue(Alloca, AllocaAddr);
+      CopyToExportRegsIfNeeded(Alloca);
+    }
+    assert(I + 1 == E || ArgLocs[I + 1].getValNo() != ArgNo);
+  }
+}
+
 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
   if (I.isAtomic())
     return visitAtomicLoad(I);
 
   const Value *SV = I.getOperand(0);
   SDValue Ptr = getValue(SV);
 
   Type *Ty = I.getType();
SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   case Intrinsic::experimental_patchpoint_void:
   case Intrinsic::experimental_patchpoint_i64: {
     visitPatchpoint(I);
     return 0;
   }
   }
 }
 
-void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
-                                      bool isTailCall,
-                                      MachineBasicBlock *LandingPad) {
+bool SelectionDAGBuilder::MaybeLowerResultAsSRet(
+    TargetLowering::CallLoweringInfo &CLI, SDValue *DemoteStackSlot) {
+  ImmutableCallSite &CS = *CLI.CS;
   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
   Type *RetTy = FTy->getReturnType();
-  MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
-  MCSymbol *BeginLabel = 0;
 
-  TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
+  TargetLowering::ArgListTy &Args = CLI.Args;
   Args.reserve(CS.arg_size());
 
   // Check whether the function can return without sret-demotion.
   SmallVector<ISD::OutputArg, 4> Outs;
   const TargetLowering *TLI = TM.getTargetLowering();
   GetReturnInfo(RetTy, CS.getAttributes(), Outs, *TLI);
 
   bool CanLowerReturn = TLI->CanLowerReturn(CS.getCallingConv(),
                                             DAG.getMachineFunction(),
                                             FTy->isVarArg(), Outs,
                                             FTy->getContext());
 
-  SDValue DemoteStackSlot;
-  int DemoteStackIdx = -100;
+  if (CanLowerReturn)
+    return true;
 
-  if (!CanLowerReturn) {
-    uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(
-                      FTy->getReturnType());
-    unsigned Align  = TLI->getDataLayout()->getPrefTypeAlignment(
-                      FTy->getReturnType());
-    MachineFunction &MF = DAG.getMachineFunction();
-    DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
-    Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType());
+  // Add an implicit sret parameter as the first argument entry.
+  uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(RetTy);
+  unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment(RetTy);
+  Type *StackSlotPtrType = PointerType::getUnqual(RetTy);
+  TargetLowering::ArgListEntry SRetEntry;
+  SRetEntry.Ty = StackSlotPtrType;
+  SRetEntry.isSRet = true;
+  SRetEntry.Alignment = Align;
 
-    DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI->getPointerTy());
-    Entry.Node = DemoteStackSlot;
-    Entry.Ty = StackSlotPtrType;
-    Entry.isSExt = false;
-    Entry.isZExt = false;
-    Entry.isInReg = false;
-    Entry.isSRet = true;
-    Entry.isNest = false;
-    Entry.isByVal = false;
-    Entry.isReturned = false;
-    Entry.Alignment = Align;
-    Args.push_back(Entry);
-    RetTy = Type::getVoidTy(FTy->getContext());
+  if (DemoteStackSlot) {
+    MachineFunction &MF = DAG.getMachineFunction();
+    int DemoteStackIdx =
+        MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
+    *DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI->getPointerTy());
+    SRetEntry.Node = *DemoteStackSlot;
   }
+  Args.push_back(SRetEntry);
+  CLI.RetTy = Type::getVoidTy(FTy->getContext());
+  return false;
+}
+
+void SelectionDAGBuilder::AddArgsToCLI(TargetLowering::CallLoweringInfo &CLI,
+                                       bool EmitNodes) {
+  ImmutableCallSite &CS = *CLI.CS;
 
   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
        i != e; ++i) {
     const Value *V = *i;
-
-    // Skip empty types
     if (V->getType()->isEmptyTy())
-      continue;
-
-    SDValue ArgNode = getValue(V);
-    Entry.Node = ArgNode; Entry.Ty = V->getType();
-
+      continue; // Skip empty types
+    TargetLowering::ArgListEntry Entry;
+    Entry.Ty = V->getType();
     // Skip the first return-type Attribute to get to params.
     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
-    Args.push_back(Entry);
-  }
+
+    if (EmitNodes)
+      Entry.Node = getValue(V);
+    CLI.Args.push_back(Entry);
+  }
+
+  const TargetLowering *TLI = TM.getTargetLowering();
+  TLI->AnalyzeCallOutArgs(CLI, EmitNodes);
 
+void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
+                                      bool isTailCall,
+                                      MachineBasicBlock *LandingPad) {
+  PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
+  FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+  Type *RetTy = FTy->getReturnType();
+  const TargetLowering *TLI = TM.getTargetLowering();
+  MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
+  MCSymbol *BeginLabel = 0;
+
+  // Check if target-independent constraints permit a tail call here.
+  // Target-dependent constraints are checked within TLI->LowerCallTo.
+  if (isTailCall && !isInTailCallPosition(CS, *TLI))
+    isTailCall = false;
+  if (isTailCall && CS.hasInAllocaArgument())
+    isTailCall = false;
+
+  TargetLowering::ArgListTy Args;
+  TargetLowering::CallLoweringInfo CLI(SDValue(), RetTy, FTy, isTailCall,
+                                       Callee, Args, DAG, getCurSDLoc(), CS);
+
+  SDValue DemoteStackSlot;
+  bool CanLowerReturn = MaybeLowerResultAsSRet(CLI, &DemoteStackSlot);
+  AddArgsToCLI(CLI, /*EmitNodes=*/true);
+
   if (LandingPad) {
     // Insert a label before the invoke call to mark the try range.  This can be
     // used to detect deletion of the invoke via the MachineModuleInfo.
     BeginLabel = MMI.getContext().CreateTempSymbol();
 
     // For SjLj, keep track of which landing pads go with which invokes
     // so as to maintain the ordering of pads in the LSDA.
     unsigned CallSiteIndex = MMI.getCurrentCallSite();
     if (CallSiteIndex) {
       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
 
       // Now that the call site is handled, stop tracking it.
       MMI.setCurrentCallSite(0);
     }
 
     // Both PendingLoads and PendingExports must be flushed here;
     // this call might not return.
     (void)getRoot();
     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
   }
 
-  // Check if target-independent constraints permit a tail call here.
-  // Target-dependent constraints are checked within TLI->LowerCallTo.
-  if (isTailCall && !isInTailCallPosition(CS, *TLI))
-    isTailCall = false;
-
-  TargetLowering::
-  CallLoweringInfo CLI(getRoot(), RetTy, FTy, isTailCall, Callee, Args, DAG,
-                       getCurSDLoc(), CS);
+  CLI.Chain = getRoot();
   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
   assert((isTailCall || Result.second.getNode()) &&
          "Non-null chain expected with non-tail call!");
   assert((Result.second.getNode() || !Result.first.getNode()) &&
          "Null value expected with tail call!");
   if (Result.first.getNode()) {
     setValue(CS.getInstruction(), Result.first);
   } else if (!CanLowerReturn && Result.second.getNode()) {
     // The instruction result is the result of loading from the
     // hidden sret parameter.
     SmallVector<EVT, 1> PVTs;
-    Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType());
+    Type *PtrRetTy = PointerType::getUnqual(RetTy);
 
     ComputeValueVTs(*TLI, PtrRetTy, PVTs);
     assert(PVTs.size() == 1 && "Pointers should fit in one register");
     EVT PtrVT = PVTs[0];
 
     SmallVector<EVT, 4> RetTys;
     SmallVector<uint64_t, 4> Offsets;
-    RetTy = FTy->getReturnType();
     ComputeValueVTs(*TLI, RetTy, RetTys, &Offsets);
 
     unsigned NumValues = RetTys.size();
     SmallVector<SDValue, 4> Values(NumValues);
     SmallVector<SDValue, 4> Chains(NumValues);
 
     for (unsigned i = 0; i < NumValues; ++i) {
       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT,
                                 DemoteStackSlot,
                                 DAG.getConstant(Offsets[i], PtrVT));
-      SDValue L = DAG.getLoad(RetTys[i], getCurSDLoc(), Result.second, Add,
-                  MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]),
-                              false, false, false, 1);
+      FrameIndexSDNode *FI = cast<FrameIndexSDNode>(DemoteStackSlot.getNode());
+      SDValue L = DAG.getLoad(
+          RetTys[i], getCurSDLoc(), Result.second, Add,
+          MachinePointerInfo::getFixedStack(FI->getIndex(), Offsets[i]),
       Values[i] = L;
       Chains[i] = L.getValue(1);
     }
 
     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
                                 MVT::Other, &Chains[0], NumValues);
     PendingLoads.push_back(Chain);
 
void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) {
   } else
     DAG.ReplaceAllUsesWith(Call, MN);
   DAG.DeleteNode(Call);
 
   // Inform the Frame Information that we have a patchpoint in this function.
   FuncInfo.MF->getFrameInfo()->setHasPatchPoint();
 }
 
-/// TargetLowering::LowerCallTo - This is the default LowerCallTo
-/// implementation, which just calls LowerCall.
-/// FIXME: When all targets are
-/// migrated to using LowerCall, this hook should be integrated into SDISel.
-std::pair<SDValue, SDValue>
-TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
-  // Handle the incoming return values from the call.
-  CLI.Ins.clear();
-  SmallVector<EVT, 4> RetTys;
-  ComputeValueVTs(*this, CLI.RetTy, RetTys);
-  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
-    EVT VT = RetTys[I];
-    MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
-    unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
-    for (unsigned i = 0; i != NumRegs; ++i) {
-      ISD::InputArg MyFlags;
-      MyFlags.VT = RegisterVT;
-      MyFlags.ArgVT = VT;
-      MyFlags.Used = CLI.IsReturnValueUsed;
-      if (CLI.RetSExt)
-        MyFlags.Flags.setSExt();
-      if (CLI.RetZExt)
-        MyFlags.Flags.setZExt();
-      if (CLI.IsInReg)
-        MyFlags.Flags.setInReg();
-      CLI.Ins.push_back(MyFlags);
-    }
-  }
-
+void TargetLowering::AnalyzeCallOutArgs(CallLoweringInfo &CLI,
+                                        bool EmitNodes) const {
   // Handle all of the outgoing arguments.
   CLI.Outs.clear();
   CLI.OutVals.clear();
   ArgListTy &Args = CLI.Args;
   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
     SmallVector<EVT, 4> ValueVTs;
     ComputeValueVTs(*this, Args[i].Ty, ValueVTs);
     for (unsigned Value = 0, NumValues = ValueVTs.size();
          Value != NumValues; ++Value) {
       EVT VT = ValueVTs[Value];
       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
-      SDValue Op = SDValue(Args[i].Node.getNode(),
-                           Args[i].Node.getResNo() + Value);
       ISD::ArgFlagsTy Flags;
       unsigned OriginalAlignment =
         getDataLayout()->getABITypeAlignment(ArgTy);
 
       if (Args[i].isZExt)
         Flags.setZExt();
       if (Args[i].isSExt)
         Flags.setSExt();
       if (Args[i].isInReg)
         Flags.setInReg();
       if (Args[i].isSRet)
         Flags.setSRet();
-      if (Args[i].isByVal) {
+      if (Args[i].isByVal)
         Flags.setByVal();
+      if (Args[i].isInAlloca) {
+        Flags.setByVal();  // TODO: Remove when isInAlloca() works.
+        Flags.setInAlloca();
+      }
+      if (Args[i].isByVal || Args[i].isInAlloca) {
         PointerType *Ty = cast<PointerType>(Args[i].Ty);
         Type *ElementTy = Ty->getElementType();
         Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy));
         // For ByVal, alignment should come from FE.  BE will guess if this
         // info is not there but there are cases it cannot get right.
         unsigned FrameAlign;
         if (Args[i].Alignment)
           FrameAlign = Args[i].Alignment;
         else
           FrameAlign = getByValTypeAlignment(ElementTy);
         Flags.setByValAlign(FrameAlign);
       }
       if (Args[i].isNest)
         Flags.setNest();
       Flags.setOrigAlign(OriginalAlignment);
 
       MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT);
       unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT);
-      SmallVector<SDValue, 4> Parts(NumParts);
       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
 
       if (Args[i].isSExt)
         ExtendKind = ISD::SIGN_EXTEND;
       else if (Args[i].isZExt)
         ExtendKind = ISD::ZERO_EXTEND;
 
       // Conservatively only handle 'returned' on non-vectors for now
-      if (Args[i].isReturned && !Op.getValueType().isVector()) {
-        assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
-               "unexpected use of 'returned'");
+      if (Args[i].isReturned && !VT.isVector()) {
+        assert(CLI.RetTy == Args[i].Ty && "unexpected use of 'returned'");
         // Before passing 'returned' to the target lowering code, ensure that
         // either the register MVT and the actual EVT are the same size or that
         // the return value and argument are extended in the same way; in these
         // cases it's safe to pass the argument register value unchanged as the
         // return register value (although it's at the target's option whether
         // to do so)
         // TODO: allow code generation to take advantage of partially preserved
         // registers rather than clobbering the entire register when the
         // parameter extension method is not compatible with the return
         // extension method
         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
             (ExtendKind != ISD::ANY_EXTEND &&
              CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt))
         Flags.setReturned();
       }
 
+      SmallVector<SDValue, 4> Parts;
+      if (EmitNodes) {
+        SDValue Op = SDValue(Args[i].Node.getNode(),
+                             Args[i].Node.getResNo() + Value);
+        Parts.resize(NumParts);
         getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts,
                        PartVT, CLI.CS ? CLI.CS->getInstruction() : 0, ExtendKind);
+      }
 
       for (unsigned j = 0; j != NumParts; ++j) {
         // if it isn't first piece, alignment must be 1
-        ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
-                               i < CLI.NumFixedArgs,
-                               i, j*Parts[j].getValueType().getStoreSize());
+        ISD::OutputArg MyFlags(Flags, PartVT, VT, i < CLI.NumFixedArgs, i,
+                               j * PartVT.getStoreSize());
         if (NumParts > 1 && j == 0)
           MyFlags.Flags.setSplit();
         else if (j != 0)
           MyFlags.Flags.setOrigAlign(1);
 
         CLI.Outs.push_back(MyFlags);
+        if (EmitNodes) {
+          assert(Parts[j].getValueType() == PartVT);
           CLI.OutVals.push_back(Parts[j]);
         }
       }
     }
+  }
+}
+
+/// TargetLowering::LowerCallTo - This is the default LowerCallTo
+/// implementation, which just calls LowerCall.
+/// FIXME: When all targets are
+/// migrated to using LowerCall, this hook should be integrated into SDISel.
+std::pair<SDValue, SDValue>
+TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
+  // Handle the incoming return values from the call.
+  CLI.Ins.clear();
+  SmallVector<EVT, 4> RetTys;
+  ComputeValueVTs(*this, CLI.RetTy, RetTys);
+  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
+    EVT VT = RetTys[I];
+    MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
+    unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
+    for (unsigned i = 0; i != NumRegs; ++i) {
+      ISD::InputArg MyFlags;
+      MyFlags.VT = RegisterVT;
+      MyFlags.ArgVT = VT;
+      MyFlags.Used = CLI.IsReturnValueUsed;
+      if (CLI.RetSExt)
+        MyFlags.Flags.setSExt();
+      if (CLI.RetZExt)
+        MyFlags.Flags.setZExt();
+      if (CLI.IsInReg)
+        MyFlags.Flags.setInReg();
+      CLI.Ins.push_back(MyFlags);
+    }
+  }
+
+  AnalyzeCallOutArgs(CLI, true);
 
   SmallVector<SDValue, 4> InVals;
   CLI.Chain = LowerCall(CLI, InVals);
 
   // Verify that the target's LowerCall behaved as expected.
   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
          "LowerCall didn't return a valid chain!");
   assert((!CLI.IsTailCall || InVals.empty()) &&
for (unsigned Value = 0, NumValues = ValueVTs.size();
       if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt))
         Flags.setZExt();
       if (F.getAttributes().hasAttribute(Idx, Attribute::SExt))
         Flags.setSExt();
       if (F.getAttributes().hasAttribute(Idx, Attribute::InReg))
         Flags.setInReg();
       if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet))
         Flags.setSRet();
-      if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) {
+      if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal))
         Flags.setByVal();
+      if (F.getAttributes().hasAttribute(Idx, Attribute::InAlloca)) {
+        Flags.setByVal();  // TODO: Remove when isInAlloca() works.
+        Flags.setInAlloca();
+      }
+      if (Flags.isByVal() || Flags.isInAlloca()) {
         PointerType *Ty = cast<PointerType>(I->getType());
         Type *ElementTy = Ty->getElementType();
         Flags.setByValSize(TD->getTypeAllocSize(ElementTy));
         // For ByVal, alignment should be passed from FE.  BE will guess if
         // this info is not there but there are cases it cannot get right.
         unsigned FrameAlign;
         if (F.getParamAlignment(Idx))
           FrameAlign = F.getParamAlignment(Idx);

lib/CodeGen/SelectionDAG/TargetLowering.cpp

 void TargetLowering::ArgListEntry::setAttributes(ImmutableCallSite *CS,
                                                  unsigned AttrIdx) {
   isSExt     = CS->paramHasAttr(AttrIdx, Attribute::SExt);
   isZExt     = CS->paramHasAttr(AttrIdx, Attribute::ZExt);
   isInReg    = CS->paramHasAttr(AttrIdx, Attribute::InReg);
   isSRet     = CS->paramHasAttr(AttrIdx, Attribute::StructRet);
   isNest     = CS->paramHasAttr(AttrIdx, Attribute::Nest);
   isByVal    = CS->paramHasAttr(AttrIdx, Attribute::ByVal);
+  isInAlloca = CS->paramHasAttr(AttrIdx, Attribute::InAlloca);
   isReturned = CS->paramHasAttr(AttrIdx, Attribute::Returned);
   Alignment  = CS->getParamAlignment(AttrIdx);
 }
 
 /// Generate a libcall taking the given operands as arguments and returning a
 /// result of type RetVT.
 std::pair<SDValue, SDValue>
 TargetLowering::makeLibCall(SelectionDAG &DAG,

lib/Target/Mangler.cpp

 /// take.
 static void AddFastCallStdCallSuffix(SmallVectorImpl<char> &OutName,
                                      const Function *F, const DataLayout &TD) {
   // Calculate arguments size total.
   unsigned ArgWords = 0;
   for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end();
        AI != AE; ++AI) {
     Type *Ty = AI->getType();
-    // 'Dereference' type in case of byval parameter attribute
-    if (AI->hasByValAttr())
+    // 'Dereference' type in case of byval or inalloca parameter attribute
+    if (AI->hasByValOrInAllocaAttr())
       Ty = cast<PointerType>(Ty)->getElementType();
     // Size should be aligned to DWORD boundary
     ArgWords += ((TD.getTypeAllocSize(Ty) + 3)/4)*4;
   }
   
   raw_svector_ostream(OutName) << '@' << ArgWords;
 }
 

lib/Target/X86/X86FastISel.cpp

bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
   bool isVarArg = FTy->isVarArg();
 
   // Don't know how to handle Win64 varargs yet.  Nothing special needed for
   // x86-32.  Special handling for x86-64 is implemented.
   if (isVarArg && isWin64)
     return false;
 
+  // Don't know about inalloca yet.
+  if (CS.hasInAllocaArgument())
+    return false;
+
   // Fast-isel doesn't know about callee-pop yet.
   if (X86::isCalleePop(CC, Subtarget->is64Bit(), isVarArg,
                        TM.Options.GuaranteedTailCallOpt))
     return false;
 
   // Check whether the function can return without sret-demotion.
   SmallVector<ISD::OutputArg, 4> Outs;
   GetReturnInfo(I->getType(), CS.getAttributes(), Outs, TLI);

lib/Target/X86/X86FrameLowering.cpp

eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
   const X86InstrInfo &TII = *TM.getInstrInfo();
   const X86RegisterInfo &RegInfo = *TM.getRegisterInfo();
   unsigned StackPtr = RegInfo.getStackRegister();
   bool reseveCallFrame = hasReservedCallFrame(MF);
   int Opcode = I->getOpcode();
   bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
   bool IsLP64 = STI.isTarget64BitLP64();
   DebugLoc DL = I->getDebugLoc();
-  uint64_t Amount = !reseveCallFrame ? I->getOperand(0).getImm() : 0;
-  uint64_t CalleeAmt = isDestroy ? I->getOperand(1).getImm() : 0;
+  int64_t Amount = !reseveCallFrame ? I->getOperand(0).getImm() : 0;
+  int64_t CalleeAmt = isDestroy ? I->getOperand(1).getImm() : 0;
   I = MBB.erase(I);
 
   if (!reseveCallFrame) {
     // If the stack pointer can be changed after prologue, turn the
     // adjcallstackup instruction into a 'sub ESP, <amt>' and the
     // adjcallstackdown instruction into 'add ESP, <amt>'
     // TODO: consider using push / pop instead of sub + store / add
-    if (Amount == 0)
+    if (Amount == 0 && CalleeAmt == 0)
       return;
 
     // We need to keep the stack aligned properly.  To do this, we round the
     // amount of space needed for the outgoing arguments up to the next
     // alignment boundary.
     unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
     Amount = (Amount + StackAlign - 1) / StackAlign * StackAlign;
 
     MachineInstr *New = 0;
     if (Opcode == TII.getCallFrameSetupOpcode()) {
       New = BuildMI(MF, DL, TII.get(getSUBriOpcode(IsLP64, Amount)),
                     StackPtr)
         .addReg(StackPtr)
         .addImm(Amount);
     } else {
       assert(Opcode == TII.getCallFrameDestroyOpcode());
 
       // Factor out the amount the callee already popped.
       Amount -= CalleeAmt;
 
-      if (Amount) {
+      if (Amount > 0) {
         unsigned Opc = getADDriOpcode(IsLP64, Amount);
         New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
           .addReg(StackPtr).addImm(Amount);
+      } else if (Amount < 0) {
+        Amount = -Amount;
+        unsigned Opc = getSUBriOpcode(IsLP64, Amount);
+        New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
+          .addReg(StackPtr).addImm(Amount);
       }
     }
 
     if (New) {
       // The EFLAGS implicit def is dead.
       New->getOperand(3).setIsDead();
 
       // Replace the pseudo instruction with a new instruction.

lib/Target/X86/X86ISelLowering.h

virtual SDValue
                            CallingConv::ID CallConv, bool isVarArg,
                            const SmallVectorImpl<ISD::InputArg> &Ins,
                            SDLoc dl, SelectionDAG &DAG,
                            SmallVectorImpl<SDValue> &InVals) const;
     virtual SDValue
       LowerCall(CallLoweringInfo &CLI,
                 SmallVectorImpl<SDValue> &InVals) const;
 
+    virtual void AnalyzeCallArgs(CallLoweringInfo &CLI, CCState &CCInfo) const;
+
     virtual SDValue
       LowerReturn(SDValue Chain,
                   CallingConv::ID CallConv, bool isVarArg,
                   const SmallVectorImpl<ISD::OutputArg> &Outs,
                   const SmallVectorImpl<SDValue> &OutVals,
                   SDLoc dl, SelectionDAG &DAG) const;
 
     virtual bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const;

lib/Target/X86/X86ISelLowering.cpp

MF.getFrameInfo()->CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
                                          false);
   SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
   Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
                        MachinePointerInfo::getFixedStack(NewReturnAddrFI),
                        false, false, 0);
   return Chain;
 }
 
+void X86TargetLowering::AnalyzeCallArgs(TargetLowering::CallLoweringInfo &CLI,
+                                        CCState &CCInfo) const {
+  SelectionDAG &DAG                     = CLI.DAG;
+  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
+  CallingConv::ID CallConv              = CLI.CallConv;
+  bool isVarArg                         = CLI.IsVarArg;
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  // Allocate shadow area for Win64
+  if (Subtarget->isCallingConvWin64(CallConv))
+    CCInfo.AllocateStack(32, 8);
+
+  CCInfo.AnalyzeCallOperands(Outs, CC_X86);
+}
+
 SDValue
 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG                     = CLI.DAG;
   SDLoc &dl                             = CLI.DL;
   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
   SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
   SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
          "Var args not supported with calling convention fastcc, ghc or hipe");
 
   // Analyze operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(),
                  ArgLocs, *DAG.getContext());
-
-  // Allocate shadow area for Win64
-  if (IsWin64)
-    CCInfo.AllocateStack(32, 8);
-
-  CCInfo.AnalyzeCallOperands(Outs, CC_X86);
+  AnalyzeCallArgs(CLI, CCInfo);
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();
   if (IsSibcall)
     // This is a sibcall. The memory operands are available in caller's
     // own caller's stack.
     NumBytes = 0;
   else if (getTargetMachine().Options.GuaranteedTailCallOpt &&
if (isTailCall && !IsSibcall) {
     FPDiff = NumBytesCallerPushed - NumBytes;
 
     // Set the delta of movement of the returnaddr stackslot.
     // But only set if delta is greater than previous delta.
     if (FPDiff < X86Info->getTCReturnAddrDelta())
       X86Info->setTCReturnAddrDelta(FPDiff);
   }
 
+  unsigned NumBytesToPush = NumBytes;
+
+  // If there were any inalloca arguments, we should've already adjusted the
+  // stack at the first inalloca alloca.  Note that the callee may clear the
+  // arguments for us, and we will have to undo it.  See NumBytesForCalleeToPop
+  // below.
+  if (CLI.CS && CLI.CS->hasInAllocaArgument())
+    NumBytesToPush = 0;
+
   if (!IsSibcall)
-    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true),
-                                 dl);
+    Chain = DAG.getCALLSEQ_START(
+        Chain, DAG.getIntPtrConstant(NumBytesToPush, true), dl);
 
   SDValue RetAddrFrIdx;
   // Load return address for tail calls.
   if (isTailCall && FPDiff)
     Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
                                     Is64Bit, FPDiff, dl);
 
   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
   SDValue StackPtr;
 
   // Walk the register/memloc assignments, inserting copies/loads.  In the case
   // of tail call optimization arguments are handle later.
   const X86RegisterInfo *RegInfo =
     static_cast<const X86RegisterInfo*>(getTargetMachine().getRegisterInfo());
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+    // Skip inalloca arguments, they have already been written.
+    ISD::ArgFlagsTy Flags = Outs[i].Flags;
+    if (Flags.isInAlloca())
+      continue;
+
     CCValAssign &VA = ArgLocs[i];
     EVT RegVT = VA.getLocVT();
     SDValue Arg = OutVals[i];
-    ISD::ArgFlagsTy Flags = Outs[i].Flags;
     bool isByVal = Flags.isByVal();
 
     // Promote the value if needed.
     switch (VA.getLocInfo()) {
     default: llvm_unreachable("Unknown loc info!");
     case CCValAssign::Full: break;
     case CCValAssign::SExt:
       Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
                                          OpFlags);
   }
 
   // Returns a chain & a flag for retval copy to use.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   SmallVector<SDValue, 8> Ops;
 
   if (!IsSibcall && isTailCall) {
-    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
+    Chain = DAG.getCALLSEQ_END(Chain,
+                               DAG.getIntPtrConstant(NumBytesToPush, true),
                                DAG.getIntPtrConstant(0, true), InFlag, dl);
     InFlag = Chain.getValue(1);
   }
 
   Ops.push_back(Chain);
   Ops.push_back(Callee);
 
   if (isTailCall)
     Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
if (isTailCall) {
     // function making a tail call to a function returning int.
     return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, &Ops[0], Ops.size());
   }
 
   Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size());
   InFlag = Chain.getValue(1);
 
   // Create the CALLSEQ_END node.
-  unsigned NumBytesForCalleeToPush;
+  unsigned NumBytesForCalleeToPop;
   if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
                        getTargetMachine().Options.GuaranteedTailCallOpt))
-    NumBytesForCalleeToPush = NumBytes;    // Callee pops everything
+    NumBytesForCalleeToPop = NumBytes;    // Callee pops everything
   else if (!Is64Bit && !IsTailCallConvention(CallConv) &&
            !Subtarget->getTargetTriple().isOSMSVCRT() &&
            SR == StackStructReturn)
     // If this is a call to a struct-return function, the callee
     // pops the hidden struct pointer, so we have to push it back.
     // This is common for Darwin/X86, Linux & Mingw32 targets.
     // For MSVC Win32 targets, the caller pops the hidden struct pointer.
-    NumBytesForCalleeToPush = 4;
+    NumBytesForCalleeToPop = 4;
   else
-    NumBytesForCalleeToPush = 0;  // Callee pops nothing.
+    NumBytesForCalleeToPop = 0;  // Callee pops nothing.
 
   // Returns a flag for retval copy to use.
   if (!IsSibcall) {
     Chain = DAG.getCALLSEQ_END(Chain,
-                               DAG.getIntPtrConstant(NumBytes, true),
-                               DAG.getIntPtrConstant(NumBytesForCalleeToPush,
+                               DAG.getIntPtrConstant(NumBytesToPush, true),
+                               DAG.getIntPtrConstant(NumBytesForCalleeToPop,
                                                      true),
                                InFlag, dl);
     InFlag = Chain.getValue(1);
   }
 
   // Handle result values, copying them out of physregs into vregs that we
   // return.
   return LowerCallResult(Chain, InFlag, CallConv, isVarArg,

test/CodeGen/X86/inalloca-ctor.ll

+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
+
+%Foo = type { i32, i32 }
+
+declare void @f(%Foo* inalloca %a, i32 %b, %Foo* inalloca %c)
+
+declare void @Foo_ctor(%Foo* %this)
+
+define void @g() {
+entry:
+  %c = alloca %Foo
+; CHECK: movl    $20, %eax
+; CHECK: calll   __chkstk
+; CHECK: movl %esp,
+  call void @Foo_ctor(%Foo* %c)
+; CHECK: leal 12(%{{.*}}),
+; CHECK: subl $4, %esp
+; CHECK: calll _Foo_ctor
+; CHECK: addl $4, %esp
+  %a = alloca %Foo
+  call void @Foo_ctor(%Foo* %a)
+; CHECK: subl $4, %esp
+; CHECK: calll _Foo_ctor
+; CHECK: addl $4, %esp
+  call void @f(%Foo* inalloca %a, i32 3, %Foo* inalloca %c)
+; CHECK: movl    $3, 8(%esp)
+; CHECK: calll   _f
+  ret void
+}

test/CodeGen/X86/inalloca-stdcall.ll

+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
+
+; Make sure we keep the argument stack space live after a callee cleanup call.
+; Otherwise we have pointers past the top of the stack in the IR.  The plan is
+; to write an MI pass to clean up these extraneous stack adjustments later.
+
+declare x86_stdcallcc void @f(i32* inalloca %a, i32* inalloca %b)
+declare x86_stdcallcc void @i(i32 %a)
+
+define void @g() {
+  %b = alloca i32
+  %a = alloca i32
+  store i32 1, i32* %a
+  store i32 2, i32* %b
+; CHECK: movl    $1, (%esp)
+; CHECK: movl    $2, 4(%esp)
+  call x86_stdcallcc void @f(i32* inalloca %a, i32* inalloca %b)
+; CHECK: calll   _f@8
+; CHECK: subl $8, %esp
+; CHECK: subl $4, %esp
+; CHECK: calll   _i@4
+  call x86_stdcallcc void @i(i32 0)
+  ret void
+}

test/CodeGen/X86/inalloca.ll

+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
+
+%Foo = type { i32, i32 }
+
+declare void @f(%Foo* inalloca %a, i32 %b, %Foo* inalloca %c)
+
+define void @g() {
+entry:
+  %c = alloca %Foo
+  %a = alloca %Foo
+; CHECK: movl    $20, %eax
+; CHECK: calll   __chkstk
+  %f1 = getelementptr %Foo* %a, i32 0, i32 0
+  %f2 = getelementptr %Foo* %a, i32 0, i32 1
+  store i32 1, i32* %f1
+  store i32 2, i32* %f2
+; CHECK: movl    $1, (%esp)
+; CHECK: movl    $2, 4(%esp)
+  %f3 = getelementptr %Foo* %c, i32 0, i32 0
+  %f4 = getelementptr %Foo* %c, i32 0, i32 1
+  store i32 4, i32* %f3
+  store i32 5, i32* %f4
+; CHECK: movl    $4, 12(%esp)
+; CHECK: movl    $5, 16(%esp)
+  call void @f(%Foo* inalloca %a, i32 3, %Foo* inalloca %c)
+; CHECK: movl    $3, 8(%esp)
+; CHECK: calll   _f
+  ret void
+}
+
+declare void @i64_with_inalloca(i64 %a, %Foo* inalloca %b)
+
+define void @foo() {
+  %b = alloca %Foo
+; CHECK:        movl    $16, %eax
+; CHECK-NEXT:   calll   __chkstk
+  %f1 = getelementptr %Foo* %b, i32 0, i32 0
+  store i32 42, i32* %f1
+; CHECK:        movl    $42, 8(%esp)
+; CHECK-NEXT:   movl    $0, 4(%esp)
+; CHECK-NEXT:   movl    $0, (%esp)
+; CHECK-NEXT:   calll   _i64_with_inalloca
+  call void @i64_with_inalloca(i64 0, %Foo* inalloca %b)
+  ret void
+}
+
+%sret_type = type { i32, i32, i32, i32 }
+
+declare %sret_type @sret_and_i64_with_inalloca(i64 %a, %Foo* inalloca %b)
+
+define void @bar() {
+  %r = alloca %sret_type
+  %b = alloca %Foo
+  %f1 = getelementptr %Foo* %b, i32 0, i32 0
+  store i32 42, i32* %f1
+; CHECK:        movl    $42, 12(%esp)
+
+; The demoted sret param comes first, then the i64.
+; CHECK:        movl    %{{.*}}, (%esp)
+; CHECK:        movl    $0,  8(%esp)
+; CHECK:        movl    $13, 4(%esp)
+; CHECK-NEXT:   calll   _sret_and_i64_with_inalloca
+  %t = call %sret_type @sret_and_i64_with_inalloca(i64 13, %Foo* inalloca %b)
+  store %sret_type %t, %sret_type* %r
+  ret void
+}

test/CodeGen/X86/inalloca2.ll

+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
+
+%Foo = type { i32, i32 }
+
+declare void @f(%Foo* inalloca %a, i32 %b, %Foo* inalloca %c)
+
+define void @g(i1 zeroext %cond) {
+  %c = alloca %Foo
+; CHECK: movl    $20, %eax
+; CHECK: calll   __chkstk
+  %f4 = getelementptr %Foo* %c, i32 0, i32 0
+  %f5 = getelementptr %Foo* %c, i32 0, i32 1
+  store i32 4, i32* %f4
+  store i32 5, i32* %f5
+; CHECK: movl    $4, 12(%esp)
+; CHECK: movl    $5, 16(%esp)
+  br i1 %cond, label %bb2, label %exit
+
+bb2:
+  %a = alloca %Foo
+  %f1 = getelementptr %Foo* %a, i32 0, i32 0
+  %f2 = getelementptr %Foo* %a, i32 0, i32 1
+  store i32 1, i32* %f1
+  store i32 2, i32* %f2
+; CHECK: movl    $1, (%esp)
+; CHECK: movl    $2, 4(%esp)
+  br i1 %cond, label %bb3, label %exit
+
+bb3:
+  call void @f(%Foo* inalloca %a, i32 3, %Foo* inalloca %c)
+; CHECK: movl    $3, 8(%esp)
+; CHECK: calll   _f
+  br label %exit
+
+exit:
+  ret void
+}