ARM: support mandatory tail calls

[t.p.northover]

Summary

This is obviously mostly for swifttailcc, but it's trivial to being tailcc and fastcc along for the ride so I did those too.

The code is pretty closely modelled on how AArch64 handles this because most RISC backends will need approximately the same behaviour, so there are a few main components:

• Track how much stack space the callee is expected to restore on return (and do it).
• In the tail call case arguments might not simply be stored to sp + N, but to an object at a fixed offset from the stack on function entry. We use a fixed frame-index for this, which naturally reserves the space so callee-saved registers won't be placed there.
• The tail call itself needs to track how much stack space it actually needs (because different tail calls vary this).

[t.p.northover]

Bother, I subscribed llvm-commits-test instead of llvm-commits. I'll make a new one.

llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp

ARMBaseRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
 
   const Function &F = MF->getFunction();
   if (F.getCallingConv() == CallingConv::GHC) {
     // GHC set of callee saved regs is empty as all those regs are
     // used for passing STG regs around
     return CSR_NoRegs_SaveList;
   } else if (F.getCallingConv() == CallingConv::CFGuard_Check) {
     return CSR_Win_AAPCS_CFGuard_Check_SaveList;
+  } else if (F.getCallingConv() == CallingConv::SwiftTail) {
+    return STI.isTargetDarwin()
+               ? CSR_iOS_SwiftTail_SaveList
+               : (UseSplitPush ? CSR_AAPCS_SplitPush_SwiftTail_SaveList
+                               : CSR_AAPCS_SwiftTail_SaveList);
   } else if (F.hasFnAttribute("interrupt")) {
     if (STI.isMClass()) {
       // M-class CPUs have hardware which saves the registers needed to allow a
       // function conforming to the AAPCS to function as a handler.
       return UseSplitPush ? CSR_AAPCS_SplitPush_SaveList : CSR_AAPCS_SaveList;
     } else if (F.getFnAttribute("interrupt").getValueAsString() == "FIQ") {
       // Fast interrupt mode gives the handler a private copy of R8-R14, so less
       // need to be saved to restore user-mode state.

 ARMBaseRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
                                           CallingConv::ID CC) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
   if (CC == CallingConv::GHC)
     // This is academic because all GHC calls are (supposed to be) tail calls
     return CSR_NoRegs_RegMask;
   if (CC == CallingConv::CFGuard_Check)
     return CSR_Win_AAPCS_CFGuard_Check_RegMask;
+  if (CC == CallingConv::SwiftTail) {
+    return STI.isTargetDarwin() ? CSR_iOS_SwiftTail_RegMask
+                                : CSR_AAPCS_SwiftTail_RegMask;
+  }
   if (STI.getTargetLowering()->supportSwiftError() &&
       MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
     return STI.isTargetDarwin() ? CSR_iOS_SwiftError_RegMask
                                 : CSR_AAPCS_SwiftError_RegMask;
 
   if (STI.isTargetDarwin() && CC == CallingConv::CXX_FAST_TLS)
     return CSR_iOS_CXX_TLS_RegMask;
   return STI.isTargetDarwin() ? CSR_iOS_RegMask : CSR_AAPCS_RegMask;
bool ARMBaseRegisterInfo::shouldRewriteCopySrc(const TargetRegisterClass *DefRC,
   if (DefRC == &ARM::SPRRegClass && DefSubReg == 0 &&
       SrcRC == &ARM::DPRRegClass &&
       (SrcSubReg == ARM::ssub_0 || SrcSubReg == ARM::ssub_1))
     return false;
 
   return TargetRegisterInfo::shouldRewriteCopySrc(DefRC, DefSubReg,
                                                   SrcRC, SrcSubReg);
 }
 No newline at end of file

llvm/lib/Target/ARM/ARMCallingConv.td

 // The Windows Control Flow Guard Check function preserves the same registers as
 // AAPCS, and also preserves all floating point registers.
 def CSR_Win_AAPCS_CFGuard_Check : CalleeSavedRegs<(add LR, R11, R10, R9, R8, R7,
                                      R6, R5, R4, (sequence "D%u", 15, 0))>;
 
 // R8 is used to pass swifterror, remove it from CSR.
 def CSR_AAPCS_SwiftError : CalleeSavedRegs<(sub CSR_AAPCS, R8)>;
 
+// R10 is used to pass swiftself, remove it from CSR.
+def CSR_AAPCS_SwiftTail : CalleeSavedRegs<(sub CSR_AAPCS, R10)>;
+
 // The order of callee-saved registers needs to match the order we actually push
 // them in FrameLowering, because this order is what's used by
 // PrologEpilogInserter to allocate frame index slots. So when R7 is the frame
 // pointer, we use this AAPCS alternative.
 def CSR_AAPCS_SplitPush : CalleeSavedRegs<(add LR, R7, R6, R5, R4,
                                                R11, R10, R9, R8,
                                                (sequence "D%u", 15, 8))>;
 
 // R8 is used to pass swifterror, remove it from CSR.
 def CSR_AAPCS_SplitPush_SwiftError : CalleeSavedRegs<(sub CSR_AAPCS_SplitPush,
                                                       R8)>;
 
+// R10 is used to pass swifterror, remove it from CSR.
+def CSR_AAPCS_SplitPush_SwiftTail : CalleeSavedRegs<(sub CSR_AAPCS_SplitPush,
+                                                     R10)>;
+
 // Constructors and destructors return 'this' in the ARM C++ ABI; since 'this'
 // and the pointer return value are both passed in R0 in these cases, this can
 // be partially modelled by treating R0 as a callee-saved register
 // Only the resulting RegMask is used; the SaveList is ignored
 def CSR_AAPCS_ThisReturn : CalleeSavedRegs<(add LR, R11, R10, R9, R8, R7, R6,
                                             R5, R4, (sequence "D%u", 15, 8),
                                             R0)>;
 
 // iOS ABI deviates from ARM standard ABI. R9 is not a callee-saved register.
 // Also save R7-R4 first to match the stack frame fixed spill areas.
 def CSR_iOS : CalleeSavedRegs<(add LR, R7, R6, R5, R4, (sub CSR_AAPCS, R9))>;
 
 // R8 is used to pass swifterror, remove it from CSR.
 def CSR_iOS_SwiftError : CalleeSavedRegs<(sub CSR_iOS, R8)>;
 
+// R10 is used to pass swiftself, remove it from CSR.
+def CSR_iOS_SwiftTail : CalleeSavedRegs<(sub CSR_iOS, R10)>;
+
 def CSR_iOS_ThisReturn : CalleeSavedRegs<(add LR, R7, R6, R5, R4,
                                          (sub CSR_AAPCS_ThisReturn, R9))>;
 
 def CSR_iOS_TLSCall
     : CalleeSavedRegs<(add LR, SP, (sub(sequence "R%u", 12, 1), R9, R12),
                       (sequence "D%u", 31, 0))>;
 
 // C++ TLS access function saves all registers except SP. Try to match

llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

case ARM::TCRETURNri: {
           STI->isThumb() ? ARM::tTAILJMPr
                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
         BuildMI(MBB, MBBI, dl,
                 TII.get(Opcode))
             .addReg(JumpTarget.getReg(), RegState::Kill);
       }
 
       auto NewMI = std::prev(MBBI);
-      for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
+      for (unsigned i = 2, e = MBBI->getNumOperands(); i != e; ++i)
         NewMI->addOperand(MBBI->getOperand(i));
 
 
       // Update call site info and delete the pseudo instruction TCRETURN.
       if (MI.isCandidateForCallSiteEntry())
         MI.getMF()->moveCallSiteInfo(&MI, &*NewMI);
       MBB.erase(MBBI);
 

llvm/lib/Target/ARM/ARMFastISel.cpp

if (Subtarget->isAAPCS_ABI()) {
         return (Return ? RetCC_ARM_AAPCS_VFP: CC_ARM_AAPCS_VFP);
       else
         return (Return ? RetCC_ARM_AAPCS: CC_ARM_AAPCS);
     } else {
       return (Return ? RetCC_ARM_APCS: CC_ARM_APCS);
     }
   case CallingConv::ARM_AAPCS_VFP:
   case CallingConv::Swift:
+  case CallingConv::SwiftTail:
     if (!isVarArg)
       return (Return ? RetCC_ARM_AAPCS_VFP: CC_ARM_AAPCS_VFP);
     // Fall through to soft float variant, variadic functions don't
     // use hard floating point ABI.
     LLVM_FALLTHROUGH;
   case CallingConv::ARM_AAPCS:
     return (Return ? RetCC_ARM_AAPCS: CC_ARM_AAPCS);
   case CallingConv::ARM_APCS:
bool ARMFastISel::fastLowerArguments() {
   default:
     return false;
   case CallingConv::Fast:
   case CallingConv::C:
   case CallingConv::ARM_AAPCS_VFP:
   case CallingConv::ARM_AAPCS:
   case CallingConv::ARM_APCS:
   case CallingConv::Swift:
+  case CallingConv::SwiftTail:
     break;
   }
 
   // Only handle simple cases. i.e. Up to 4 i8/i16/i32 scalar arguments
   // which are passed in r0 - r3.
   for (const Argument &Arg : F->args()) {
     if (Arg.getArgNo() >= 4)
       return false;

llvm/lib/Target/ARM/ARMFrameLowering.cpp

 /// call frame pseudos can be simplified.  Unlike most targets, having a FP
 /// is not sufficient here since we still may reference some objects via SP
 /// even when FP is available in Thumb2 mode.
 bool
 ARMFrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
   return hasReservedCallFrame(MF) || MF.getFrameInfo().hasVarSizedObjects();
 }
 
+// Returns how much of the incoming argument stack area we should clean up in an
+// epilogue. For the C calling convention this will be 0, for guaranteed tail
+// call conventions it can be positive (a normal return or a tail call to a
+// function that uses less stack space for arguments) or negative (for a tail
+// call to a function that needs more stack space than us for arguments).
+static int getArgumentStackToRestore(MachineFunction &MF,
+                                     MachineBasicBlock &MBB) {
+  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
+  bool IsTailCallReturn = false;
+  if (MBB.end() != MBBI) {
+    unsigned RetOpcode = MBBI->getOpcode();
+    IsTailCallReturn = RetOpcode == ARM::TCRETURNdi ||
+                       RetOpcode == ARM::TCRETURNri;
+  }
+  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+
+  int ArgumentPopSize = 0;
+  if (IsTailCallReturn) {
+    MachineOperand &StackAdjust = MBBI->getOperand(1);
+
+    // For a tail-call in a callee-pops-arguments environment, some or all of
+    // the stack may actually be in use for the call's arguments, this is
+    // calculated during LowerCall and consumed here...
+    ArgumentPopSize = StackAdjust.getImm();
+  } else {
+    // ... otherwise the amount to pop is *all* of the argument space,
+    // conveniently stored in the MachineFunctionInfo by
+    // LowerFormalArguments. This will, of course, be zero for the C calling
+    // convention.
+    ArgumentPopSize = AFI->getArgumentStackToRestore();
+  }
+
+  return ArgumentPopSize;
+}
+
 static void emitRegPlusImmediate(
     bool isARM, MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     const DebugLoc &dl, const ARMBaseInstrInfo &TII, unsigned DestReg,
     unsigned SrcReg, int NumBytes, unsigned MIFlags = MachineInstr::NoFlags,
     ARMCC::CondCodes Pred = ARMCC::AL, unsigned PredReg = 0) {
   if (isARM)
     emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
                             Pred, PredReg, TII, MIFlags);
void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
   const ARMBaseInstrInfo &TII =
       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
   assert(!AFI->isThumb1OnlyFunction() &&
          "This emitEpilogue does not support Thumb1!");
   bool isARM = !AFI->isThumbFunction();
 
-  unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
+  // Amount of stack space we reserved next to incoming args for either
+  // varargs registers or stack arguments in tail calls made by this function.
+  unsigned ReservedArgStack = AFI->getArgRegsSaveSize();
+
+  // How much of the stack used by incoming arguments this function is expected
+  // to restore in this particular epilogue.
+  int IncomingArgStackToRestore = getArgumentStackToRestore(MF, MBB);
   int NumBytes = (int)MFI.getStackSize();
   Register FramePtr = RegInfo->getFrameRegister(MF);
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     return;
 
   // First put ourselves on the first (from top) terminator instructions.
   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   if (!AFI->hasStackFrame()) {
-    if (NumBytes - ArgRegsSaveSize != 0)
-      emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ArgRegsSaveSize,
+    if (NumBytes - ReservedArgStack != 0)
+      emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ReservedArgStack,
                    MachineInstr::FrameDestroy);
   } else {
     // Unwind MBBI to point to first LDR / VLDRD.
     if (MBBI != MBB.begin()) {
       do {
         --MBBI;
       } while (MBBI != MBB.begin() &&
                MBBI->getFlag(MachineInstr::FrameDestroy));
       if (!MBBI->getFlag(MachineInstr::FrameDestroy))
         ++MBBI;
     }
 
     // Move SP to start of FP callee save spill area.
-    NumBytes -= (ArgRegsSaveSize +
+    NumBytes -= (ReservedArgStack +
                  AFI->getFPCXTSaveAreaSize() +
                  AFI->getGPRCalleeSavedArea1Size() +
                  AFI->getGPRCalleeSavedArea2Size() +
                  AFI->getDPRCalleeSavedGapSize() +
                  AFI->getDPRCalleeSavedAreaSize());
 
     // Reset SP based on frame pointer only if the stack frame extends beyond
     // frame pointer stack slot or target is ELF and the function has FP.
if (AFI->getDPRCalleeSavedGapSize()) {
                    MachineInstr::FrameDestroy);
     }
 
     if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
     if (AFI->getFPCXTSaveAreaSize()) MBBI++;
   }
 
-  if (ArgRegsSaveSize)
-    emitSPUpdate(isARM, MBB, MBBI, dl, TII, ArgRegsSaveSize,
+  if (ReservedArgStack || IncomingArgStackToRestore) {
+    assert(ReservedArgStack + IncomingArgStackToRestore >= 0 &&
+           "attempting to restore negative stack amount");
+    emitSPUpdate(isARM, MBB, MBBI, dl, TII,
+                 ReservedArgStack + IncomingArgStackToRestore,
                  MachineInstr::FrameDestroy);
   }
+}
 
 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
 /// debug info.  It's the same as what we use for resolving the code-gen
 /// references for now.  FIXME: This can go wrong when references are
 /// SP-relative and simple call frames aren't used.
 StackOffset ARMFrameLowering::getFrameIndexReference(const MachineFunction &MF,
                                                      int FI,
                                                      Register &FrameReg) const {
ARMFrameLowering::getCalleeSavedSpillSlots(unsigned &NumEntries) const {
   return FixedSpillOffsets;
 }
 
 MachineBasicBlock::iterator ARMFrameLowering::eliminateCallFramePseudoInstr(
     MachineFunction &MF, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator I) const {
   const ARMBaseInstrInfo &TII =
       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  bool isARM = !AFI->isThumbFunction();
+  DebugLoc dl = I->getDebugLoc();
+  unsigned Opc = I->getOpcode();
+  bool IsDestroy = Opc == TII.getCallFrameDestroyOpcode();
+  unsigned CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
+
+  assert(!AFI->isThumb1OnlyFunction() &&
+         "This eliminateCallFramePseudoInstr does not support Thumb1!");
+
+  int PIdx = I->findFirstPredOperandIdx();
+  ARMCC::CondCodes Pred = (PIdx == -1)
+                              ? ARMCC::AL
+                              : (ARMCC::CondCodes)I->getOperand(PIdx).getImm();
+  unsigned PredReg = TII.getFramePred(*I);
+
   if (!hasReservedCallFrame(MF)) {
+    // Bail early if the callee is expected to do the adjustment. If
+    // CalleePopAmount is valid but 0 anyway, Amount will be 0 too so it doesn't
+    // matter if we continue a bit longer.
+    if (IsDestroy && CalleePopAmount != 0)
+      return MBB.erase(I);
+
     // If we have alloca, convert as follows:
     // ADJCALLSTACKDOWN -> sub, sp, sp, amount
     // ADJCALLSTACKUP   -> add, sp, sp, amount
-    MachineInstr &Old = *I;
-    DebugLoc dl = Old.getDebugLoc();
-    unsigned Amount = TII.getFrameSize(Old);
+    unsigned Amount = TII.getFrameSize(*I);
     if (Amount != 0) {
       // We need to keep the stack aligned properly.  To do this, we round the
       // amount of space needed for the outgoing arguments up to the next
       // alignment boundary.
       Amount = alignSPAdjust(Amount);
 
-      ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-      assert(!AFI->isThumb1OnlyFunction() &&
-             "This eliminateCallFramePseudoInstr does not support Thumb1!");
-      bool isARM = !AFI->isThumbFunction();
-
-      // Replace the pseudo instruction with a new instruction...
-      unsigned Opc = Old.getOpcode();
-      int PIdx = Old.findFirstPredOperandIdx();
-      ARMCC::CondCodes Pred =
-          (PIdx == -1) ? ARMCC::AL
-                       : (ARMCC::CondCodes)Old.getOperand(PIdx).getImm();
-      unsigned PredReg = TII.getFramePred(Old);
       if (Opc == ARM::ADJCALLSTACKDOWN || Opc == ARM::tADJCALLSTACKDOWN) {
         emitSPUpdate(isARM, MBB, I, dl, TII, -Amount, MachineInstr::NoFlags,
                      Pred, PredReg);
       } else {
         assert(Opc == ARM::ADJCALLSTACKUP || Opc == ARM::tADJCALLSTACKUP);
         emitSPUpdate(isARM, MBB, I, dl, TII, Amount, MachineInstr::NoFlags,
                      Pred, PredReg);
       }
     }
+  } else if (CalleePopAmount != 0) {
+    // If the calling convention demands that the callee pops arguments from the
+    // stack, we want to add it back if we have a reserved call frame.
+    emitSPUpdate(isARM, MBB, I, dl, TII, -CalleePopAmount,
+                 MachineInstr::NoFlags, Pred, PredReg);
   }
   return MBB.erase(I);
 }
 
 /// Get the minimum constant for ARM that is greater than or equal to the
 /// argument. In ARM, constants can have any value that can be produced by
 /// rotating an 8-bit value to the right by an even number of bits within a
 /// 32-bit word.

llvm/lib/Target/ARM/ARMISelLowering.h

private:
 
     using RegsToPassVector = SmallVector<std::pair<unsigned, SDValue>, 8>;
 
     void PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, SDValue Chain,
                           SDValue &Arg, RegsToPassVector &RegsToPass,
                           CCValAssign &VA, CCValAssign &NextVA,
                           SDValue &StackPtr,
                           SmallVectorImpl<SDValue> &MemOpChains,
-                          ISD::ArgFlagsTy Flags) const;
+                          bool IsTailCall,
+                          int SPDiff) const;
     SDValue GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
                                  SDValue &Root, SelectionDAG &DAG,
                                  const SDLoc &dl) const;
 
     CallingConv::ID getEffectiveCallingConv(CallingConv::ID CC,
                                             bool isVarArg) const;
     CCAssignFn *CCAssignFnForNode(CallingConv::ID CC, bool Return,
                                   bool isVarArg) const;
-    SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
-                             const SDLoc &dl, SelectionDAG &DAG,
-                             const CCValAssign &VA,
-                             ISD::ArgFlagsTy Flags) const;
+    std::pair<SDValue, MachinePointerInfo>
+    computeAddrForCallArg(const SDLoc &dl, SelectionDAG &DAG,
+                          const CCValAssign &VA, SDValue StackPtr,
+                          bool IsTailCall, int SPDiff) const;
     SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG,
                                     const ARMSubtarget *Subtarget) const;
     SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
                                     const ARMSubtarget *Subtarget) const;
     SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/ARM/ARMISelLowering.cpp

case CallingConv::CFGuard_Check:
     return CC;
   case CallingConv::PreserveMost:
     return CallingConv::PreserveMost;
   case CallingConv::ARM_AAPCS_VFP:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
     return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
   case CallingConv::C:
+  case CallingConv::Tail:
     if (!Subtarget->isAAPCS_ABI())
       return CallingConv::ARM_APCS;
     else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() &&
              getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
              !isVarArg)
       return CallingConv::ARM_AAPCS_VFP;
     else
       return CallingConv::ARM_AAPCS;
if (VA.needsCustom() &&
       Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
 
     InVals.push_back(Val);
   }
 
   return Chain;
 }
 
-/// LowerMemOpCallTo - Store the argument to the stack.
-SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
-                                            SDValue Arg, const SDLoc &dl,
-                                            SelectionDAG &DAG,
-                                            const CCValAssign &VA,
-                                            ISD::ArgFlagsTy Flags) const {
-  unsigned LocMemOffset = VA.getLocMemOffset();
-  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
-  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
+    const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
+    bool IsTailCall, int SPDiff) const {
+  SDValue DstAddr;
+  MachinePointerInfo DstInfo;
+  int32_t Offset = VA.getLocMemOffset();
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  if (IsTailCall) {
+        Offset += SPDiff;
+        auto PtrVT = getPointerTy(DAG.getDataLayout());
+        int Size = VA.getLocVT().getFixedSizeInBits() / 8;
+        int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
+        DstAddr = DAG.getFrameIndex(FI, PtrVT);
+        DstInfo =
+            MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
+  } else {
+        SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
+        DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
                               StackPtr, PtrOff);
-  return DAG.getStore(
-      Chain, dl, Arg, PtrOff,
-      MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset));
+        DstInfo =
+            MachinePointerInfo::getStack(DAG.getMachineFunction(), Offset);
+  }
+
+  return std::make_pair(DstAddr, DstInfo);
 }
 
 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
                                          SDValue Chain, SDValue &Arg,
                                          RegsToPassVector &RegsToPass,
                                          CCValAssign &VA, CCValAssign &NextVA,
                                          SDValue &StackPtr,
                                          SmallVectorImpl<SDValue> &MemOpChains,
-                                         ISD::ArgFlagsTy Flags) const {
+                                         bool IsTailCall,
+                                         int SPDiff) const {
   SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
                               DAG.getVTList(MVT::i32, MVT::i32), Arg);
   unsigned id = Subtarget->isLittle() ? 0 : 1;
   RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
 
   if (NextVA.isRegLoc())
     RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
   else {
     assert(NextVA.isMemLoc());
     if (!StackPtr.getNode())
       StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
                                     getPointerTy(DAG.getDataLayout()));
 
-    MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id),
-                                           dl, DAG, NextVA,
-                                           Flags));
+    SDValue DstAddr;
+    MachinePointerInfo DstInfo;
+    std::tie(DstAddr, DstInfo) =
+        computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
+    MemOpChains.push_back(
+        DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo));
   }
 }
 
+static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
+  return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
+         CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
+}
+
 /// LowerCall - Lowering a call into a callseq_start <-
 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
 /// nodes.
 SDValue
 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG                     = CLI.DAG;
   SDLoc &dl                             = CLI.DL;
   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
   SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
   SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
   SDValue Chain                         = CLI.Chain;
   SDValue Callee                        = CLI.Callee;
   bool &isTailCall                      = CLI.IsTailCall;
   CallingConv::ID CallConv              = CLI.CallConv;
   bool doesNotRet                       = CLI.DoesNotReturn;
   bool isVarArg                         = CLI.IsVarArg;
 
   MachineFunction &MF = DAG.getMachineFunction();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   MachineFunction::CallSiteInfo CSInfo;
   bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
   bool isThisReturn = false;
   bool isCmseNSCall   = false;
+  bool isSibCall = false;
   bool PreferIndirect = false;
 
   // Determine whether this is a non-secure function call.
   if (CLI.CB && CLI.CB->getAttributes().hasFnAttribute("cmse_nonsecure_call"))
     isCmseNSCall = true;
 
   // Disable tail calls if they're not supported.
   if (!Subtarget->supportsTailCall())
if (isTailCall) {
     // Check if it's really possible to do a tail call.
     isTailCall = IsEligibleForTailCallOptimization(
         Callee, CallConv, isVarArg, isStructRet,
         MF.getFunction().hasStructRetAttr(), Outs, OutVals, Ins, DAG,
         PreferIndirect);
     if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
       report_fatal_error("failed to perform tail call elimination on a call "
                          "site marked musttail");
+
+    if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
+        CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
+      isSibCall = true;
+
     // We don't support GuaranteedTailCallOpt for ARM, only automatically
     // detected sibcalls.
     if (isTailCall)
       ++NumTailCalls;
   }
 
   // Analyze operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                  *DAG.getContext());
   CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg));
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();
 
-  if (isTailCall) {
-    // For tail calls, memory operands are available in our caller's stack.
+  // SPDiff is the byte offset of the call's argument area from the callee's.
+  // Stores to callee stack arguments will be placed in FixedStackSlots offset
+  // by this amount for a tail call. In a sibling call it must be 0 because the
+  // caller will deallocate the entire stack and the callee still expects its
+  // arguments to begin at SP+0. Completely unused for non-tail calls.
+  int SPDiff = 0;
+
+  if (isTailCall && !isSibCall) {
+    auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
+    unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
+
+    // Since callee will pop argument stack as a tail call, we must keep the
+    // popped size 16-byte aligned.
+    Align StackAlign = DAG.getDataLayout().getStackAlignment();
+    NumBytes = alignTo(NumBytes, StackAlign);
+
+    // SPDiff will be negative if this tail call requires more space than we
+    // would automatically have in our incoming argument space. Positive if we
+    // can actually shrink the stack.
+    SPDiff = NumReusableBytes - NumBytes;
+
+    // If this call requires more stack than we have available from
+    // LowerFormalArguments, tell FrameLowering to reserve space for it.
+    if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
+      AFI->setArgRegsSaveSize(-SPDiff);
+  }
+
+  if (isSibCall) {
+    // For sibling tail calls, memory operands are available in our caller's stack.
     NumBytes = 0;
   } else {
     // Adjust the stack pointer for the new arguments...
     // These operations are automatically eliminated by the prolog/epilog pass
-    Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
+    Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl);
   }
 
   SDValue StackPtr =
       DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
 
   RegsToPassVector RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
 
+  // During a tail call, stores to the argument area must happen after all of
+  // the function's incoming arguments have been loaded because they may alias.
+  // This is done by folding in a TokenFactor from LowerFormalArguments, but
+  // there's no point in doing so repeatedly so this tracks whether that's
+  // happened yet.
+  bool AfterFormalArgLoads = false;
+
   // Walk the register/memloc assignments, inserting copies/loads.  In the case
   // of tail call optimization, arguments are handled later.
   for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
        i != e;
        ++i, ++realArgIdx) {
     CCValAssign &VA = ArgLocs[i];
     SDValue Arg = OutVals[realArgIdx];
     ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
     case CCValAssign::AExt:
       Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
       break;
     case CCValAssign::BCvt:
       Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
       break;
     }
 
+    if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
+      Chain = DAG.getStackArgumentTokenFactor(Chain);
+      AfterFormalArgLoads = true;
+    }
+
     // f16 arguments have their size extended to 4 bytes and passed as if they
     // had been copied to the LSBs of a 32-bit register.
     // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
     if (VA.needsCustom() &&
         (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
       Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
     } else {
       // f16 arguments could have been extended prior to argument lowering.
for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
     // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
     if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
       SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                                 DAG.getConstant(0, dl, MVT::i32));
       SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                                 DAG.getConstant(1, dl, MVT::i32));
 
       PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
-                       StackPtr, MemOpChains, Flags);
+                       StackPtr, MemOpChains, isTailCall, SPDiff);
 
       VA = ArgLocs[++i]; // skip ahead to next loc
       if (VA.isRegLoc()) {
         PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
-                         StackPtr, MemOpChains, Flags);
+                         StackPtr, MemOpChains, isTailCall, SPDiff);
       } else {
         assert(VA.isMemLoc());
-
-        MemOpChains.push_back(
-            LowerMemOpCallTo(Chain, StackPtr, Op1, dl, DAG, VA, Flags));
+        SDValue DstAddr;
+        MachinePointerInfo DstInfo;
+        std::tie(DstAddr, DstInfo) =
+            computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
+        MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo));
       }
     } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
       PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
-                       StackPtr, MemOpChains, Flags);
+                       StackPtr, MemOpChains, isTailCall, SPDiff);
     } else if (VA.isRegLoc()) {
       if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
           Outs[0].VT == MVT::i32) {
         assert(VA.getLocVT() == MVT::i32 &&
                "unexpected calling convention register assignment");
         assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
                "unexpected use of 'returned'");
         isThisReturn = true;
if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
         // helps us to calculate stack slot for remained part properly.
         offset = RegEnd - RegBegin;
 
         CCInfo.nextInRegsParam();
       }
 
       if (Flags.getByValSize() > 4*offset) {
         auto PtrVT = getPointerTy(DAG.getDataLayout());
-        unsigned LocMemOffset = VA.getLocMemOffset();
-        SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
-        SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
+        SDValue Dst;
+        MachinePointerInfo DstInfo;
+        std::tie(Dst, DstInfo) =
+            computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
         SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
         SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
         SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
                                            MVT::i32);
         SDValue AlignNode =
             DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
 
         SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
         SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
         MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
                                           Ops));
       }
-    } else if (!isTailCall) {
+    } else {
       assert(VA.isMemLoc());
+      SDValue DstAddr;
+      MachinePointerInfo DstInfo;
+      std::tie(DstAddr, DstInfo) =
+          computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
 
-      MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
-                                             dl, DAG, VA, Flags));
+      SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo);
+      MemOpChains.push_back(Store);
     }
   }
 
   if (!MemOpChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
 
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
              // Emit regular call when code size is the priority
              !Subtarget->hasMinSize())
       // "mov lr, pc; b _foo" to avoid confusing the RSP
       CallOpc = ARMISD::CALL_NOLINK;
     else
       CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
   }
 
+  // We don't usually want to end the call-sequence here because we would tidy
+  // the frame up *after* the call, however in the ABI-changing tail-call case
+  // we've carefully laid out the parameters so that when sp is reset they'll be
+  // in the correct location.
+  if (isTailCall && !isSibCall) {
+    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
+                               DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
+    InFlag = Chain.getValue(1);
+  }
+
   std::vector<SDValue> Ops;
   Ops.push_back(Chain);
   Ops.push_back(Callee);
 
+  if (isTailCall) {
+    Ops.push_back(DAG.getTargetConstant(SPDiff, dl, MVT::i32));
+  }
+
   // Add argument registers to the end of the list so that they are known live
   // into the call.
   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
     Ops.push_back(DAG.getRegister(RegsToPass[i].first,
                                   RegsToPass[i].second.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   if (!isTailCall) {
ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   // Returns a chain and a flag for retval copy to use.
   Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
   InFlag = Chain.getValue(1);
   DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
 
+  // If we're guaranteeing tail-calls will be honoured, the callee must
+  // pop its own argument stack on return. But this call is *not* a tail call so
+  // we need to undo that after it returns to restore the status-quo.
+  bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
+  uint64_t CalleePopBytes =
+      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
+
   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
-                             DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
+                             DAG.getIntPtrConstant(CalleePopBytes, dl, true),
+                             InFlag, dl);
   if (!Ins.empty())
     InFlag = Chain.getValue(1);
 
   // Handle result values, copying them out of physregs into vregs that we
   // return.
   return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
                          InVals, isThisReturn,
                          isThisReturn ? OutVals[0] : SDValue());
bool ARMTargetLowering::IsEligibleForTailCallOptimization(
   // require ABI changes. This is what gcc calls sibcall.
 
   // Exception-handling functions need a special set of instructions to indicate
   // a return to the hardware. Tail-calling another function would probably
   // break this.
   if (CallerF.hasFnAttribute("interrupt"))
     return false;
 
+  if (canGuaranteeTCO(CalleeCC, getTargetMachine().Options.GuaranteedTailCallOpt))
+    return CalleeCC == CallerCC;
+
   // Also avoid sibcall optimization if either caller or callee uses struct
   // return semantics.
   if (isCalleeStructRet || isCallerStructRet)
     return false;
 
   // Externally-defined functions with weak linkage should not be
   // tail-called on ARM when the OS does not support dynamic
   // pre-emption of symbols, as the AAELF spec requires normal calls
if (isVarArg && MFI.hasVAStart()) {
     if (AFI->isCmseNSEntryFunction()) {
       DiagnosticInfoUnsupported Diag(
           DAG.getMachineFunction().getFunction(),
           "secure entry function must not be variadic", dl.getDebugLoc());
       DAG.getContext()->diagnose(Diag);
     }
   }
 
-  AFI->setArgumentStackSize(CCInfo.getNextStackOffset());
+  unsigned StackArgSize = CCInfo.getNextStackOffset();
+  bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
+  if (canGuaranteeTCO(CallConv, TailCallOpt)) {
+    // The only way to guarantee a tail call is if the callee restores its
+    // argument area, but it must also keep the stack aligned when doing so.
+    const DataLayout &DL = DAG.getDataLayout();
+    StackArgSize = alignTo(StackArgSize, DL.getStackAlignment());
+
+    AFI->setArgumentStackToRestore(StackArgSize);
+  }
+  AFI->setArgumentStackSize(StackArgSize);
 
   if (CCInfo.getNextStackOffset() > 0 && AFI->isCmseNSEntryFunction()) {
     DiagnosticInfoUnsupported Diag(
         DAG.getMachineFunction().getFunction(),
         "secure entry function requires arguments on stack", dl.getDebugLoc());
     DAG.getContext()->diagnose(Diag);
   }
 

llvm/lib/Target/ARM/ARMInstrInfo.td

 def SDT_ARMEH_SJLJ_Longjmp: SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisInt<1>]>;
 def SDT_ARMEH_SJLJ_SetupDispatch: SDTypeProfile<0, 0, []>;
 
 def SDT_ARMMEMBARRIER     : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
 def SDT_ARMPREFETCH : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisSameAs<1, 2>,
                                            SDTCisInt<1>]>;
 
-def SDT_ARMTCRET : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
+def SDT_ARMTCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
 
 def SDT_ARMBFI : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                       SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
 
 def SDT_WIN__DBZCHK : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
 
 def SDT_ARMMEMCPY  : SDTypeProfile<2, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                           SDTCisVT<2, i32>, SDTCisVT<3, i32>,
def BXJ : ABI<0b0001, (outs), (ins GPR:$func), NoItinerary, "bxj", "\t$func",
   let Inst{3-0} = func;
   let isBranch = 1;
   let isIndirectBranch = 1;
 }
 
 // Tail calls.
 
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [SP] in {
-  def TCRETURNdi : PseudoInst<(outs), (ins i32imm:$dst), IIC_Br, []>,
+  def TCRETURNdi : PseudoInst<(outs), (ins i32imm:$dst, i32imm:$SPDiff), IIC_Br, []>,
                    Sched<[WriteBr]>;
 
-  def TCRETURNri : PseudoInst<(outs), (ins tcGPR:$dst), IIC_Br, []>,
+  def TCRETURNri : PseudoInst<(outs), (ins tcGPR:$dst, i32imm:$SPDiff), IIC_Br, []>,
                    Sched<[WriteBr]>;
 
   def TAILJMPd : ARMPseudoExpand<(outs), (ins arm_br_target:$dst),
                                  4, IIC_Br, [],
                                  (Bcc arm_br_target:$dst, (ops 14, zero_reg))>,
                                  Requires<[IsARM]>, Sched<[WriteBr]>;
 
   def TAILJMPr : ARMPseudoExpand<(outs), (ins tcGPR:$dst),

 def : ARMPat<(ARMWrapper texternalsym :$dst), (MOVi32imm texternalsym :$dst)>,
             Requires<[IsARM, UseMovt]>;
 def : ARMPat<(ARMWrapperJT tjumptable:$dst),
              (LEApcrelJT tjumptable:$dst)>;
 
 // TODO: add,sub,and, 3-instr forms?
 
 // Tail calls. These patterns also apply to Thumb mode.
-def : Pat<(ARMtcret tcGPR:$dst), (TCRETURNri tcGPR:$dst)>;
-def : Pat<(ARMtcret (i32 tglobaladdr:$dst)), (TCRETURNdi texternalsym:$dst)>;
-def : Pat<(ARMtcret (i32 texternalsym:$dst)), (TCRETURNdi texternalsym:$dst)>;
+def : Pat<(ARMtcret tcGPR:$dst, (i32 timm:$SPDiff)),
+          (TCRETURNri tcGPR:$dst, timm:$SPDiff)>;
+def : Pat<(ARMtcret (i32 tglobaladdr:$dst), (i32 timm:$SPDiff)),
+          (TCRETURNdi texternalsym:$dst, (i32 timm:$SPDiff))>;
+def : Pat<(ARMtcret (i32 texternalsym:$dst), (i32 timm:$SPDiff)),
+          (TCRETURNdi texternalsym:$dst, i32imm:$SPDiff)>;
 
 // Direct calls
 def : ARMPat<(ARMcall texternalsym:$func), (BL texternalsym:$func)>;
 def : ARMPat<(ARMcall_nolink texternalsym:$func),
              (BMOVPCB_CALL texternalsym:$func)>;
 
 // zextload i1 -> zextload i8
 def : ARMPat<(zextloadi1 addrmode_imm12:$addr), (LDRBi12 addrmode_imm12:$addr)>;

llvm/lib/Target/ARM/ARMMachineFunctionInfo.h

class ARMFunctionInfo : public MachineFunctionInfo {
 
   /// StByValParamsPadding - For parameter that is split between
   /// GPRs and memory; while recovering GPRs part, when
   /// StackAlignment > 4, and GPRs-part-size mod StackAlignment != 0,
   /// we need to insert gap before parameter start address. It allows to
   /// "attach" GPR-part to the part that was passed via stack.
   unsigned StByValParamsPadding = 0;
 
-  /// VarArgsRegSaveSize - Size of the register save area for vararg functions.
+  /// ArgsRegSaveSize - Size of the register save area for vararg functions or
+  /// those making guaranteed tail calls that need more stack argument space
+  /// than is provided by this functions incoming parameters.
   ///
   unsigned ArgRegsSaveSize = 0;
 
   /// ReturnRegsCount - Number of registers used up in the return.
   unsigned ReturnRegsCount = 0;
 
   /// HasStackFrame - True if this function has a stack frame. Set by
   /// determineCalleeSaves().
class ARMFunctionInfo : public MachineFunctionInfo {
   /// CPEClones - Track constant pool entries clones created by Constant Island
   /// pass.
   DenseMap<unsigned, unsigned> CPEClones;
 
   /// ArgumentStackSize - amount of bytes on stack consumed by the arguments
   /// being passed on the stack
   unsigned ArgumentStackSize = 0;
 
+  /// ArgumentStackToRestore - amount of bytes on stack consumed that we must
+  /// restore on return.
+  unsigned ArgumentStackToRestore = 0;
+
   /// CoalescedWeights - mapping of basic blocks to the rolling counter of
   /// coalesced weights.
   DenseMap<const MachineBasicBlock*, unsigned> CoalescedWeights;
 
   /// True if this function has a subset of CSRs that is handled explicitly via
   /// copies.
   bool IsSplitCSR = false;
 
public:
   void setGPRCalleeSavedArea1Size(unsigned s) { GPRCS1Size = s; }
   void setGPRCalleeSavedArea2Size(unsigned s) { GPRCS2Size = s; }
   void setDPRCalleeSavedGapSize(unsigned s)   { DPRCSAlignGapSize = s; }
   void setDPRCalleeSavedAreaSize(unsigned s)  { DPRCSSize = s; }
 
   unsigned getArgumentStackSize() const { return ArgumentStackSize; }
   void setArgumentStackSize(unsigned size) { ArgumentStackSize = size; }
 
+  unsigned getArgumentStackToRestore() const { return ArgumentStackToRestore; }
+  void setArgumentStackToRestore(unsigned v) { ArgumentStackToRestore = v; }
+
   void initPICLabelUId(unsigned UId) {
     PICLabelUId = UId;
   }
 
   unsigned getNumPICLabels() const {
     return PICLabelUId;
   }
 

llvm/lib/Target/ARM/ARMSubtarget.cpp

void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   // to be used. We take the optimistic approach of generating the tail call and
   // perhaps taking a hit if we need to restore the LR.
 
   // Thumb1 PIC calls to external symbols use BX, so they can be tail calls,
   // but we need to make sure there are enough registers; the only valid
   // registers are the 4 used for parameters.  We don't currently do this
   // case.
 
-  SupportsTailCall = !isThumb() || hasV8MBaselineOps();
+  SupportsTailCall = !isThumb1Only() || hasV8MBaselineOps();
 
   if (isTargetMachO() && isTargetIOS() && getTargetTriple().isOSVersionLT(5, 0))
     SupportsTailCall = false;
 
   switch (IT) {
   case DefaultIT:
     RestrictIT = hasV8Ops() && !hasMinSize();
     break;

llvm/test/CodeGen/ARM/dbg-tcreturn.ll

 ; RUN: llc %s -o - -stop-after=finalize-isel -verify-machineinstrs | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-Fi8-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
 target triple = "thumbv7-apple-ios7.0.0"
 
 ; CHECK-LABEL: name: test
 ; CHECK:       body:
 ; CHECK-NEXT:   bb.0.entry:
 ; CHECK-NEXT:     liveins: $r0, $r1
 ; CHECK:          %1:gpr = COPY $r1
 ; CHECK-NEXT:     %0:gpr = COPY $r0
 ; CHECK-NEXT:     $r0 = COPY %0
 ; CHECK-NEXT:     $r1 = COPY %1
 ; CHECK-NEXT:     DBG_VALUE $noreg, $noreg, !13, !DIExpression(), debug-location !16
-; CHECK-NEXT:     TCRETURNdi &__divsi3, implicit $sp, implicit $r0, implicit $r1
+; CHECK-NEXT:     TCRETURNdi &__divsi3, 0, implicit $sp, implicit $r0, implicit $r1
 
 define i32 @test(i32 %a1, i32 %a2) !dbg !5 {
 entry:
   %res = sdiv i32 %a1, %a2
   call void @llvm.dbg.value(metadata i32 %res, metadata !13, metadata !DIExpression()), !dbg !16
   ret i32 %res
 }
 

llvm/test/CodeGen/ARM/fastcc-tailcall.ll

+; RUN: llc -verify-machineinstrs < %s -mtriple=thumbv7k-apple-watchos -tailcallopt | FileCheck %s
+
+declare fastcc void @callee_stack0()
+declare fastcc void @callee_stack4([4 x i32], i32)
+declare fastcc void @callee_stack20([4 x i32], [5 x i32])
+declare extern_weak fastcc void @callee_weak()
+
+define fastcc void @caller_to0_from0() nounwind {
+; CHECK-LABEL: _caller_to0_from0:
+
+  tail call fastcc void @callee_stack0()
+  ret void
+; CHECK-NOT: add
+; CHECK-NOT: sub
+; CHECK: b.w _callee_stack0
+}
+
+define fastcc void @caller_to0_from4([4 x i32], i32) {
+; CHECK-LABEL: _caller_to0_from4:
+
+  tail call fastcc void @callee_stack0()
+  ret void
+
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack0
+}
+
+define fastcc void @caller_to4_from0() {
+; Key point is that the "42" should go #16 below incoming stack
+; pointer (we didn't have arg space to reuse).
+  tail call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK-LABEL: _caller_to4_from0:
+; CHECK: sub sp, #16
+; CHECK: movs [[TMP:r[0-9]+]], #42
+; CHECK: str [[TMP]], [sp]
+; CHECK-NOT: add sp
+; CHECK: b.w _callee_stack4
+
+}
+
+define fastcc void @caller_to4_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to4_from4:
+; CHECK-NOT: sub sp
+; Key point is that the "%a" should go where at SP on entry.
+  tail call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{r[0-9]+}}, [sp]
+; CHECK-NOT: add sp
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+define fastcc void @caller_to20_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to20_from4:
+; CHECK: sub sp, #16
+
+; Important point is that the call reuses the "dead" argument space
+; above %a on the stack. If it tries to go below incoming-SP then the
+; _callee will not deallocate the space, even in fastcc.
+  tail call fastcc void @callee_stack20([4 x i32] undef, [5 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5])
+
+; CHECK: str {{.*}}, [sp]
+; CHECK: str {{.*}}, [sp, #4]
+; CHECK: str {{.*}}, [sp, #8]
+; CHECK: str {{.*}}, [sp, #12]
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK-NOT: add sp
+; CHECK-NOT: sub sp
+; CHECK: b.w _callee_stack20
+  ret void
+}
+
+
+define fastcc void @caller_to4_from24([4 x i32], i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: _caller_to4_from24:
+
+
+; Key point is that the "%a" should go where at #16 above SP on entry.
+  tail call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+
+define fastcc void @caller_to20_from20([4 x i32], [5 x i32] %a) {
+; CHECK-LABEL: _caller_to20_from20:
+; CHECK-NOT: add sp,
+; CHECK-NOT: sub sp,
+
+; Here we want to make sure that both loads happen before the stores:
+; otherwise either %a or %b.w will be wrongly clobbered.
+  tail call fastcc void @callee_stack20([4 x i32] undef, [5 x i32] %a)
+  ret void
+
+  ; If these ever get interleaved make sure aliasing slots don't clobber each
+  ; other.
+; CHECK: ldrd {{.*}}, {{.*}}, [sp, #12]
+; CHECK: ldm.w sp,
+; CHECK: stm.w
+; CHECK: strd
+; CHECK-NEXT: b.w _callee_stack20
+}
+
+define fastcc void @disable_tail_calls() nounwind "disable-tail-calls"="true" {
+; CHECK-LABEL: disable_tail_calls:
+
+  tail call fastcc void @callee_stack0()
+  ret void
+
+; CHECK: bl _callee_stack0
+; CHECK: ret
+}
+
+define fastcc void @normal_ret_with_stack([4 x i32], i32 %a) {
+; CHECK: _normal_ret_with_stack:
+; CHECK: add sp, #16
+; CHECK: bx lr
+  ret void
+}
+
+declare { [2 x float] } @get_vec2()
+
+define void @fromC_totail() {
+; COMMON-LABEL: fromC_totail:
+; COMMON: puch {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; We must reset the stack to where it was before the call by undoing its extra stack pop.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl callee_stack4
+; COMMON: sub sp, #16
+
+  call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+define void @fromC_totail_noreservedframe(i32 %len) {
+; COMMON-LABEL: fromC_totail_noreservedframe:
+; COMMON: sub.w sp, sp, r{{.*}}
+
+; COMMON: movs [[TMP:r[0-9]+]], #42
+  ; Note stack is subtracted here to allocate space for arg
+; COMMON: sub.w sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; And here.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; But not restored here because callee_stack8 did that for us.
+; COMMON-NOT: sub sp,
+
+  ; Variable sized allocation prevents reserving frame at start of function so each call must allocate any stack space it needs.
+  %var = alloca i32, i32 %len
+
+  call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  call fastcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+declare void @Ccallee_stack4([4 x i32], i32)
+
+define fastcc void @fromtail_toC() {
+; COMMON-LABEL: fromtail_toC:
+; COMMON: push {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _Ccallee_stack4
+  ; C callees will return with the stack exactly where we left it, so we mustn't try to fix anything.
+; COMMON-NOT: add sp,
+; COMMON-NOT: sub sp,
+; COMMON: str [[TMP]], [sp]{{$}}
+; COMMON: bl _Ccallee_stack4
+; COMMON-NOT: sub sp,
+
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  ret void
+}

llvm/test/CodeGen/ARM/peephole-callee-save-regalloc.mir

bb.0 (%ir-block.0):
     %3 = COPY $r3
 
     %4 = t2MOVi32imm 270337
     %5 = t2ADDrr killed %2, killed %4, 14, $noreg, $noreg
     $r0 = COPY %0
     $r1 = COPY %1
     $r2 = COPY %2
     $r3 = COPY %3
-    TCRETURNri killed %5, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3
+    TCRETURNri killed %5, 0, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3
 ...

llvm/test/CodeGen/ARM/swifttailcc-call.ll

+; RUN: llc -verify-machineinstrs < %s -mtriple=thumbv7k-apple-watchos | FileCheck %s
+
+declare swifttailcc void @callee_stack0()
+declare swifttailcc void @callee_stack4([4 x i32], i32)
+declare swifttailcc void @callee_stack20([4 x i32], [5 x i32])
+declare extern_weak swifttailcc void @callee_weak()
+
+define swifttailcc void @caller_to0_from0() nounwind {
+; CHECK-LABEL: _caller_to0_from0:
+
+  tail call swifttailcc void @callee_stack0()
+  ret void
+; CHECK-NOT: add
+; CHECK-NOT: sub
+; CHECK: b.w _callee_stack0
+}
+
+define swifttailcc void @caller_to0_from4([4 x i32], i32) {
+; CHECK-LABEL: _caller_to0_from4:
+
+  tail call swifttailcc void @callee_stack0()
+  ret void
+
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack0
+}
+
+define swifttailcc void @caller_to4_from0() {
+; Key point is that the "42" should go #16 below incoming stack
+; pointer (we didn't have arg space to reuse).
+  tail call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK-LABEL: _caller_to4_from0:
+; CHECK: sub sp, #16
+; CHECK: movs [[TMP:r[0-9]+]], #42
+; CHECK: str [[TMP]], [sp]
+; CHECK-NOT: add sp
+; CHECK: b.w _callee_stack4
+
+}
+
+define swifttailcc void @caller_to4_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to4_from4:
+; CHECK-NOT: sub sp
+; Key point is that the "%a" should go where at SP on entry.
+  tail call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{r[0-9]+}}, [sp]
+; CHECK-NOT: add sp
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+define swifttailcc void @caller_to20_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to20_from4:
+; CHECK: sub sp, #16
+
+; Important point is that the call reuses the "dead" argument space
+; above %a on the stack. If it tries to go below incoming-SP then the
+; _callee will not deallocate the space, even in swifttailcc.
+  tail call swifttailcc void @callee_stack20([4 x i32] undef, [5 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5])
+
+; CHECK: str {{.*}}, [sp]
+; CHECK: str {{.*}}, [sp, #4]
+; CHECK: str {{.*}}, [sp, #8]
+; CHECK: str {{.*}}, [sp, #12]
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK-NOT: add sp
+; CHECK-NOT: sub sp
+; CHECK: b.w _callee_stack20
+  ret void
+}
+
+
+define swifttailcc void @caller_to4_from24([4 x i32], i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: _caller_to4_from24:
+
+
+; Key point is that the "%a" should go where at #16 above SP on entry.
+  tail call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+
+define swifttailcc void @caller_to20_from20([4 x i32], [5 x i32] %a) {
+; CHECK-LABEL: _caller_to20_from20:
+; CHECK-NOT: add sp,
+; CHECK-NOT: sub sp,
+
+; Here we want to make sure that both loads happen before the stores:
+; otherwise either %a or %b.w will be wrongly clobbered.
+  tail call swifttailcc void @callee_stack20([4 x i32] undef, [5 x i32] %a)
+  ret void
+
+  ; If these ever get interleaved make sure aliasing slots don't clobber each
+  ; other.
+; CHECK: ldrd {{.*}}, {{.*}}, [sp, #12]
+; CHECK: ldm.w sp,
+; CHECK: stm.w
+; CHECK: strd
+; CHECK-NEXT: b.w _callee_stack20
+}
+
+define swifttailcc void @disable_tail_calls() nounwind "disable-tail-calls"="true" {
+; CHECK-LABEL: disable_tail_calls:
+
+  tail call swifttailcc void @callee_stack0()
+  ret void
+
+; CHECK: bl _callee_stack0
+; CHECK: ret
+}
+
+define swifttailcc void @normal_ret_with_stack([4 x i32], i32 %a) {
+; CHECK: _normal_ret_with_stack:
+; CHECK: add sp, #16
+; CHECK: bx lr
+  ret void
+}
+
+declare { [2 x float] } @get_vec2()
+
+define void @fromC_totail() {
+; COMMON-LABEL: fromC_totail:
+; COMMON: puch {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; We must reset the stack to where it was before the call by undoing its extra stack pop.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl callee_stack4
+; COMMON: sub sp, #16
+
+  call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+define void @fromC_totail_noreservedframe(i32 %len) {
+; COMMON-LABEL: fromC_totail_noreservedframe:
+; COMMON: sub.w sp, sp, r{{.*}}
+
+; COMMON: movs [[TMP:r[0-9]+]], #42
+  ; Note stack is subtracted here to allocate space for arg
+; COMMON: sub.w sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; And here.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; But not restored here because callee_stack8 did that for us.
+; COMMON-NOT: sub sp,
+
+  ; Variable sized allocation prevents reserving frame at start of function so each call must allocate any stack space it needs.
+  %var = alloca i32, i32 %len
+
+  call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  call swifttailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+declare void @Ccallee_stack4([4 x i32], i32)
+
+define swifttailcc void @fromtail_toC() {
+; COMMON-LABEL: fromtail_toC:
+; COMMON: push {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _Ccallee_stack4
+  ; C callees will return with the stack exactly where we left it, so we mustn't try to fix anything.
+; COMMON-NOT: add sp,
+; COMMON-NOT: sub sp,
+; COMMON: str [[TMP]], [sp]{{$}}
+; COMMON: bl _Ccallee_stack4
+; COMMON-NOT: sub sp,
+
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+declare swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure)
+define swiftcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) {
+; CHECK-LABEL: CallSwiftSelf:
+; CHECK: push{{.*}}r10
+  %res = call swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure)
+  ret i8* %res
+}

llvm/test/CodeGen/ARM/swifttailcc-fastisel.ll

+; RUN: llc -mtriple=thumbv7-apple-ios -O0 -fast-isel %s -o - | FileCheck %s
+
+declare swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure)
+
+define swifttailcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) {
+; CHECK-LABEL: CallSwiftSelf:
+; CHECK: bl _SwiftSelf
+; CHECK: pop {r7, pc}
+  %res = call swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself null)
+  ret i8* %res
+}

llvm/test/CodeGen/ARM/tailcc-call.ll

+; RUN: llc -verify-machineinstrs < %s -mtriple=thumbv7k-apple-watchos | FileCheck %s
+
+declare tailcc void @callee_stack0()
+declare tailcc void @callee_stack4([4 x i32], i32)
+declare tailcc void @callee_stack20([4 x i32], [5 x i32])
+declare extern_weak tailcc void @callee_weak()
+
+define tailcc void @caller_to0_from0() nounwind {
+; CHECK-LABEL: _caller_to0_from0:
+
+  tail call tailcc void @callee_stack0()
+  ret void
+; CHECK-NOT: add
+; CHECK-NOT: sub
+; CHECK: b.w _callee_stack0
+}
+
+define tailcc void @caller_to0_from4([4 x i32], i32) {
+; CHECK-LABEL: _caller_to0_from4:
+
+  tail call tailcc void @callee_stack0()
+  ret void
+
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack0
+}
+
+define tailcc void @caller_to4_from0() {
+; Key point is that the "42" should go #16 below incoming stack
+; pointer (we didn't have arg space to reuse).
+  tail call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK-LABEL: _caller_to4_from0:
+; CHECK: sub sp, #16
+; CHECK: movs [[TMP:r[0-9]+]], #42
+; CHECK: str [[TMP]], [sp]
+; CHECK-NOT: add sp
+; CHECK: b.w _callee_stack4
+
+}
+
+define tailcc void @caller_to4_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to4_from4:
+; CHECK-NOT: sub sp
+; Key point is that the "%a" should go where at SP on entry.
+  tail call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{r[0-9]+}}, [sp]
+; CHECK-NOT: add sp
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+define tailcc void @caller_to20_from4([4 x i32], i32 %a) {
+; CHECK-LABEL: _caller_to20_from4:
+; CHECK: sub sp, #16
+
+; Important point is that the call reuses the "dead" argument space
+; above %a on the stack. If it tries to go below incoming-SP then the
+; _callee will not deallocate the space, even in tailcc.
+  tail call tailcc void @callee_stack20([4 x i32] undef, [5 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5])
+
+; CHECK: str {{.*}}, [sp]
+; CHECK: str {{.*}}, [sp, #4]
+; CHECK: str {{.*}}, [sp, #8]
+; CHECK: str {{.*}}, [sp, #12]
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK-NOT: add sp
+; CHECK-NOT: sub sp
+; CHECK: b.w _callee_stack20
+  ret void
+}
+
+
+define tailcc void @caller_to4_from24([4 x i32], i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: _caller_to4_from24:
+
+
+; Key point is that the "%a" should go where at #16 above SP on entry.
+  tail call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+
+; CHECK: str {{.*}}, [sp, #16]
+; CHECK: add sp, #16
+; CHECK-NEXT: b.w _callee_stack4
+}
+
+
+define tailcc void @caller_to20_from20([4 x i32], [5 x i32] %a) {
+; CHECK-LABEL: _caller_to20_from20:
+; CHECK-NOT: add sp,
+; CHECK-NOT: sub sp,
+
+; Here we want to make sure that both loads happen before the stores:
+; otherwise either %a or %b.w will be wrongly clobbered.
+  tail call tailcc void @callee_stack20([4 x i32] undef, [5 x i32] %a)
+  ret void
+
+  ; If these ever get interleaved make sure aliasing slots don't clobber each
+  ; other.
+; CHECK: ldrd {{.*}}, {{.*}}, [sp, #12]
+; CHECK: ldm.w sp,
+; CHECK: stm.w
+; CHECK: strd
+; CHECK-NEXT: b.w _callee_stack20
+}
+
+define tailcc void @disable_tail_calls() nounwind "disable-tail-calls"="true" {
+; CHECK-LABEL: disable_tail_calls:
+
+  tail call tailcc void @callee_stack0()
+  ret void
+
+; CHECK: bl _callee_stack0
+; CHECK: ret
+}
+
+define tailcc void @normal_ret_with_stack([4 x i32], i32 %a) {
+; CHECK: _normal_ret_with_stack:
+; CHECK: add sp, #16
+; CHECK: bx lr
+  ret void
+}
+
+declare { [2 x float] } @get_vec2()
+
+define void @fromC_totail() {
+; COMMON-LABEL: fromC_totail:
+; COMMON: puch {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; We must reset the stack to where it was before the call by undoing its extra stack pop.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl callee_stack4
+; COMMON: sub sp, #16
+
+  call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+define void @fromC_totail_noreservedframe(i32 %len) {
+; COMMON-LABEL: fromC_totail_noreservedframe:
+; COMMON: sub.w sp, sp, r{{.*}}
+
+; COMMON: movs [[TMP:r[0-9]+]], #42
+  ; Note stack is subtracted here to allocate space for arg
+; COMMON: sub.w sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; And here.
+; COMMON: sub sp, #16
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _callee_stack4
+  ; But not restored here because callee_stack8 did that for us.
+; COMMON-NOT: sub sp,
+
+  ; Variable sized allocation prevents reserving frame at start of function so each call must allocate any stack space it needs.
+  %var = alloca i32, i32 %len
+
+  call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  call tailcc void @callee_stack4([4 x i32] undef, i32 42)
+  ret void
+}
+
+declare void @Ccallee_stack4([4 x i32], i32)
+
+define tailcc void @fromtail_toC() {
+; COMMON-LABEL: fromtail_toC:
+; COMMON: push {r4, lr}
+; COMMON: sub sp, #8
+
+; COMMON-NOT: sub sp,
+; COMMON: movs [[TMP:r[0-9]+]], #42
+; COMMON: str [[TMP]], [sp]
+; COMMON: bl _Ccallee_stack4
+  ; C callees will return with the stack exactly where we left it, so we mustn't try to fix anything.
+; COMMON-NOT: add sp,
+; COMMON-NOT: sub sp,
+; COMMON: str [[TMP]], [sp]{{$}}
+; COMMON: bl _Ccallee_stack4
+; COMMON-NOT: sub sp,
+
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  call void @Ccallee_stack4([4 x i32] undef, i32 42)
+  ret void
+}

llvm/test/CodeGen/ARM/v8m-tail-call.ll

; CHECK-NEXT:    b h1
   %2 = tail call i32 @h1(i32 %1)
   ret i32 %2
 }
 
 declare i32 @h2(i32, i32, i32, i32, i32)
 define hidden i32 @f2(i32, i32, i32, i32, i32) {
 ; CHECK-LABEL: f2:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    mov r4, r3
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    ldr r7, [sp, #24]
 ; CHECK-NEXT:    bl g
 ; CHECK-NEXT:    cbz r0, .LBB2_2
 ; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:    str r7, [sp, #24]
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    mov r2, r5
 ; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    ldr r4, [sp, #12]
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    ldr r4, [sp, #16]
 ; CHECK-NEXT:    mov lr, r4
-; CHECK-NEXT:    pop {r4, r5, r6}
+; CHECK-NEXT:    pop {r4, r5, r6, r7}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    b h2
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    mvns r0, r0
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
   %6 = tail call i32 bitcast (i32 (...)* @g to i32 ()*)()
   %7 = icmp eq i32 %6, 0
   br i1 %7, label %10, label %8
 
   %9 = tail call i32 @h2(i32 %6, i32 %1, i32 %2, i32 %3, i32 %4)
   br label %10
 
   %11 = phi i32 [ %9, %8 ], [ -1, %5 ]