[PATCH 2/2] [x86] Add support for "probe-stack"

[Zoxc]

Summary

This is an alternative to http://reviews.llvm.org/D9654

Stack probes are done in an loop in the prologue instead of calling external functions.

Note that when modifying basic blocks in the emitPrologue the epilog will be inserted into the wrong basic block. This is because
the basic blocks to insert epilogs into is calculated before
emitPrologue is called.
I fixed this by adding the following code after the emitPrologue call
in PEI::insertPrologEpilogCode:

RestoreBlocks.clear();
calculateSets(Fn);

This doesn't seem like a very nice solution.

Another thing is that I'm not sure that I modified all the test for stack probing correctly (there were quite a few cases of it).

[Zoxc]

Code generation on Windows is preserved in this version.

[Zoxc]

Rebased tests and added a test for variable length stack allocation.

[AndyAyers]

We have some changes coming in our fork (see https://github.com/Microsoft/llvm/pull/79) where for the inline probes we emit a call to a stub during prolog generation, and then we come back later after epilogs are created to expand the probe sequence inline. We should be able to upstream this soon.

This seems cleaner to us then trying to repair state after prolog generation...

[Zoxc]

Is said state actually used before prolog generation? Just moving the generation to after prolog generation seems like the nicest solution if it's possible.

[Zoxc]

This now uses the same method as the CoreCLR inline stack probes.

[Zoxc]

Rebased patch.

include/llvm/CodeGen/MachineFunction.h

MachineBasicBlock *getBlockNumbered(unsigned N) const {
     assert(N < MBBNumbering.size() && "Illegal block number");
     assert(MBBNumbering[N] && "Block was removed from the machine function!");
     return MBBNumbering[N];
   }
 
   /// Should we be emitting segmented stack stuff for the function
   bool shouldSplitStack();
 
+  /// \brief Should we be probing the stack for the function.
+  ///
+  /// Probing the stack means that we must read or write to the stack on every
+  /// page. This is to ensure that a guard page will be hit and stack overflow
+  /// can be detected. We insert instructions to do this when allocating from
+  /// the stack.
+  bool shouldProbeStack() const;
+
   /// getNumBlockIDs - Return the number of MBB ID's allocated.
   ///
   unsigned getNumBlockIDs() const { return (unsigned)MBBNumbering.size(); }
   
   /// RenumberBlocks - This discards all of the MachineBasicBlock numbers and
   /// recomputes them.  This guarantees that the MBB numbers are sequential,
   /// dense, and match the ordering of the blocks within the function.  If a
   /// specific MachineBasicBlock is specified, only that block and those after

lib/CodeGen/MachineFunction.cpp

getOrCreateJumpTableInfo(unsigned EntryKind) {
   return JumpTableInfo;
 }
 
 /// Should we be emitting segmented stack stuff for the function
 bool MachineFunction::shouldSplitStack() {
   return getFunction()->hasFnAttribute("split-stack");
 }
 
+bool MachineFunction::shouldProbeStack() const {
+  return getFunction()->hasFnAttribute("probe-stack");
+}
+
 /// This discards all of the MachineBasicBlock numbers and recomputes them.
 /// This guarantees that the MBB numbers are sequential, dense, and match the
 /// ordering of the blocks within the function.  If a specific MachineBasicBlock
 /// is specified, only that block and those after it are renumbered.
 void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) {
   if (empty()) { MBBNumbering.clear(); return; }
   MachineFunction::iterator MBBI, E = end();
   if (MBB == nullptr)

lib/CodeGen/PrologEpilogInserter.cpp

 /// prolog and epilog code to the function.
 ///
 void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
   const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering();
 
   // Add prologue to the function...
   TFI.emitPrologue(Fn, *SaveBlock);
 
+  // RestoreBlocks can be clobbered by emitPrologue. Recalculate it.
+  RestoreBlocks.clear();
+  calculateSets(Fn);
+
   // Add epilogue to restore the callee-save registers in each exiting block.
   for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
     TFI.emitEpilogue(Fn, *RestoreBlock);
 
   // Emit additional code that is required to support segmented stacks, if
   // we've been asked for it.  This, when linked with a runtime with support
   // for segmented stacks (libgcc is one), will result in allocating stack
   // space in small chunks instead of one large contiguous block.

lib/Target/X86/X86FrameLowering.h

public:
 
   /// True if the 64-bit frame or stack pointer should be used. True for most
   /// 64-bit targets with the exception of x32. If this is false, 32-bit
   /// instruction operands should be used to manipulate StackPtr and FramePtr.
   bool Uses64BitFramePtr;
 
   unsigned StackPtr;
 
+  void pushRegForStackProbeCall(MachineFunction &MF,
+                                MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator MBBI,
+                                DebugLoc DL,
+                                bool &IsAlive,
+                                unsigned RegType,
+                                uint64_t &NumBytes) const;
+  void popRegForStackProbeCall(MachineFunction &MF,
+                               MachineBasicBlock &MBB,
+                               MachineBasicBlock::iterator MBBI,
+                               DebugLoc DL,
+                               bool &IsAlive,
+                               unsigned RegType,
+                               uint64_t &NumBytes) const;
   /// Emit a call to the target's stack probe function. This is required for all
   /// large stack allocations on Windows. The caller is required to materialize
   /// the number of bytes to probe in RAX/EAX.
-  void emitStackProbeCall(MachineFunction &MF, MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator MBBI, DebugLoc DL) const;
+  MachineInstr *emitStackProbes(MachineFunction &MF, MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator MBBI, DebugLoc DL,
+                                bool InProlog) const;
 
   void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MBBI,
                                  DebugLoc DL) const;
 
   /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
   /// the function.
-  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+  void emitPrologue(MachineFunction &MF, MachineBasicBlock &InMBB) const override;
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   void adjustForSegmentedStacks(MachineFunction &MF,
                                 MachineBasicBlock &PrologueMBB) const override;
 
   void adjustForHiPEPrologue(MachineFunction &MF,
                              MachineBasicBlock &PrologueMBB) const override;
 

lib/Target/X86/X86FrameLowering.cpp

for (; *CS; ++CS)
       if (!Uses.count(*CS))
         return *CS;
   }
   }
 
   return 0;
 }
 
-static bool isEAXLiveIn(MachineFunction &MF) {
+static bool isLiveIn(MachineFunction &MF, unsigned CheckReg) {
+  CheckReg = getX86SubSuperRegister(CheckReg, MVT::i32);
+
   for (MachineRegisterInfo::livein_iterator II = MF.getRegInfo().livein_begin(),
        EE = MF.getRegInfo().livein_end(); II != EE; ++II) {
     unsigned Reg = II->first;
 
-    if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
-        Reg == X86::AH || Reg == X86::AL)
+    if (getX86SubSuperRegisterOrZero(Reg, MVT::i32) == CheckReg)
       return true;
   }
 
   return false;
 }
 
 /// Check whether or not the terminators of \p MBB needs to read EFLAGS.
 static bool terminatorsNeedFlagsAsInput(const MachineBasicBlock &MBB) {
void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
   DebugLoc DL = MBB.findDebugLoc(MBBI);
 
   while (Offset) {
     if (Offset > Chunk) {
       // Rather than emit a long series of instructions for large offsets,
       // load the offset into a register and do one sub/add
       unsigned Reg = 0;
 
-      if (isSub && !isEAXLiveIn(*MBB.getParent()))
+      if (isSub && !isLiveIn(*MBB.getParent(), X86::EAX))
         Reg = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
       else
         Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
 
       if (Reg) {
         unsigned Opc = Is64Bit ? X86::MOV64ri : X86::MOV32ri;
         BuildMI(MBB, MBBI, DL, TII.get(Opc), Reg)
           .addImm(Offset);
for (MachineRegisterInfo::reg_instr_iterator
        ri = MRI.reg_instr_begin(X86::EFLAGS), re = MRI.reg_instr_end();
        ri != re; ++ri)
     if (ri->isCopy())
       return true;
 
   return false;
 }
 
-void X86FrameLowering::emitStackProbeCall(MachineFunction &MF,
+void X86FrameLowering::pushRegForStackProbeCall(MachineFunction &MF,
                                                 MachineBasicBlock &MBB,
                                                 MachineBasicBlock::iterator MBBI,
-                                          DebugLoc DL) const {
-  bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
+                                                DebugLoc DL,
+                                                bool &IsAlive,
+                                                unsigned RegType,
+                                                uint64_t &NumBytes) const {
+  IsAlive = isLiveIn(MF, RegType);
 
-  unsigned CallOp;
-  if (Is64Bit)
-    CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
-  else
-    CallOp = X86::CALLpcrel32;
+  if (!IsAlive) {
+    return;
+  }
 
-  const char *Symbol;
-  if (Is64Bit) {
-    if (STI.isTargetCygMing()) {
-      Symbol = "___chkstk_ms";
-    } else {
-      Symbol = "__chkstk";
-    }
-  } else if (STI.isTargetCygMing())
-    Symbol = "_alloca";
-  else
-    Symbol = "_chkstk";
+  auto Reg = getX86SubSuperRegister(RegType, Is64Bit ? MVT::i64 : MVT::i32);
+
+  // Save the register on the stack.
+  BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
+      .addReg(Reg, RegState::Kill)
+      .setMIFlag(MachineInstr::FrameSetup);
+
+  // Reuse the space from the spill as a stack allocation.
+  NumBytes -= SlotSize;
+}
+
+void X86FrameLowering::popRegForStackProbeCall(MachineFunction &MF,
+                                                MachineBasicBlock &MBB,
+                                                MachineBasicBlock::iterator MBBI,
+                                                DebugLoc DL,
+                                                bool &IsAlive,
+                                                unsigned RegType,
+                                                uint64_t &NumBytes) const {
+  if (!IsAlive) {
+    return;
 
-  MachineInstrBuilder CI;
+  // Restore the register from the stack slot.
 
-  // All current stack probes take AX and SP as input, clobber flags, and
-  // preserve all registers. x86_64 probes leave RSP unmodified.
-  if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
-    // For the large code model, we have to call through a register. Use R11,
-    // as it is scratch in all supported calling conventions.
-    BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11)
-        .addExternalSymbol(Symbol);
-    CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11);
-  } else {
-    CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addExternalSymbol(Symbol);
-  }
-
-  unsigned AX = Is64Bit ? X86::RAX : X86::EAX;
-  unsigned SP = Is64Bit ? X86::RSP : X86::ESP;
-  CI.addReg(AX, RegState::Implicit)
-      .addReg(SP, RegState::Implicit)
-      .addReg(AX, RegState::Define | RegState::Implicit)
-      .addReg(SP, RegState::Define | RegState::Implicit)
-      .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit);
+  auto Reg = getX86SubSuperRegister(RegType, Is64Bit ? MVT::i64 : MVT::i32);
 
-  if (Is64Bit) {
-    // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
-    // themselves. It also does not clobber %rax so we can reuse it when
-    // adjusting %rsp.
-    BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
-        .addReg(X86::RSP)
-        .addReg(X86::RAX);
+  auto MIB = BuildMI(MF, DL,
+                     TII.get(Is64Bit ? X86::MOV64rm : X86::MOV32rm),
+                     Reg);
+  MachineInstr *MI = addRegOffset(MIB, StackPtr, false, NumBytes);
+  MI->setFlag(MachineInstr::FrameSetup);
+  MBB.insert(MBBI, MI);
+
+  NumBytes += SlotSize;
+}
+
+MachineInstr *X86FrameLowering::emitStackProbes(MachineFunction &MF,
+                                                MachineBasicBlock &MBB,
+                                                MachineBasicBlock::iterator MBBI,
+                                                DebugLoc DL,
+                                                bool InProlog) const {
+  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
+  const TargetInstrInfo &TII = *STI.getInstrInfo();
+  const BasicBlock *LLVM_BB = MBB.getBasicBlock();
+
+  // RAX contains the number of bytes of desired stack adjustment.
+  // The handling here assumes this value has already been updated so as to
+  // maintain stack alignment.
+  //
+  // We need to exit with RSP modified by this amount and execute suitable
+  // page touches to notify the OS that we're growing the stack responsibly.
+  // All stack probing must be done without modifying RSP.
+
+  // Set up the new basic blocks
+  MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB);
+
+  MachineFunction::iterator MBBIter = MBB;
+  ++MBBIter;
+
+  MF.insert(MBBIter, LoopMBB);
+  MF.insert(MBBIter, ContinueMBB);
+
+  // Split MBB and move the tail portion down to ContinueMBB.
+  MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI);
+  ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end());
+  ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB);
+
+  const int64_t GuardSize = 0x1000;
+
+  auto RType = Is64Bit ? MVT::i64 : MVT::i32;
+
+  auto InputReg = getX86SubSuperRegister(X86::RAX, RType);
+
+  // Registers we need. For the normal case we use virtual
+  // registers. For the prolog expansion we use RAX, RCX and RDX.
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const TargetRegisterClass *RegClass = Is64Bit ? &X86::GR64RegClass : &X86::GR32RegClass;
+  unsigned
+      SizeReg = InProlog ? InputReg :
+                           MRI.createVirtualRegister(RegClass),
+      StackReg = InProlog ? getX86SubSuperRegister(X86::RCX, RType) :
+                            MRI.createVirtualRegister(RegClass),
+      CountReg = InProlog ? getX86SubSuperRegister(X86::RDX, RType) :
+                            MRI.createVirtualRegister(RegClass),
+      PStackReg = InProlog ? getX86SubSuperRegister(X86::RCX, RType) :
+                            MRI.createVirtualRegister(RegClass),
+      PCountReg = InProlog ? getX86SubSuperRegister(X86::RDX, RType) :
+                            MRI.createVirtualRegister(RegClass),
+      LStackReg = InProlog ? getX86SubSuperRegister(X86::RCX, RType) :
+                            MRI.createVirtualRegister(RegClass),
+      LCountReg = InProlog ? getX86SubSuperRegister(X86::RDX, RType) :
+                            MRI.createVirtualRegister(RegClass);
+
+  auto SPReg = getX86SubSuperRegister(X86::RSP, RType);
+
+  if (InProlog) {
+    ContinueMBB->addLiveIn(InputReg);
+    LoopMBB->addLiveIn(InputReg);
+    LoopMBB->addLiveIn(LCountReg);
+    LoopMBB->addLiveIn(LStackReg);
+
+    for (MachineBasicBlock::livein_iterator i = MBB.livein_begin(),
+                                            e = MBB.livein_end();
+         i != e; i++) {
+      if (!LoopMBB->isLiveIn(*i))
+        LoopMBB->addLiveIn(*i);
+      if (!ContinueMBB->isLiveIn(*i))
+        ContinueMBB->addLiveIn(*i);
+    }
+  } else {
+    BuildMI(&MBB, DL, TII.get(X86::COPY), SizeReg).addReg(InputReg);
+  }
+
+  BuildMI(&MBB, DL, TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr), CountReg).addReg(SizeReg);
+  BuildMI(&MBB, DL, TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr), StackReg).addReg(SPReg);
+
+  if (!InProlog) {
+    BuildMI(LoopMBB, DL, TII.get(X86::PHI), PCountReg)
+        .addReg(CountReg)
+        .addMBB(&MBB)
+        .addReg(LCountReg)
+        .addMBB(LoopMBB);
+    BuildMI(LoopMBB, DL, TII.get(X86::PHI), PStackReg)
+        .addReg(StackReg)
+        .addMBB(&MBB)
+        .addReg(LStackReg)
+        .addMBB(LoopMBB);
+  }
+
+  BuildMI(LoopMBB, DL, TII.get(Is64Bit ? X86::OR64mi8 : X86::OR32mi8))
+    .addReg(PStackReg)
+    .addImm(1)
+    .addReg(0)
+    .addImm(0)
+    .addReg(0)
+    .addImm(0);
+
+  BuildMI(LoopMBB, DL, TII.get(Is64Bit ? X86::SUB64ri32 : X86::SUB32ri),
+          LStackReg)
+      .addReg(PStackReg)
+      .addImm(GuardSize);
+
+  BuildMI(LoopMBB, DL, TII.get(Is64Bit ? X86::SUB64ri32 : X86::SUB32ri),
+          LCountReg)
+      .addReg(PCountReg)
+      .addImm(GuardSize);
+
+  BuildMI(LoopMBB, DL, TII.get(X86::JAE_1)).addMBB(LoopMBB);
+
+  MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
+
+  // Now that the probing is done, add code to continueMBB to update
+  // the stack pointer for real.
+  BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(Is64Bit ? X86::SUB64rr : X86::SUB32rr), SPReg)
+      .addReg(SPReg)
+      .addReg(SizeReg);
+
+  // Add the control flow edges we need.
+  MBB.addSuccessor(LoopMBB);
+  LoopMBB->addSuccessor(ContinueMBB);
+  LoopMBB->addSuccessor(LoopMBB);
+
+  // Mark all the instructions added to the prolog as frame setup.
+  if (InProlog) {
+    for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
+      BeforeMBBI->setFlag(MachineInstr::FrameSetup);
+    }
+    for (MachineBasicBlock::iterator MI = LoopMBB->begin();
+         MI != LoopMBB->end(); ++MI) {
+      MI->setFlag(MachineInstr::FrameSetup);
+    }
+    for (MachineBasicBlock::iterator CMBBI = ContinueMBB->begin();
+         CMBBI != ContinueMBBI; ++CMBBI) {
+      CMBBI->setFlag(MachineInstr::FrameSetup);
     }
   }
 
+  return ContinueMBBI;
+}
+
 static unsigned calculateSetFPREG(uint64_t SPAdjust) {
   // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
   // and might require smaller successive adjustments.
   const uint64_t Win64MaxSEHOffset = 128;
   uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
   // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
   return SEHFrameOffset & -16;
 }
/*
 
   Notes:
   - .seh directives are emitted only for Windows 64 ABI
   - .cfi directives are emitted for all other ABIs
   - for 32-bit code, substitute %e?? registers for %r??
 */
 
 void X86FrameLowering::emitPrologue(MachineFunction &MF,
-                                    MachineBasicBlock &MBB) const {
+                                    MachineBasicBlock &InMBB) const {
   assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
          "MF used frame lowering for wrong subtarget");
-  MachineBasicBlock::iterator MBBI = MBB.begin();
+  MachineBasicBlock *MBB = &InMBB;
+  MachineBasicBlock::iterator MBBI = MBB->begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const Function *Fn = MF.getFunction();
   MachineModuleInfo &MMI = MF.getMMI();
   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
   uint64_t StackSize = MFI->getStackSize();    // Number of bytes to allocate.
   bool HasFP = hasFP(MF);
   bool IsWin64CC = STI.isCallingConvWin64(Fn->getCallingConv());
void X86FrameLowering::emitPrologue(MachineFunction &MF,
   int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
   if (TailCallReturnAddrDelta && IsWin64Prologue)
     report_fatal_error("Can't handle guaranteed tail call under win64 yet");
 
   if (TailCallReturnAddrDelta < 0)
     X86FI->setCalleeSavedFrameSize(
       X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta);
 
-  bool UseStackProbe = (STI.isOSWindows() && !STI.isTargetMachO());
+  bool UseRedZone = false;
+  bool UseStackProbe =
+      (STI.isOSWindows() && !STI.isTargetMachO()) || MF.shouldProbeStack();
 
   // The default stack probe size is 4096 if the function has no stackprobesize
   // attribute.
   unsigned StackProbeSize = 4096;
   if (Fn->hasFnAttribute("stack-probe-size"))
     Fn->getFnAttribute("stack-probe-size")
         .getValueAsString()
         .getAsInteger(0, StackProbeSize);
 
   // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
   // function, and use up to 128 bytes of stack space, don't have a frame
   // pointer, calls, or dynamic alloca then we do not need to adjust the
   // stack pointer (we fit in the Red Zone). We also check that we don't
   // push and pop from the stack.
   if (Is64Bit && !Fn->hasFnAttribute(Attribute::NoRedZone) &&
       !TRI->needsStackRealignment(MF) &&
       !MFI->hasVarSizedObjects() && // No dynamic alloca.
       !MFI->adjustsStack() &&       // No calls.
       !IsWin64CC &&                 // Win64 has no Red Zone
+
+      !(UseStackProbe && StackSize > 128) && // Only use the Red Zone if we can
+                                             // fit the whole stack in it
+                                             // and thus stack probes won't be
+                                             // needed
+
       !usesTheStack(MF) &&          // Don't push and pop.
       !MF.shouldSplitStack()) {     // Regular stack
     uint64_t MinSize = X86FI->getCalleeSavedFrameSize();
     if (HasFP) MinSize += SlotSize;
     StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
     MFI->setStackSize(StackSize);
+    UseRedZone = true;
   }
 
   // Insert stack pointer adjustment for later moving of return addr.  Only
   // applies to tail call optimized functions where the callee argument stack
   // size is bigger than the callers.
   if (TailCallReturnAddrDelta < 0) {
-    BuildStackAdjustment(MBB, MBBI, DL, TailCallReturnAddrDelta,
+    BuildStackAdjustment(*MBB, MBBI, DL, TailCallReturnAddrDelta,
                          /*InEpilogue=*/false)
         .setMIFlag(MachineInstr::FrameSetup);
   }
 
   // Mapping for machine moves:
   //
   //   DST: VirtualFP AND
   //        SRC: VirtualFP              => DW_CFA_def_cfa_offset
if (TRI->needsStackRealignment(MF) && !IsWin64Prologue)
       NumBytes = RoundUpToAlignment(NumBytes, MaxAlign);
 
     // Get the offset of the stack slot for the EBP register, which is
     // guaranteed to be the last slot by processFunctionBeforeFrameFinalized.
     // Update the frame offset adjustment.
     MFI->setOffsetAdjustment(-NumBytes);
 
     // Save EBP/RBP into the appropriate stack slot.
-    BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
+    BuildMI(*MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
       .addReg(MachineFramePtr, RegState::Kill)
       .setMIFlag(MachineInstr::FrameSetup);
 
     if (NeedsDwarfCFI) {
       // Mark the place where EBP/RBP was saved.
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      BuildCFI(MBB, MBBI, DL,
+      BuildCFI(*MBB, MBBI, DL,
                MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth));
 
       // Change the rule for the FramePtr to be an "offset" rule.
       unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
-      BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset(
+      BuildCFI(*MBB, MBBI, DL, MCCFIInstruction::createOffset(
                                   nullptr, DwarfFramePtr, 2 * stackGrowth));
     }
 
     if (NeedsWinCFI) {
-      BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
+      BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
           .addImm(FramePtr)
           .setMIFlag(MachineInstr::FrameSetup);
     }
 
     if (!IsWin64Prologue) {
       // Update EBP with the new base value.
-      BuildMI(MBB, MBBI, DL,
+      BuildMI(*MBB, MBBI, DL,
               TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
               FramePtr)
           .addReg(StackPtr)
           .setMIFlag(MachineInstr::FrameSetup);
     }
 
     if (NeedsDwarfCFI) {
       // Mark effective beginning of when frame pointer becomes valid.
       // Define the current CFA to use the EBP/RBP register.
       unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
-      BuildCFI(MBB, MBBI, DL,
+      BuildCFI(*MBB, MBBI, DL,
                MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr));
     }
 
     // Mark the FramePtr as live-in in every block.
     for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
       I->addLiveIn(MachineFramePtr);
   } else {
     NumBytes = StackSize - X86FI->getCalleeSavedFrameSize();
   }
 
   // Skip the callee-saved push instructions.
   bool PushedRegs = false;
   int StackOffset = 2 * stackGrowth;
 
-  while (MBBI != MBB.end() &&
+  while (MBBI != MBB->end() &&
          MBBI->getFlag(MachineInstr::FrameSetup) &&
          (MBBI->getOpcode() == X86::PUSH32r ||
           MBBI->getOpcode() == X86::PUSH64r)) {
     PushedRegs = true;
     unsigned Reg = MBBI->getOperand(0).getReg();
     ++MBBI;
 
     if (!HasFP && NeedsDwarfCFI) {
       // Mark callee-saved push instruction.
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      BuildCFI(MBB, MBBI, DL,
+      BuildCFI(*MBB, MBBI, DL,
                MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset));
       StackOffset += stackGrowth;
     }
 
     if (NeedsWinCFI) {
-      BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)).addImm(Reg).setMIFlag(
+      BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_PushReg)).addImm(Reg).setMIFlag(
           MachineInstr::FrameSetup);
     }
   }
 
   // Realign stack after we pushed callee-saved registers (so that we'll be
   // able to calculate their offsets from the frame pointer).
   // Don't do this for Win64, it needs to realign the stack after the prologue.
   if (!IsWin64Prologue && TRI->needsStackRealignment(MF)) {
     assert(HasFP && "There should be a frame pointer if stack is realigned.");
-    BuildStackAlignAND(MBB, MBBI, DL, MaxAlign);
+    BuildStackAlignAND(*MBB, MBBI, DL, MaxAlign);
   }
 
   // If there is an SUB32ri of ESP immediately before this instruction, merge
   // the two. This can be the case when tail call elimination is enabled and
   // the callee has more arguments then the caller.
-  NumBytes -= mergeSPUpdates(MBB, MBBI, true);
+  NumBytes -= mergeSPUpdates(*MBB, MBBI, true);
 
   // Adjust stack pointer: ESP -= numbytes.
 
   // Windows and cygwin/mingw require a prologue helper routine when allocating
   // more than 4K bytes on the stack.  Windows uses __chkstk and cygwin/mingw
   // uses __alloca.  __alloca and the 32-bit version of __chkstk will probe the
   // stack and adjust the stack pointer in one go.  The 64-bit version of
   // __chkstk is only responsible for probing the stack.  The 64-bit prologue is
   // responsible for adjusting the stack pointer.  Touching the stack at 4K
   // increments is necessary to ensure that the guard pages used by the OS
   // virtual memory manager are allocated in correct sequence.
   uint64_t AlignedNumBytes = NumBytes;
   if (IsWin64Prologue && TRI->needsStackRealignment(MF))
     AlignedNumBytes = RoundUpToAlignment(AlignedNumBytes, MaxAlign);
   if (AlignedNumBytes >= StackProbeSize && UseStackProbe) {
-    // Check whether EAX is livein for this function.
-    bool isEAXAlive = isEAXLiveIn(MF);
+    assert(!UseRedZone && "The Red Zone is not accounted for in stack probes");
+
+    uint64_t PageSize = 0x1000;
 
-    if (isEAXAlive) {
-      // Sanity check that EAX is not livein for this function.
-      // It should not be, so throw an assert.
-      assert(!Is64Bit && "EAX is livein in x64 case!");
-
-      // Save EAX
-      BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
-        .addReg(X86::EAX, RegState::Kill)
+    // If we only need to probe 5 pages or below, we just emit instructions to
+    // do that instead of calling the function. This is just what the loop in
+    // the called function would do. 5 probes is what GCC will do before using
+    // a loop.
+    if (NumBytes <= 5 * PageSize) {
+      for (uint64_t i = 0; i < NumBytes / PageSize; ++i) {
+        BuildMI(*MBB, MBBI, DL, TII.get(Is64Bit ? X86::OR64mi8 : X86::OR32mi8))
+          .addReg(StackPtr)
+          .addImm(1)
+          .addReg(0)
+          .addImm(- (i + 1) * PageSize)
+          .addReg(0)
+          .addImm(0)
           .setMIFlag(MachineInstr::FrameSetup);
       }
 
+      BuildMI(*MBB, MBBI, DL, TII.get(Is64Bit ? X86::SUB64ri32 : X86::SUB32ri),
+              StackPtr)
+         .addReg(StackPtr)
+         .addImm(NumBytes)
+         .setMIFlag(MachineInstr::FrameSetup);
+
+    } else {
+      // We spill the registers we need for the stack probe loop.
+
+      bool RAXAlive, RCXAlive, RDXAlive;
+
+      // TODO: Push the registers if they are callee-saved.
+
+      pushRegForStackProbeCall(MF, *MBB, MBBI, DL, RAXAlive, X86::RAX, NumBytes);
+      pushRegForStackProbeCall(MF, *MBB, MBBI, DL, RCXAlive, X86::RCX, NumBytes);
+      pushRegForStackProbeCall(MF, *MBB, MBBI, DL, RDXAlive, X86::RDX, NumBytes);
+
       if (Is64Bit) {
         // Handle the 64-bit Windows ABI case where we need to call __chkstk.
         // Function prologue is responsible for adjusting the stack pointer.
         if (isUInt<32>(NumBytes)) {
-        BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
+          BuildMI(*MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
               .addImm(NumBytes)
               .setMIFlag(MachineInstr::FrameSetup);
         } else if (isInt<32>(NumBytes)) {
-        BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX)
+          BuildMI(*MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX)
               .addImm(NumBytes)
               .setMIFlag(MachineInstr::FrameSetup);
         } else {
-        BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX)
+          BuildMI(*MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX)
               .addImm(NumBytes)
               .setMIFlag(MachineInstr::FrameSetup);
         }
       } else {
-      // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
-      // We'll also use 4 already allocated bytes for EAX.
-      BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
-        .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
+        BuildMI(*MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
+            .addImm(NumBytes)
             .setMIFlag(MachineInstr::FrameSetup);
       }
 
       // Save a pointer to the MI where we set AX.
       MachineBasicBlock::iterator SetRAX = MBBI;
       --SetRAX;
 
-    // Call __chkstk, __chkstk_ms, or __alloca.
-    emitStackProbeCall(MF, MBB, MBBI, DL);
-
-    // Apply the frame setup flag to all inserted instrs.
-    for (; SetRAX != MBBI; ++SetRAX)
-      SetRAX->setFlag(MachineInstr::FrameSetup);
-
-    if (isEAXAlive) {
-      // Restore EAX
-      MachineInstr *MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm),
-                                              X86::EAX),
-                                      StackPtr, false, NumBytes - 4);
-      MI->setFlag(MachineInstr::FrameSetup);
-      MBB.insert(MBBI, MI);
+      // Emit the stack probes.
+      MachineInstr *NextInstr = emitStackProbes(MF, *MBB, MBBI, DL, true);
+      MBBI = NextInstr;
+      MBB = NextInstr->getParent();
+
+      // Now we restore the spilled registers from the stack
+
+      popRegForStackProbeCall(MF, *MBB, MBBI, DL, RDXAlive, X86::RDX, NumBytes);
+      popRegForStackProbeCall(MF, *MBB, MBBI, DL, RCXAlive, X86::RCX, NumBytes);
+      popRegForStackProbeCall(MF, *MBB, MBBI, DL, RAXAlive, X86::RAX, NumBytes);
     }
   } else if (NumBytes) {
-    emitSPUpdate(MBB, MBBI, -(int64_t)NumBytes, /*InEpilogue=*/false);
+    emitSPUpdate(*MBB, MBBI, -(int64_t)NumBytes, /*InEpilogue=*/false);
   }
 
   if (NeedsWinCFI && NumBytes)
-    BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
+    BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
         .addImm(NumBytes)
         .setMIFlag(MachineInstr::FrameSetup);
 
   int SEHFrameOffset = 0;
   if (IsWin64Prologue && HasFP) {
     SEHFrameOffset = calculateSetFPREG(NumBytes);
     if (SEHFrameOffset)
-      addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
+      addRegOffset(BuildMI(*MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
                    StackPtr, false, SEHFrameOffset);
     else
-      BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr).addReg(StackPtr);
+      BuildMI(*MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr).addReg(StackPtr);
 
     if (NeedsWinCFI)
-      BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
+      BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
           .addImm(FramePtr)
           .addImm(SEHFrameOffset)
           .setMIFlag(MachineInstr::FrameSetup);
   }
 
-  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
+  while (MBBI != MBB->end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
     const MachineInstr *FrameInstr = &*MBBI;
     ++MBBI;
 
     if (NeedsWinCFI) {
       int FI;
       if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
         if (X86::FR64RegClass.contains(Reg)) {
           unsigned IgnoredFrameReg;
           int Offset = getFrameIndexReference(MF, FI, IgnoredFrameReg);
           Offset += SEHFrameOffset;
 
-          BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
+          BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
               .addImm(Reg)
               .addImm(Offset)
               .setMIFlag(MachineInstr::FrameSetup);
         }
       }
     }
   }
 
   if (NeedsWinCFI)
-    BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
+    BuildMI(*MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
         .setMIFlag(MachineInstr::FrameSetup);
 
   // Realign stack after we spilled callee-saved registers (so that we'll be
   // able to calculate their offsets from the frame pointer).
   // Win64 requires aligning the stack after the prologue.
   if (IsWin64Prologue && TRI->needsStackRealignment(MF)) {
     assert(HasFP && "There should be a frame pointer if stack is realigned.");
-    BuildStackAlignAND(MBB, MBBI, DL, MaxAlign);
+    BuildStackAlignAND(*MBB, MBBI, DL, MaxAlign);
   }
 
   // If we need a base pointer, set it up here. It's whatever the value
   // of the stack pointer is at this point. Any variable size objects
   // will be allocated after this, so we can still use the base pointer
   // to reference locals.
   if (TRI->hasBasePointer(MF)) {
     // Update the base pointer with the current stack pointer.
     unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
-    BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
+    BuildMI(*MBB, MBBI, DL, TII.get(Opc), BasePtr)
       .addReg(StackPtr)
       .setMIFlag(MachineInstr::FrameSetup);
     if (X86FI->getRestoreBasePointer()) {
       // Stash value of base pointer.  Saving RSP instead of EBP shortens
       // dependence chain. Used by SjLj EH.
       unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
-      addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)),
+      addRegOffset(BuildMI(*MBB, MBBI, DL, TII.get(Opm)),
                    FramePtr, true, X86FI->getRestoreBasePointerOffset())
         .addReg(StackPtr)
         .setMIFlag(MachineInstr::FrameSetup);
     }
 
     if (X86FI->getHasSEHFramePtrSave()) {
       // Stash the value of the frame pointer relative to the base pointer for
       // Win32 EH. This supports Win32 EH, which does the inverse of the above:
       // it recovers the frame pointer from the base pointer rather than the
       // other way around.
       unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
       unsigned IgnoredFrameReg;
-      addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), BasePtr, true,
+      addRegOffset(BuildMI(*MBB, MBBI, DL, TII.get(Opm)), BasePtr, true,
                    getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(),
                                           IgnoredFrameReg))
           .addReg(FramePtr)
           .setMIFlag(MachineInstr::FrameSetup);
     }
   }
 
   if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
     // Mark end of stack pointer adjustment.
     if (!HasFP && NumBytes) {
       // Define the current CFA rule to use the provided offset.
       assert(StackSize);
-      BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset(
+      BuildCFI(*MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset(
                                   nullptr, -StackSize + stackGrowth));
     }
 
     // Emit DWARF info specifying the offsets of the callee-saved registers.
     if (PushedRegs)
-      emitCalleeSavedFrameMoves(MBB, MBBI, DL);
+      emitCalleeSavedFrameMoves(*MBB, MBBI, DL);
   }
 }
 
 bool X86FrameLowering::canUseLEAForSPInEpilogue(
     const MachineFunction &MF) const {
   // We can't use LEA instructions for adjusting the stack pointer if this is a
   // leaf function in the Win64 ABI.  Only ADD instructions may be used to
   // deallocate the stack.

lib/Target/X86/X86ISelLowering.cpp

 // that the guard pages used by the OS virtual memory manager are allocated in
 // correct sequence.
 SDValue
 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                            SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   bool SplitStack = MF.shouldSplitStack();
   bool Lower = (Subtarget->isOSWindows() && !Subtarget->isTargetMachO()) ||
-               SplitStack;
+               SplitStack || MF.shouldProbeStack();
   SDLoc dl(Op);
 
   if (!Lower) {
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     SDNode* Node = Op.getNode();
 
     unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
     assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
if (SplitStack) {
     SDValue Ops1[2] = { Value, Chain };
     return DAG.getMergeValues(Ops1, dl);
   } else {
     SDValue Flag;
     const unsigned Reg = (Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX);
 
     Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag);
     Flag = Chain.getValue(1);
+
     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
     Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
 
     const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
     unsigned SPReg = RegInfo->getStackRegister();
     SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
     Chain = SP.getValue(1);
X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI,
   // And we're done.
   return continueMBB;
 }
 
 MachineBasicBlock *
 X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI,
                                         MachineBasicBlock *BB) const {
   DebugLoc DL = MI->getDebugLoc();
-
-  assert(!Subtarget->isTargetMachO());
-
-  Subtarget->getFrameLowering()->emitStackProbeCall(*BB->getParent(), *BB, MI,
-                                                    DL);
-
+  MachineInstr *ResumeMI =
+    Subtarget->getFrameLowering()->emitStackProbes(*BB->getParent(), *BB, MI,
+                                                    DL, false);
+  MachineBasicBlock *ResumeBB = ResumeMI->getParent();
   MI->eraseFromParent();   // The pseudo instruction is gone now.
-  return BB;
+  return ResumeBB;
 }
 
 MachineBasicBlock *
 X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
                                       MachineBasicBlock *BB) const {
   // This is pretty easy.  We're taking the value that we received from
   // our load from the relocation, sticking it in either RDI (x86-64)
   // or EAX and doing an indirect call.  The return value will then

test/CodeGen/X86/dynamic-alloca-in-entry.ll

 ; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 ; Allocas with unknown size in the entry block are dynamic.
 define void @foo(i32 %n) {
   %m = alloca i32, i32 %n
   ret void
 }
 ; CHECK-LABEL: _foo:
-; CHECK: calll __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 ; CHECK: retl
 
 ; Use of inalloca implies that that the alloca is not static.
 define void @bar() {
   %m = alloca inalloca i32
   ret void
 }
 ; CHECK-LABEL: _bar:
-; CHECK: calll __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 ; CHECK: retl

test/CodeGen/X86/inalloca-ctor.ll

 ; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 %Foo = type { i32, i32 }
 
 %frame = type { %Foo, i32, %Foo }
 
 declare void @f(%frame* inalloca %a)
 
 declare void @Foo_ctor(%Foo* %this)
 
 define void @g() {
 entry:
   %args = alloca inalloca %frame
   %c = getelementptr %frame, %frame* %args, i32 0, i32 2
 ; CHECK: movl    $20, %eax
-; CHECK: calll   __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 ; CHECK: movl %esp,
   call void @Foo_ctor(%Foo* %c)
 ; CHECK: leal 12(%{{.*}}),
 ; CHECK-NEXT: pushl
 ; CHECK-NEXT: calll _Foo_ctor
 ; CHECK: addl $4, %esp
   %b = getelementptr %frame, %frame* %args, i32 0, i32 1
   store i32 42, i32* %b

test/CodeGen/X86/inalloca-invoke.ll

define i32 @main() personality i32 (...)* @pers {
   br label %blah
 
 blah:
   %inalloca.save = call i8* @llvm.stacksave()
   %rev_args = alloca inalloca %frame.reverse, align 4
   %beg = getelementptr %frame.reverse, %frame.reverse* %rev_args, i32 0, i32 0
   %end = getelementptr %frame.reverse, %frame.reverse* %rev_args, i32 0, i32 1
 
-; CHECK:  calll   __chkstk
+; CHECK:  or{{.}}     $0, {{.*}}
 ; CHECK:  movl %esp, %[[beg:[^ ]*]]
 ; CHECK:  leal 12(%[[beg]]), %[[end:[^ ]*]]
 
   call void @begin(%Iter* sret %temp.lvalue)
 ; CHECK:  calll _begin
 
   invoke void @plus(%Iter* sret %end, %Iter* %temp.lvalue, i32 4)
           to label %invoke.cont unwind label %lpad

test/CodeGen/X86/inalloca-stdcall.ll

 ; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 %Foo = type { i32, i32 }
 
 declare x86_stdcallcc void @f(%Foo* inalloca %a)
 declare x86_stdcallcc void @i(i32 %a)
 
 define void @g() {
 ; CHECK-LABEL: _g:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
-; CHECK: calll   __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
 ; CHECK: movl    $13, (%esp)
 ; CHECK: movl    $42, 4(%esp)
   call x86_stdcallcc void @f(%Foo* inalloca %b)
 ; CHECK: calll   _f@8
 ; CHECK-NOT: %esp
 ; CHECK: pushl
 ; CHECK: calll   _i@4
   call x86_stdcallcc void @i(i32 0)
   ret void
 }

test/CodeGen/X86/inalloca.ll

 ; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 %Foo = type { i32, i32 }
 
 declare void @f(%Foo* inalloca %b)
 
 define void @a() {
 ; CHECK-LABEL: _a:
 entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
-; CHECK: calll   __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
 ; CHECK: movl    $13, (%esp)
 ; CHECK: movl    $42, 4(%esp)
   call void @f(%Foo* inalloca %b)
 ; CHECK: calll   _f
   ret void
 }
 
 declare void @inreg_with_inalloca(i32 inreg %a, %Foo* inalloca %b)
 
 define void @b() {
 ; CHECK-LABEL: _b:
 entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
-; CHECK: calll   __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
 ; CHECK: movl    $13, (%esp)
 ; CHECK: movl    $42, 4(%esp)
   call void @inreg_with_inalloca(i32 inreg 1, %Foo* inalloca %b)
 ; CHECK: movl    $1, %eax
 ; CHECK: calll   _inreg_with_inalloca
   ret void
 }
 
 declare x86_thiscallcc void @thiscall_with_inalloca(i8* %a, %Foo* inalloca %b)
 
 define void @c() {
 ; CHECK-LABEL: _c:
 entry:
   %b = alloca inalloca %Foo
 ; CHECK: movl    $8, %eax
-; CHECK: calll   __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
   %f1 = getelementptr %Foo, %Foo* %b, i32 0, i32 0
   %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1
   store i32 13, i32* %f1
   store i32 42, i32* %f2
 ; CHECK-DAG: movl    $13, (%esp)
 ; CHECK-DAG: movl    $42, 4(%esp)
   call x86_thiscallcc void @thiscall_with_inalloca(i8* null, %Foo* inalloca %b)
 ; CHECK-DAG: xorl    %ecx, %ecx
 ; CHECK: calll   _thiscall_with_inalloca
   ret void
 }

test/CodeGen/X86/mem-intrin-base-reg.ll

spill_vectors:
   ret i32 %r
 }
 
 ; CHECK-LABEL: _memcpy_vla_vector:
 ; CHECK: andl $-16, %esp
 ; CHECK: movl %esp, %esi
 ; CHECK: pushl $128
 ; CHECK: calll _memcpy
-; CHECK: calll __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 
 ; stosd doesn't clobber esi, so we can use it.
 
 define i32 @memset_vla_vector(<4 x i32>* %vp0, i8* %a, i32 %n, i1 zeroext %cond) {
   call void @llvm.memset.p0i8.i32(i8* %a, i8 42, i32 128, i32 4, i1 false)
   br i1 %cond, label %spill_vectors, label %no_vectors
 
 no_vectors:

test/CodeGen/X86/mingw-alloca.ll

 ; RUN: llc < %s -mtriple=i386-pc-mingw32      | FileCheck %s -check-prefix=COFF
 ; RUN: llc < %s -mtriple=i386-pc-mingw32-elf  | FileCheck %s -check-prefix=ELF
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
 
 define void @foo1(i32 %N) nounwind {
 entry:
 ; COFF: _foo1:
-; COFF: calll __alloca
+; COFF: or{{.}}     $0, {{.*}}
 ; ELF: foo1:
-; ELF: calll _alloca
+; ELF: or{{.}}     $0, {{.*}}
   %tmp14 = alloca i32, i32 %N   ; <i32*> [#uses=1]
   call void @bar1( i32* %tmp14 )
   ret void
 }
 
 declare void @bar1(i32*)
 
 define void @foo2(i32 inreg  %N) nounwind {
 entry:
 ; COFF: _foo2:
 ; COFF: andl $-16, %esp
-; COFF: pushl %eax
-; COFF: calll __alloca
-; COFF: movl  8028(%esp), %eax
+; COFF: or{{.}}     $0, {{.*}}
 ; ELF: foo2:
 ; ELF: andl $-16, %esp
-; ELF: pushl %eax
-; ELF: calll _alloca
-; ELF: movl 8028(%esp), %eax
+; ELF: or{{.}}     $0, {{.*}}
   %A2 = alloca [2000 x i32], align 16    ; <[2000 x i32]*> [#uses=1]
   %A2.sub = getelementptr [2000 x i32], [2000 x i32]* %A2, i32 0, i32 0    ; <i32*> [#uses=1]
   call void @bar2( i32* %A2.sub, i32 %N )
   ret void
 }
 
 declare void @bar2(i32*, i32)

test/CodeGen/X86/movtopush.ll

 define void @test1c() minsize {
 entry:
   call void @good(i32 1, i32 2, i32 3, i32 4)
   ret void
 }
 
 ; If we have a reserved frame, we should have pushes
 ; NORMAL-LABEL: test2:
-; NORMAL-NOT: subl {{.*}} %esp
 ; NORMAL: pushl   $4
 ; NORMAL-NEXT: pushl   $3
 ; NORMAL-NEXT: pushl   $2
 ; NORMAL-NEXT: pushl   $1
 ; NORMAL-NEXT: call
 define void @test2(i32 %k) {
 entry:
   %a = alloca i32, i32 %k

test/CodeGen/X86/pr17631.ll

allocas:
 
   %A1 = bitcast <8 x i32> %A to <8 x float>
   %A2 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %A1)
   ret i32 %A2
 }
 
 ; CHECK: equal
 ; CHECK-NOT: vzeroupper
-; CHECK: _chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 ; CHECK: ret
 
 define <8 x float> @foo(<8 x float> %y, i64* %p, double %x) {
   %i = fptoui double %x to i64
   store i64 %i, i64* %p
   %ret = fadd <8 x float> %y, %y
   ret <8 x float> %ret
 }
 
 ; CHECK: foo
 ; CHECK-NOT: vzeroupper
 ; CHECK: {{cvtt|fist}}
 ; CHECK: ret

test/CodeGen/X86/stack-probe-size.ll

 ; This test is attempting to detect that the compiler correctly generates stack
 ; probe calls when the size of the local variables exceeds the specified stack
 ; probe size.
 ;
 ; Testing the default value of 4096 bytes makes sense, because the default
 ; stack probe size equals the page size (4096 bytes for all x86 targets), and
 ; this is unlikely to change in the future.
 ;
 ; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
 ; RUN: llc -mtriple=i686-windows-coreclr < %s | FileCheck %s
 
 target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
 
-define i32 @test1() "stack-probe-size"="0" {
-  %buffer = alloca [4095 x i8]
-
-  ret i32 0
-
-; CHECK-LABEL: _test1:
-; CHECK-NOT: subl $4095, %esp
-; CHECK: movl $4095, %eax
-; CHECK: calll __chkstk
-}
-
 define i32 @test2() {
   %buffer = alloca [4095 x i8]
 
   ret i32 0
 
 ; CHECK-LABEL: _test2:
 ; CHECK-NOT: movl $4095, %eax
 ; CHECK: subl $4095, %esp
-; CHECK-NOT: calll __chkstk
+; CHECK-NOT: or{{.}}     $0, {{.*}}
 }
 
 define i32 @test3() "stack-probe-size"="8192" {
   %buffer = alloca [4095 x i8]
 
   ret i32 0
 
 ; CHECK-LABEL: _test3:
 ; CHECK-NOT: movl $4095, %eax
 ; CHECK: subl $4095, %esp
-; CHECK-NOT: calll __chkstk
+; CHECK-NOT: or{{.}}     $0, {{.*}}
 }
 
 define i32 @test4() "stack-probe-size"="0" {
   %buffer = alloca [4096 x i8]
 
   ret i32 0
 
 ; CHECK-LABEL: _test4:
 ; CHECK-NOT: subl $4096, %esp
-; CHECK: movl $4096, %eax
-; CHECK: calll __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 }
 
 define i32 @test5() {
   %buffer = alloca [4096 x i8]
 
   ret i32 0
 
 ; CHECK-LABEL: _test5:
 ; CHECK-NOT: subl $4096, %esp
-; CHECK: movl $4096, %eax
-; CHECK: calll __chkstk
+; CHECK: or{{.}}     $0, {{.*}}
 }
 
 define i32 @test6() "stack-probe-size"="8192" {
   %buffer = alloca [4096 x i8]
 
   ret i32 0
 
 ; CGECK-LABEL: _test6:
 ; CGECK-NOT: movl $4096, %eax
 ; CGECK: subl $4096, %esp
-; CGECK-NOT: calll __chkstk
+; CGECK-NOT: or{{.}}     $0, {{.*}}
 }

test/CodeGen/X86/stack-probes.ll

+; RUN: llc -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck --check-prefix=X86-LINUX %s
+; RUN: llc -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=X64-LINUX %s
+
+declare void @use([40096 x i8]*)
+
+; Ensure calls to __probestack occur for large stack frames
+define void @test() "probe-stack" {
+  %array = alloca [40096 x i8]
+  call void @use([40096 x i8]* %array)
+  ret void
+
+; X86-LINUX-LABEL: test:
+; X86-LINUX:       movl    $40108, %eax            # imm = 0x9CAC
+; X86-LINUX-NEXT:  movl    %eax, %edx
+; X86-LINUX-NEXT:  movl    %esp, %ecx
+; X86-LINUX-LABEL: .LBB0_1
+; X86-LINUX-NEXT:  orl     $0, (%ecx)
+; X86-LINUX-NEXT:  subl    $4096, %ecx
+; X86-LINUX-NEXT:  subl    $4096, %edx
+; X86-LINUX-NEXT:  jae     .LBB0_1
+; X86-LINUX:       subl    %eax, %esp
+; X86-LINUX:       addl    $40108, %esp            # imm = 0x9CAC
+
+; X64-LINUX-LABEL: test:
+; X64-LINUX:       movl    $40104, %eax            # imm = 0x9CA8
+; X64-LINUX-NEXT:  movq    %rax, %rdx
+; X64-LINUX-NEXT:  movq    %rsp, %rcx
+; X64-LINUX-LABEL: .LBB0_1
+; X64-LINUX-NEXT:  orq     $0, (%rcx)
+; X64-LINUX-NEXT:  subq    $4096, %rcx
+; X64-LINUX-NEXT:  subq    $4096, %rdx
+; X64-LINUX-NEXT:  jae     .LBB0_1
+; X64-LINUX:       subq    %rax, %rsp
+; X64-LINUX:       addq    $40104, %rsp            # imm = 0x9CA8
+
+}
+
+declare void @useFast([4096 x i8]*)
+
+; Ensure the stack is probed for medium stack frames
+define void @testFast() "probe-stack" {
+  %array = alloca [4096 x i8]
+  call void @useFast([4096 x i8]* %array)
+  ret void
+
+; X86-LINUX-LABEL: testFast:
+; X86-LINUX:       orl     $0, -4096(%esp)
+; X86-LINUX-NEXT:  subl    $4108, %esp             # imm = 0x100C
+
+; X64-LINUX-LABEL: testFast:
+; X64-LINUX:       orq     $0, -4096(%rsp)
+; X64-LINUX-NEXT:  subq    $4104, %rsp             # imm = 0x1008
+
+}

test/CodeGen/X86/win64_alloca_dynalloca.ll

 ; RUN: llc < %s -mcpu=generic -enable-misched=false -mtriple=x86_64-mingw32     | FileCheck %s -check-prefix=M64
 ; RUN: llc < %s -mcpu=generic -enable-misched=false -mtriple=x86_64-win32       | FileCheck %s -check-prefix=W64
-; RUN: llc < %s -mcpu=generic -enable-misched=false -mtriple=x86_64-win32 -code-model=large | FileCheck %s -check-prefix=L64
 ; RUN: llc < %s -mcpu=generic -enable-misched=false -mtriple=x86_64-win32-macho | FileCheck %s -check-prefix=EFI
 ; PR8777
 ; PR8778
 
 define i64 @unaligned(i64 %n, i64 %x) nounwind {
 ; M64-LABEL: unaligned:
 ; W64-LABEL: unaligned:
 ; EFI-LABEL: unaligned:
 entry:
 
   %buf0 = alloca i8, i64 4096, align 1
 
 ; ___chkstk_ms does not adjust %rsp.
-; M64:       $4096, %eax
-; M64: callq ___chkstk_ms
-; M64: subq  %rax, %rsp
+; M64: or{{.}}     $0, {{.*}}
+; M64: subq  {{.*}}, %rsp
 ; M64: leaq 128(%rsp), %rbp
 
 ; __chkstk does not adjust %rsp.
-; W64:       $4096, %eax
-; W64: callq __chkstk
-; W64: subq  %rax, %rsp
+; W64: or{{.}}     $0, {{.*}}
+; W64: subq  {{.*}}, %rsp
 ; W64: leaq 128(%rsp), %rbp
 
-; Use %r11 for the large model.
-; L64:       $4096, %eax
-; L64: movabsq $__chkstk, %r11
-; L64: callq *%r11
-; L64: subq  %rax, %rsp
-
 ; Freestanding
 ; EFI:       $[[B0OFS:4096|4104]], %rsp
 ; EFI-NOT:   call
 
   %buf1 = alloca i8, i64 %n, align 1
 
 ; M64: leaq  15(%{{.*}}), %rax
 ; M64: andq  $-16, %rax
-; M64: callq ___chkstk_ms
-; M64: subq  %rax, %rsp
+; M64: or{{.}}     $0, {{.*}}
+; M64: subq  {{.*}}, %rsp
 ; M64: movq  %rsp, %rax
 
 ; W64: leaq  15(%{{.*}}), %rax
 ; W64: andq  $-16, %rax
-; W64: callq __chkstk
-; W64: subq  %rax, %rsp
+; W64: or{{.}}     $0, {{.*}}
+; W64: subq  {{.*}}, %rsp
 ; W64: movq  %rsp, %rax
 
-; L64: leaq  15(%{{.*}}), %rax
-; L64: andq  $-16, %rax
-; L64: movabsq $__chkstk, %r11
-; L64: callq *%r11
-; L64: subq  %rax, %rsp
-; L64: movq  %rsp, %rax
-
 ; EFI: leaq  15(%{{.*}}), [[R1:%r.*]]
 ; EFI: andq  $-16, [[R1]]
 ; EFI: movq  %rsp, [[R64:%r.*]]
 ; EFI: subq  [[R1]], [[R64]]
 ; EFI: movq  [[R64]], %rsp
 
   %r = call i64 @bar(i64 %n, i64 %x, i64 %n, i8* %buf0, i8* %buf1) nounwind
 

 ; W64-LABEL: aligned:
 ; EFI-LABEL: aligned:
 entry:
 
   %buf1 = alloca i8, i64 %n, align 128
 
 ; M64: leaq  15(%{{.*}}), %rax
 ; M64: andq  $-16, %rax
-; M64: callq ___chkstk_ms
-; M64: subq  %rax, %rsp
+; M64: or{{.}}     $0, {{.*}}
+; M64: subq  {{.*}}, %rsp
 ; M64: movq  %rsp, [[R2:%r.*]]
 ; M64: andq  $-128, [[R2]]
 ; M64: movq  [[R2]], %rsp
 
 ; W64: leaq  15(%{{.*}}), %rax
 ; W64: andq  $-16, %rax
-; W64: callq __chkstk
-; W64: subq  %rax, %rsp
+; W64: or{{.}}     $0, {{.*}}
+; W64: subq  {{.*}}, %rsp
 ; W64: movq  %rsp, [[R2:%r.*]]
 ; W64: andq  $-128, [[R2]]
 ; W64: movq  [[R2]], %rsp
 
 ; EFI: leaq  15(%{{.*}}), [[R1:%r.*]]
 ; EFI: andq  $-16, [[R1]]
 ; EFI: movq  %rsp, [[R64:%r.*]]
 ; EFI: subq  [[R1]], [[R64]]

test/CodeGen/X86/win64_eh.ll

 ; Checks a stack allocation requiring call to __chkstk/___chkstk_ms
 define void @foo2() uwtable {
 entry:
   %baz = alloca [4000 x i16], align 2
   ret void
 }
 ; WIN64-LABEL: foo2:
 ; WIN64: .seh_proc foo2
-; WIN64: movl $8000, %eax
-; WIN64: callq {{__chkstk|___chkstk_ms}}
-; WIN64: subq %rax, %rsp
+; WIN64: or{{.}}     $0, {{.*}}
 ; WIN64: .seh_stackalloc 8000
 ; WIN64: .seh_endprologue
 ; WIN64: addq $8000, %rsp
 ; WIN64: ret
 ; WIN64: .seh_endproc
 
 
 ; Checks stack push

test/CodeGen/X86/win64_frame.ll

define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"="true" {
   ; CHECK:        andq    $-64, %rsp
   ; CHECK:        movq    %rsp, %rbx
 
   alloca i32, i32 %a
   ; CHECK:        movl    %ecx, %eax
   ; CHECK:        leaq    15(,%rax,4), %rcx
   ; CHECK:        movabsq $34359738352, %rax
   ; CHECK:        andq    %rcx, %rax
-  ; CHECK:        callq   __chkstk
+  ; CHECK:        or{{.}}     $0, {{.*}}
   ; CHECK:        subq    %rax, %rsp
 
   %gep = getelementptr [300 x i8], [300 x i8]* %alloca, i32 0, i32 0
   call void @external(i8* %gep)
   ; CHECK:        subq    $32, %rsp
   ; CHECK:        leaq    (%rbx), %rcx
   ; CHECK:        callq   external
   ; CHECK:        addq    $32, %rsp

test/CodeGen/X86/win_chkstk.ll

-; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s -check-prefix=WIN_X32
-; RUN: llc < %s -mtriple=x86_64-pc-win32 | FileCheck %s -check-prefix=WIN_X64
-; RUN: llc < %s -mtriple=x86_64-pc-win32 -code-model=large | FileCheck %s -check-prefix=WIN64_LARGE
-; RUN: llc < %s -mtriple=i686-pc-mingw32 | FileCheck %s -check-prefix=MINGW_X32
-; RUN: llc < %s -mtriple=x86_64-pc-mingw32 | FileCheck %s -check-prefix=MINGW_X64
+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s -check-prefix=WIN
+; RUN: llc < %s -mtriple=x86_64-pc-win32 | FileCheck %s -check-prefix=WIN
+; RUN: llc < %s -mtriple=i686-pc-mingw32 | FileCheck %s -check-prefix=WIN
+; RUN: llc < %s -mtriple=x86_64-pc-mingw32 | FileCheck %s -check-prefix=WIN
 ; RUN: llc < %s -mtriple=i386-pc-linux | FileCheck %s -check-prefix=LINUX
 ; RUN: llc < %s -mtriple=x86_64-pc-win32-macho | FileCheck %s -check-prefix=LINUX
 
 ; Windows and mingw require a prologue helper routine if more than 4096 bytes area
 ; allocated on the stack.  Windows uses __chkstk and mingw uses __alloca.  __alloca
 ; and the 32-bit version of __chkstk will probe the stack and adjust the stack pointer.
 ; The 64-bit version of __chkstk is only responsible for probing the stack.  The 64-bit
 ; prologue is responsible for adjusting the stack pointer.
 
 ; Stack allocation >= 4096 bytes will require call to __chkstk in the Windows ABI.
 define i32 @main4k() nounwind {
 entry:
-; WIN_X32:    calll __chkstk
-; WIN_X64:    callq __chkstk
-; WIN64_LARGE: movabsq $__chkstk, %r11
-; WIN64_LARGE: callq *%r11
-; MINGW_X32:  calll __alloca
-; MINGW_X64:  callq ___chkstk_ms
-; LINUX-NOT:  call __chkstk
+; WIN: or{{.}}     $0, {{.*}}
+; LINUX-NOT: or{{[ql]}}     $0, {{.*}}
   %array4096 = alloca [4096 x i8], align 16       ; <[4096 x i8]*> [#uses=0]
   ret i32 0
 }
 
 ; Make sure we don't call __chkstk or __alloca when we have less than a 4096 stack
 ; allocation.
 define i32 @main128() nounwind {
 entry:
-; WIN_X32:       # BB#0:
-; WIN_X32-NOT:   calll __chkstk
-; WIN_X32:       ret
-
-; WIN_X64:       # BB#0:
-; WIN_X64-NOT:   callq __chkstk
-; WIN_X64:       ret
-
-; MINGW_X64:     # BB#0:
-; MINGW_X64-NOT: callq ___chkstk_ms
-; MINGW_X64:     ret
-
-; LINUX:         # BB#0:
-; LINUX-NOT:     call __chkstk
-; LINUX:         ret
+; WIN-NOT: or{{.}}     $0, {{.*}}
+; LINUX-NOT: or{{.}}     $0, {{.*}}
   %array128 = alloca [128 x i8], align 16         ; <[128 x i8]*> [#uses=0]
   ret i32 0
 }
 
 ; Make sure we don't call __chkstk or __alloca on non-Windows even if the
 ; caller has the Win64 calling convention.
 define x86_64_win64cc i32 @main4k_win64() nounwind {
 entry:
-; WIN_X32:    calll __chkstk
-; WIN_X64:    callq __chkstk
-; WIN64_LARGE: movabsq $__chkstk, %r11
-; WIN64_LARGE: callq *%r11
-; MINGW_X32:  calll __alloca
-; MINGW_X64:  callq ___chkstk_ms
-; LINUX-NOT:  call __chkstk
+; WIN: or{{.}}     $0, {{.*}}
+; LINUX-NOT: or{{.}}     $0, {{.*}}
   %array4096 = alloca [4096 x i8], align 16       ; <[4096 x i8]*> [#uses=0]
   ret i32 0
 }

test/CodeGen/X86/windows-itanium-alloca.ll

 ; RUN: llc -mtriple i686-windows-itanium -filetype asm -o - %s | FileCheck %s
 
 target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
 target triple = "i686--windows-itanium"
 
 declare void @external(i8*)
 
 define dllexport void @alloca(i32 %sz) {
 entry:
   %vla = alloca i8, i32 %sz, align 1
   call void @external(i8* %vla)
   ret void
 }
 
-; CHECK: __chkstk
-
+; CHECK: orl     $0, (%{{.*}})