Introduce API for stack state modifying instructions

[anna]

Summary

This patch introduces an API to identify instructions that modify the
function stack state. The name here may not be accurate enough, but the
idea is to use a common API for transforms that move instructions (see
updated transforms).

This is also used in an upcoming change in InstCombine.

[anna]

ping

[mkazantsev]

LG

[lebedev.ri]

This review seems to be stuck/dead, consider abandoning if no longer relevant.

llvm/include/llvm/IR/Instruction.h

bool isFenceLike() const {
     case Instruction::CatchPad:
     case Instruction::CatchRet:
     case Instruction::Call:
     case Instruction::Invoke:
       return true;
     }
   }
 
+  /// Returns true if I modifies the stack state of the function.
+  /// These are:
+  ///   1. static and dynamic allocas
+  ///   2. llvm.stacksave and llvm.stackrestore intrinsics used with dynamic
+  ///      allocas. 
+  /// Use this API to identify instructions that are illegal to move (even if
+  /// they are marked side-effect free: allocas for example).
+  bool isFunctionStackStateModifying() const;
+
   /// Return true if the instruction may have side effects.
   ///
   /// Side effects are:
   ///  * Writing to memory.
   ///  * Unwinding.
   ///  * Not returning (e.g. an infinite loop).
   ///
   /// Note that this does not consider malloc and alloca to have side

llvm/lib/IR/Instruction.cpp

if (const CallInst *CI = dyn_cast<CallInst>(this))
     return !CI->doesNotThrow();
   if (const auto *CRI = dyn_cast<CleanupReturnInst>(this))
     return CRI->unwindsToCaller();
   if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(this))
     return CatchSwitch->unwindsToCaller();
   return isa<ResumeInst>(this);
 }
 
+bool Instruction::isFunctionStackStateModifying() const {
+  // This includes both static allocas (in entry blocks) and dynamic allocas
+  // (recorded between a stacksave/stackrestore pair). 
+  // Correctly sinking dynamic allocas require proper checks for
+  // stacksave/stackrestore intrinsics.
+  if (isa<AllocaInst>(this))
+    return true;
+  auto *II = dyn_cast<IntrinsicInst>(this);
+  if (!II)
+    return false;
+  Intrinsic::ID ID = II->getIntrinsicID();
+  return ID == Intrinsic::stacksave || ID == Intrinsic::stackrestore;
+}
+
 bool Instruction::mayHaveSideEffects() const {
   return mayWriteToMemory() || mayThrow() || !willReturn();
 }
 
 bool Instruction::isSafeToRemove() const {
   return (!isa<CallInst>(this) || !this->mayHaveSideEffects()) &&
          !this->isTerminator();
 }

llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
   if (isa<PHINode>(I) || I->isEHPad() || I->mayHaveSideEffects() ||
       I->isTerminator())
     return false;
 
   // Do not sink static or dynamic alloca instructions. Static allocas must
   // remain in the entry block, and dynamic allocas must not be sunk in between
   // a stacksave / stackrestore pair, which would incorrectly shorten its
   // lifetime.
-  if (isa<AllocaInst>(I))
+  if (I->isFunctionStackStateModifying())
     return false;
 
   // Do not sink into catchswitch blocks.
   if (isa<CatchSwitchInst>(DestBlock->getTerminator()))
     return false;
 
   // Do not sink convergent call instructions.
   if (auto *CI = dyn_cast<CallInst>(I)) {

llvm/lib/Transforms/Scalar/IndVarSimplify.cpp

while (I != Preheader->begin()) {
 
     // Skip eh pad instructions.
     if (I->isEHPad())
       continue;
 
     // Don't sink alloca: we never want to sink static alloca's out of the
     // entry block, and correctly sinking dynamic alloca's requires
     // checks for stacksave/stackrestore intrinsics.
-    // FIXME: Refactor this check somehow?
-    if (isa<AllocaInst>(I))
+    if (I->isFunctionStackStateModifying())
       continue;
 
     // Determine if there is a use in or before the loop (direct or
     // otherwise).
     bool UsedInLoop = false;
     for (Use &U : I->uses()) {
       Instruction *User = cast<Instruction>(U.getUser());
       BasicBlock *UseBB = User->getParent();