Use ImmutableStatepoint during statepoint lowering.

[igor-laevsky]

Summary

This change is a first step towards lowering of an invoke statepoints. It moves statepoint lowering in a separate function 'LowerStatepoint' which uses ImmutableStatepoint instead of a CallInst. Also related utility functions are changed to receive ImmutableCallSite.

[reames]

LGTM w/comments addressed. Please make sure to rebase carefully since I saw at least one cleanup patch does this today.

include/llvm/IR/Statepoint.h

20

Remove this please.

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

665

Comment isn't particularly helpful.

lib/CodeGen/SelectionDAG/StatepointLowering.cpp

307

Is this over 80 lines? I can't tell in the diff. If so, reformat.

include/llvm/IR/Statepoint.h

 // gc.result intrinsics
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef __LLVM_IR_STATEPOINT_H
 #define __LLVM_IR_STATEPOINT_H
 
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 class GCRelocateOperands;

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

 #define LLVM_LIB_CODEGEN_SELECTIONDAG_SELECTIONDAGBUILDER_H
 
 #include "StatepointLowering.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/CallSite.h"
+#include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetLowering.h"
 #include <vector>
 
 namespace llvm {
 
 class AddrSpaceCastInst;
std::pair<SDValue, SDValue> lowerCallOperands(
           bool UseVoidTy = false,
           MachineBasicBlock *LandingPad = nullptr,
           bool IsPatchPoint = false);
 
   /// UpdateSplitBlock - When an MBB was split during scheduling, update the
   /// references that need to refer to the last resulting block.
   void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last);
 
+  // This function is responsible for the whole statepoint lowering process.
+  void LowerStatepoint(ImmutableStatepoint Statepoint);
 private:
   std::pair<SDValue, SDValue> lowerInvokable(
           TargetLowering::CallLoweringInfo &CLI,
           MachineBasicBlock *LandingPad);
 
   // Terminator instructions.
   void visitRet(const ReturnInst &I);
   void visitBr(const BranchInst &I);

lib/CodeGen/SelectionDAG/StatepointLowering.cpp

static void removeDuplicatesGCPtrs(SmallVectorImpl<const Value *> &Bases,
   Relocs = NewRelocs;
   assert(Ptrs.size() == Bases.size());
   assert(Ptrs.size() == Relocs.size());
 }
 
 /// Extract call from statepoint, lower it and return pointer to the
 /// call node. Also update NodeMap so that getValue(statepoint) will
 /// reference lowered call result
-static SDNode *lowerCallFromStatepoint(const CallInst &CI,
+static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite,
                                        SelectionDAGBuilder &Builder) {
 
-  assert(Intrinsic::experimental_gc_statepoint ==
-             dyn_cast<IntrinsicInst>(&CI)->getIntrinsicID() &&
-         "function called must be the statepoint function");
-
-  ImmutableStatepoint StatepointOperands(&CI);
+  ImmutableCallSite CS(StatepointSite.getCallSite());
 
   // Lower the actual call itself - This is a bit of a hack, but we want to
   // avoid modifying the actual lowering code.  This is similiar in intent to
   // the LowerCallOperands mechanism used by PATCHPOINT, but is structured
   // differently.  Hopefully, this is slightly more robust w.r.t. calling
   // convention, return values, and other function attributes.
-  Value *ActualCallee = const_cast<Value *>(StatepointOperands.actualCallee());
+  Value *ActualCallee = const_cast<Value *>(StatepointSite.actualCallee());
 
   std::vector<Value *> Args;
-  CallInst::const_op_iterator arg_begin = StatepointOperands.call_args_begin();
-  CallInst::const_op_iterator arg_end = StatepointOperands.call_args_end();
+  CallInst::const_op_iterator arg_begin = StatepointSite.call_args_begin();
+  CallInst::const_op_iterator arg_end = StatepointSite.call_args_end();
   Args.insert(Args.end(), arg_begin, arg_end);
   // TODO: remove the creation of a new instruction!  We should not be
   // modifying the IR (even temporarily) at this point.
   CallInst *Tmp = CallInst::Create(ActualCallee, Args);
-  Tmp->setTailCall(CI.isTailCall());
-  Tmp->setCallingConv(CI.getCallingConv());
-  Tmp->setAttributes(CI.getAttributes());
+  Tmp->setTailCall(CS.isTailCall());
+  Tmp->setCallingConv(CS.getCallingConv());
+  Tmp->setAttributes(CS.getAttributes());
   Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false);
 
   // Handle the return value of the call iff any.
   const bool HasDef = !Tmp->getType()->isVoidTy();
   if (HasDef) {
     // The value of the statepoint itself will be the value of call itself.
     // We'll replace the actually call node shortly.  gc_result will grab
     // this value.
-    Builder.setValue(&CI, Builder.getValue(Tmp));
+    Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp));
   } else {
     // The token value is never used from here on, just generate a poison value
-    Builder.setValue(&CI, Builder.DAG.getIntPtrConstant(-1));
+    Builder.setValue(CS.getInstruction(), Builder.DAG.getIntPtrConstant(-1));
   }
   // Remove the fake entry we created so we don't have a hanging reference
   // after we delete this node.
   Builder.removeValue(Tmp);
   delete Tmp;
   Tmp = nullptr;
 
   // Search for the call node

 ///   Ptrs - derived pointers incoming to this statepoint
 ///   Relocs - the gc_relocate corresponding to each base/ptr pair
 /// Elements of this arrays should be in one-to-one correspondence with each
 /// other i.e Bases[i], Ptrs[i] are from the same gcrelocate call
 static void
 getIncomingStatepointGCValues(SmallVectorImpl<const Value *> &Bases,
                               SmallVectorImpl<const Value *> &Ptrs,
                               SmallVectorImpl<const Value *> &Relocs,
-                              ImmutableCallSite Statepoint,
+                              ImmutableStatepoint StatepointSite,
                               SelectionDAGBuilder &Builder) {
-  // Search for relocated pointers.  Note that working backwards from the
-  // gc_relocates ensures that we only get pairs which are actually relocated
-  // and used after the statepoint.
-  // TODO: This logic should probably become a utility function in Statepoint.h
-  for (const User *U : cast<CallInst>(Statepoint.getInstruction())->users()) {
-    if (!isGCRelocate(U)) {
-      continue;
-    }
-    GCRelocateOperands relocateOpers(U);
-    Relocs.push_back(cast<Value>(U));
+  for (GCRelocateOperands relocateOpers :
+         StatepointSite.getRelocates(StatepointSite)) {
+    Relocs.push_back(relocateOpers.getUnderlyingCallSite().getInstruction());
     Bases.push_back(relocateOpers.basePtr());
     Ptrs.push_back(relocateOpers.derivedPtr());
   }
 
   // Remove any redundant llvm::Values which map to the same SDValue as another
   // input.  Also has the effect of removing duplicates in the original
   // llvm::Value input list as well.  This is a useful optimization for
   // reducing the size of the StackMap section.  It has no other impact.

 /// Lower deopt state and gc pointer arguments of the statepoint.  The actual
 /// lowering is described in lowerIncomingStatepointValue.  This function is
 /// responsible for lowering everything in the right position and playing some
 /// tricks to avoid redundant stack manipulation where possible.  On
 /// completion, 'Ops' will contain ready to use operands for machine code
 /// statepoint. The chain nodes will have already been created and the DAG root
 /// will be set to the last value spilled (if any were).
 static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
-                                    ImmutableStatepoint Statepoint,
+                                    ImmutableStatepoint StatepointSite,
                                     SelectionDAGBuilder &Builder) {
 
   // Lower the deopt and gc arguments for this statepoint.  Layout will
   // be: deopt argument length, deopt arguments.., gc arguments...
 
   SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
   getIncomingStatepointGCValues(Bases, Ptrs, Relocations,
-                                Statepoint.getCallSite(), Builder);
+                                StatepointSite, Builder);
 
 #ifndef NDEBUG
   // Check that each of the gc pointer and bases we've gotten out of the
   // safepoint is something the strategy thinks might be a pointer into the GC
   // heap.  This is basically just here to help catch errors during statepoint
   // insertion. TODO: This should actually be in the Verifier, but we can't get
   // to the GCStrategy from there (yet).
   if (Builder.GFI) {
#endif
 
 
 
   // Before we actually start lowering (and allocating spill slots for values),
   // reserve any stack slots which we judge to be profitable to reuse for a
   // particular value.  This is purely an optimization over the code below and
   // doesn't change semantics at all.  It is important for performance that we
   // reserve slots for both deopt and gc values before lowering either.
-  for (auto I = Statepoint.vm_state_begin() + 1, E = Statepoint.vm_state_end();
+  for (auto I = StatepointSite.vm_state_begin() + 1,
+            E = StatepointSite.vm_state_end();
        I != E; ++I) {
     Value *V = *I;
     SDValue Incoming = Builder.getValue(V);
     reservePreviousStackSlotForValue(Incoming, Builder);
   }
   for (unsigned i = 0; i < Bases.size() * 2; ++i) {
     // Even elements will contain base, odd elements - derived ptr
     const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
     SDValue Incoming = Builder.getValue(V);
     reservePreviousStackSlotForValue(Incoming, Builder);
   }
 
   // First, prefix the list with the number of unique values to be
   // lowered.  Note that this is the number of *Values* not the
   // number of SDValues required to lower them.
-  const int NumVMSArgs = Statepoint.numTotalVMSArgs();
+  const int NumVMSArgs = StatepointSite.numTotalVMSArgs();
   Ops.push_back(
       Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
   Ops.push_back(Builder.DAG.getTargetConstant(NumVMSArgs, MVT::i64));
 
-  assert(NumVMSArgs + 1 == std::distance(Statepoint.vm_state_begin(),
-                                         Statepoint.vm_state_end()));
+  assert(NumVMSArgs + 1 == std::distance(StatepointSite.vm_state_begin(),
+                                         StatepointSite.vm_state_end()));
 
   // The vm state arguments are lowered in an opaque manner.  We do
   // not know what type of values are contained within.  We skip the
   // first one since that happens to be the total number we lowered
   // explicitly just above.  We could have left it in the loop and
   // not done it explicitly, but it's far easier to understand this
   // way.
-  for (auto I = Statepoint.vm_state_begin() + 1, E = Statepoint.vm_state_end();
+  for (auto I = StatepointSite.vm_state_begin() + 1,
+            E = StatepointSite.vm_state_end();
        I != E; ++I) {
     const Value *V = *I;
     SDValue Incoming = Builder.getValue(V);
     lowerIncomingStatepointValue(Incoming, Ops, Builder);
   }
 
   // Finally, go ahead and lower all the gc arguments.  There's no prefixed
   // length for this one.  After lowering, we'll have the base and pointer
   // arrays interwoven with each (lowered) base pointer immediately followed by
   // it's (lowered) derived pointer.  i.e
   // (base[0], ptr[0], base[1], ptr[1], ...)
   for (unsigned i = 0; i < Bases.size() * 2; ++i) {
     // Even elements will contain base, odd elements - derived ptr
     const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
     SDValue Incoming = Builder.getValue(V);
     lowerIncomingStatepointValue(Incoming, Ops, Builder);
   }
 }
+
 void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
+  // Check some preconditions for sanity
+  assert(isStatepoint(&CI) &&
+         "function called must be the statepoint function");
+
+  LowerStatepoint(ImmutableStatepoint(&CI));
+}
+
+void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP) {
   // The basic scheme here is that information about both the original call and
   // the safepoint is encoded in the CallInst.  We create a temporary call and
   // lower it, then reverse engineer the calling sequence.
 
-  // Check some preconditions for sanity
-  assert(isStatepoint(&CI) &&
-         "function called must be the statepoint function");
   NumOfStatepoints++;
   // Clear state
   StatepointLowering.startNewStatepoint(*this);
 
+  ImmutableCallSite CS(ISP.getCallSite());
+
 #ifndef NDEBUG
   // Consistency check
-  for (const User *U : CI.users()) {
+  for (const User *U : CS->users()) {
     const CallInst *Call = cast<CallInst>(U);
     if (isGCRelocate(Call))
       StatepointLowering.scheduleRelocCall(*Call);
   }
 #endif
 
-  ImmutableStatepoint ISP(&CI);
 #ifndef NDEBUG
   // If this is a malformed statepoint, report it early to simplify debugging.
   // This should catch any IR level mistake that's made when constructing or
   // transforming statepoints.
   ISP.verify();
 
   // Check that the associated GCStrategy expects to encounter statepoints.
   // TODO: This if should become an assert.  For now, we allow the GCStrategy
   // to be optional for backwards compatibility.  This will only last a short
   // period (i.e. a couple of weeks).
   if (GFI) {
     assert(GFI->getStrategy().useStatepoints() &&
            "GCStrategy does not expect to encounter statepoints");
   }
 #endif
 
 
   // Lower statepoint vmstate and gcstate arguments
   SmallVector<SDValue, 10> LoweredArgs;
   lowerStatepointMetaArgs(LoweredArgs, ISP, *this);
 
   // Get call node, we will replace it later with statepoint
-  SDNode *CallNode = lowerCallFromStatepoint(CI, *this);
+  SDNode *CallNode = lowerCallFromStatepoint(ISP, *this);
 
   // Construct the actual STATEPOINT node with all the appropriate arguments
   // and return values.
 
   // TODO: Currently, all of these operands are being marked as read/write in
   // PrologEpilougeInserter.cpp, we should special case the VMState arguments
   // and flags to be read-only.
   SmallVector<SDValue, 40> Ops;
#endif
   if (Glue.getNode())
     RegMaskIt = CallNode->op_end() - 2;
   else
     RegMaskIt = CallNode->op_end() - 1;
   Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
 
   // Add a leading constant argument with the Flags and the calling convention
   // masked together
-  CallingConv::ID CallConv = CI.getCallingConv();
-  int Flags = dyn_cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
+  CallingConv::ID CallConv = CS.getCallingConv();
+  int Flags = dyn_cast<ConstantInt>(CS.getArgument(2))->getZExtValue();
   assert(Flags == 0 && "not expected to be used");
   Ops.push_back(DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
   Ops.push_back(
       DAG.getTargetConstant(Flags | ((unsigned)CallConv << 1), MVT::i64));
 
   // Insert all vmstate and gcstate arguments
   Ops.insert(Ops.end(), LoweredArgs.begin(), LoweredArgs.end());