Add a "gc-transition" operand bundle

[sanjoy]

Summary

This adds a new kind of operand bundle to LLVM denoted by the
"gc-transition" tag. Inputs to "gc-transition" operand bundle are
lowered into the "transition args" section of gc.statepoint by
RewriteStatepointsForGC.

This removes the last bit of functionality that was unsupported in the
deopt bundle based code path in RewriteStatepointsForGC.

The documentation for this operand bundle type is basically a TODO --
I'm hoping someone from the LLILC team can guide me towards what
should be put on the LangRef or add something in a subsequent commit.

[pgavlin]

LGTM aside from a few nits. Some suggested docs are above.

docs/LangRef.rst

1609 ↗

(On Diff #45334)

I'd suggest something like:

GC transition operand bundles are characterized by the
``"gc-transition"`` operand bundle tag. These operand
bundles mark a call as a transition between a function
with one GC strategy to a function with a different GC
strategy. If coordinating the transition between GC
strategies requires additional code generation at the
call site, these bundles may contain any values that
are needed by the generated code.

It may also be helpful to link to the "GC Transitions" section of the Statepoints doc.

lib/IR/Verifier.cpp

2516 ↗

(On Diff #45334)

Nit: this comment needs updating for GC transition bundles.

2526 ↗

(On Diff #45334)

Nit: this if statement and its successor could be else if clauses.

llvm/trunk/docs/LangRef.rst

 * does not have a ``"funclet"`` bundle and is not a ``call`` to a nounwind
   intrinsic, or
 * has a ``"funclet"`` bundle whose operand is not the most-recently-entered
   not-yet-exited funclet EH pad.
 
 Similarly, if no funclet EH pads have been entered-but-not-yet-exited,
 executing a ``call`` or ``invoke`` with a ``"funclet"`` bundle is undefined behavior.
 
+GC Transition Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+GC transition operand bundles are characterized by the
+``"gc-transition"`` operand bundle tag. These operand bundles mark a
+call as a transition between a function with one GC strategy to a
+function with a different GC strategy. If coordinating the transition
+between GC strategies requires additional code generation at the call
+site, these bundles may contain any values that are needed by the
+generated code.  For more details, see :ref:`GC Transitions
+<gc_transition_args>`.
+
 .. _moduleasm:
 
 Module-Level Inline Assembly
 ----------------------------
 
 Modules may contain "module-level inline asm" blocks, which corresponds
 to the GCC "file scope inline asm" blocks. These blocks are internally
 concatenated by LLVM and treated as a single unit, but may be separated

llvm/trunk/docs/Statepoints.rst

define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj)
     %obj.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %token, i32 7, i32 7)
     %gep.relocated = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %token, i32 7, i32 8)
     %p = getelementptr i8, i8 addrspace(1)* %gep, i64 -20000
     ret i8 addrspace(1)* %p
   }
 
 Note that in this example %p and %obj.relocate are the same address and we
 could replace one with the other, potentially removing the derived pointer
-from the live set at the safepoint entirely.  
+from the live set at the safepoint entirely.
 
+.. _gc_transition_args:
+
 GC Transitions
 ^^^^^^^^^^^^^^^^^^
 
 As a practical consideration, many garbage-collected systems allow code that is
 collector-aware ("managed code") to call code that is not collector-aware
 ("unmanaged code"). It is common that such calls must also be safepoints, since
 it is desirable to allow the collector to run during the execution of
 unmanaged code. Futhermore, it is common that coordinating the transition from

llvm/trunk/include/llvm/IR/LLVMContext.h

enum {
     MD_align = 17 // "align"
   };
 
   /// Known operand bundle tag IDs, which always have the same value.  All
   /// operand bundle tags that LLVM has special knowledge of are listed here.
   /// Additionally, this scheme allows LLVM to efficiently check for specific
   /// operand bundle tags without comparing strings.
   enum {
-    OB_deopt = 0,   // "deopt"
-    OB_funclet = 1, // "funclet"
+    OB_deopt = 0,         // "deopt"
+    OB_funclet = 1,       // "funclet"
+    OB_gc_transition = 2, // "gc-transition"
   };
 
   /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
   /// This ID is uniqued across modules in the current LLVMContext.
   unsigned getMDKindID(StringRef Name) const;
 
   /// getMDKindNames - Populate client supplied SmallVector with the name for
   /// custom metadata IDs registered in this LLVMContext.

llvm/trunk/lib/IR/LLVMContext.cpp

LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
   assert(DeoptEntry->second == LLVMContext::OB_deopt &&
          "deopt operand bundle id drifted!");
   (void)DeoptEntry;
 
   auto *FuncletEntry = pImpl->getOrInsertBundleTag("funclet");
   assert(FuncletEntry->second == LLVMContext::OB_funclet &&
          "funclet operand bundle id drifted!");
   (void)FuncletEntry;
+
+  auto *GCTransitionEntry = pImpl->getOrInsertBundleTag("gc-transition");
+  assert(GCTransitionEntry->second == LLVMContext::OB_gc_transition &&
+         "gc-transition operand bundle id drifted!");
+  (void)GCTransitionEntry;
 }
 LLVMContext::~LLVMContext() { delete pImpl; }
 
 void LLVMContext::addModule(Module *M) {
   pImpl->OwnedModules.insert(M);
 }
 
 void LLVMContext::removeModule(Module *M) {

llvm/trunk/lib/IR/Verifier.cpp

void Verifier::VerifyCallSite(CallSite CS) {
   if (CS.getCalledFunction() == nullptr)
     Assert(!FTy->getReturnType()->isTokenTy(),
            "Return type cannot be token for indirect call!");
 
   if (Function *F = CS.getCalledFunction())
     if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID())
       visitIntrinsicCallSite(ID, CS);
 
-  // Verify that a callsite has at most one "deopt" and one "funclet" operand
-  // bundle.
-  bool FoundDeoptBundle = false, FoundFuncletBundle = false;
+  // Verify that a callsite has at most one "deopt", at most one "funclet" and
+  // at most one "gc-transition" operand bundle.
+  bool FoundDeoptBundle = false, FoundFuncletBundle = false,
+       FoundGCTransitionBundle = false;
   for (unsigned i = 0, e = CS.getNumOperandBundles(); i < e; ++i) {
     OperandBundleUse BU = CS.getOperandBundleAt(i);
     uint32_t Tag = BU.getTagID();
     if (Tag == LLVMContext::OB_deopt) {
       Assert(!FoundDeoptBundle, "Multiple deopt operand bundles", I);
       FoundDeoptBundle = true;
-    }
-    if (Tag == LLVMContext::OB_funclet) {
+    } else if (Tag == LLVMContext::OB_gc_transition) {
+      Assert(!FoundGCTransitionBundle, "Multiple gc-transition operand bundles",
+             I);
+      FoundGCTransitionBundle = true;
+    } else if (Tag == LLVMContext::OB_funclet) {
       Assert(!FoundFuncletBundle, "Multiple funclet operand bundles", I);
       FoundFuncletBundle = true;
       Assert(BU.Inputs.size() == 1,
              "Expected exactly one funclet bundle operand", I);
       Assert(isa<FuncletPadInst>(BU.Inputs.front()),
              "Funclet bundle operands should correspond to a FuncletPadInst",
              I);
     }

llvm/trunk/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp

makeStatepointExplicitImpl(const CallSite CS, /* to replace */
   ArrayRef<Use> DeoptArgs;
   ArrayRef<Use> TransitionArgs;
 
   Value *CallTarget = nullptr;
 
   if (UseDeoptBundles) {
     CallArgs = {CS.arg_begin(), CS.arg_end()};
     DeoptArgs = GetDeoptBundleOperands(CS);
-    // TODO: we don't fill in TransitionArgs or Flags in this branch, but we
-    // could have an operand bundle for that too.
+    if (auto TransitionBundle =
+            CS.getOperandBundle(LLVMContext::OB_gc_transition)) {
+      Flags |= uint32_t(StatepointFlags::GCTransition);
+      TransitionArgs = TransitionBundle->Inputs;
+    }
     AttributeSet OriginalAttrs = CS.getAttributes();
 
     Attribute AttrID = OriginalAttrs.getAttribute(AttributeSet::FunctionIndex,
                                                   "statepoint-id");
     if (AttrID.isStringAttribute())
       AttrID.getValueAsString().getAsInteger(10, StatepointID);
 
     Attribute AttrNumPatchBytes = OriginalAttrs.getAttribute(

llvm/trunk/test/Transforms/RewriteStatepointsForGC/deopt-bundles/basic.ll

 
 ; CHECK: store i32 [[RESULT_F3]], i32 addrspace(1)* [[ARG_RELOCATED_CASTED]]
   ret i32 addrspace(1)* %arg
 
  unwind_dest: 
   %lpad = landingpad token cleanup
   resume token undef
 }
+
+define i32 addrspace(1)* @f4(i32 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: @f4(
+ entry:
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 1, i32 2, i32 400, i8 90,
+  call void @g() [ "gc-transition"(i32 400, i8 90) ]
+  ret i32 addrspace(1)* %arg
+}

llvm/trunk/test/Verifier/operand-bundles.ll

 
  entry:
   %l = load i32, i32* %ptr
   call void @g() [ "deopt"(i32 42, i64 100, i32 %x), "deopt"(float 0.0, i64 100, i32 %l) ]
   call void @g() [ "deopt"(i32 42, i64 120) ]  ;; The verifier should not complain about this one
   %x = add i32 42, 1
   ret void
 }
+
+define void @f_gc_transition(i32* %ptr) {
+; CHECK: Multiple gc-transition operand bundles
+; CHECK-NEXT: call void @g() [ "gc-transition"(i32 42, i64 100, i32 %x), "gc-transition"(float 0.000000e+00, i64 100, i32 %l) ]
+; CHECK-NOT: call void @g() [ "gc-transition"(i32 42, i64 120, i32 %x) ]
+
+ entry:
+  %l = load i32, i32* %ptr
+  call void @g() [ "gc-transition"(i32 42, i64 100, i32 %x), "gc-transition"(float 0.0, i64 100, i32 %l) ]
+  call void @g() [ "gc-transition"(i32 42, i64 120) ]  ;; The verifier should not complain about this one
+  %x = add i32 42, 1
+  ret void
+}