Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -2240,6 +2240,19 @@ ; initialize %b call void @bar(i32 42, %foo* preallocated(%foo) %b) ["preallocated"(token %t)] +.. _ob_gc_live + +GC Live Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A "gc-live" operand bundle is only valid on a :ref:`gc.statepoint ` +intrinsic. The operand bundle must contain every pointer to a garbage collected +object which potentially needs to be updated by the garbage collector. + +When lowered, any relocated value will be recorded in the corresponding +:ref:`stackmap entry `. See the intrinsic description +for further details. + .. _moduleasm: Module-Level Inline Assembly Index: llvm/docs/Statepoints.rst =================================================================== --- llvm/docs/Statepoints.rst +++ llvm/docs/Statepoints.rst @@ -434,6 +434,8 @@ Intrinsics =========== +.. _gc_statepoint: + 'llvm.experimental.gc.statepoint' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -530,7 +532,9 @@ collector. Note that the argument list must explicitly contain a base pointer for every derived pointer listed. The order of arguments is unimportant. Unlike the other variable length parameter sets, this -list is not length prefixed. +list is not length prefixed. Use of the 'gc parameters' list is +deprecated and will eventually be replaced entirely with the +:ref:`gc-live ` operand bundle. Semantics: """""""""" @@ -611,21 +615,25 @@ Despite the typing of this as a generic token, *only* the value defined by a ``gc.statepoint`` is legal here. -The second argument is an index into the statepoints list of arguments -which specifies the allocation for the pointer being relocated. -This index must land within the 'gc parameter' section of the -statepoint's argument list. The associated value must be within the -object with which the pointer being relocated is associated. The optimizer -is free to change *which* interior derived pointer is reported, provided that -it does not replace an actual base pointer with another interior derived -pointer. Collectors are allowed to rely on the base pointer operand -remaining an actual base pointer if so constructed. - -The third argument is an index into the statepoint's list of arguments -which specify the (potentially) derived pointer being relocated. It -is legal for this index to be the same as the second argument -if-and-only-if a base pointer is being relocated. This index must land -within the 'gc parameter' section of the statepoint's argument list. +The second and third arguments are both indices into operands of their +corresponding statepoint. If the statepoint has a :ref:`gc-live ` +operand bundle, then the value is an index into the operand bundle's operands. +If there is no "gc-live" bundle, then the index is into the statepoint's list +of arguments. This index must land within the 'gc parameter' section of the +statepoint's argument list. Use of the "gc-live" form is recommended. + +The second argument is an index which specifies the allocation for the pointer +being relocated. The associated value must be within the object with which the +pointer being relocated is associated. The optimizer is free to change *which* +interior derived pointer is reported, provided that it does not replace an +actual base pointer with another interior derived pointer. Collectors are +allowed to rely on the base pointer operand remaining an actual base pointer if +so constructed. + +The third argument is an index which specify the (potentially) derived pointer +being relocated. It is legal for this index to be the same as the second +argument if-and-only-if a base pointer is being relocated. This index must +land within the 'gc parameter' section of the statepoint's argument list. Semantics: """""""""" Index: llvm/include/llvm/IR/LLVMContext.h =================================================================== --- llvm/include/llvm/IR/LLVMContext.h +++ llvm/include/llvm/IR/LLVMContext.h @@ -92,6 +92,7 @@ OB_gc_transition = 2, // "gc-transition" OB_cfguardtarget = 3, // "cfguardtarget" OB_preallocated = 4, // "preallocated" + OB_gc_live = 5, // "gc-live" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. Index: llvm/include/llvm/IR/Statepoint.h =================================================================== --- llvm/include/llvm/IR/Statepoint.h +++ llvm/include/llvm/IR/Statepoint.h @@ -157,6 +157,9 @@ /// Returns an iterator to the begining of the argument range describing gc /// values for the statepoint. const_op_iterator gc_args_begin() const { + if (auto Opt = getOperandBundle(LLVMContext::OB_gc_live)) + return Opt->Inputs.begin(); + // The current format has two length prefix bundles between call args and // start of gc args. This will be removed in the near future. const Value *NumGCTransitionArgs = *actual_arg_end(); @@ -170,10 +173,16 @@ } /// Return an end iterator for the gc argument range - const_op_iterator gc_args_end() const { return arg_end(); } + const_op_iterator gc_args_end() const { + if (auto Opt = getOperandBundle(LLVMContext::OB_gc_live)) + return Opt->Inputs.end(); + + return arg_end(); + } /// Return the operand index at which the gc args begin unsigned gcArgsStartIdx() const { + assert(!getOperandBundle(LLVMContext::OB_gc_live)); return gc_args_begin() - op_begin(); } @@ -458,10 +467,14 @@ } Value *getBasePtr() const { + if (auto Opt = getStatepoint()->getOperandBundle(LLVMContext::OB_gc_live)) + return *(Opt->Inputs.begin() + getBasePtrIndex()); return *(getStatepoint()->arg_begin() + getBasePtrIndex()); } Value *getDerivedPtr() const { + if (auto Opt = getStatepoint()->getOperandBundle(LLVMContext::OB_gc_live)) + return *(Opt->Inputs.begin() + getDerivedPtrIndex()); return *(getStatepoint()->arg_begin() + getDerivedPtrIndex()); } }; Index: llvm/lib/IR/LLVMContext.cpp =================================================================== --- llvm/lib/IR/LLVMContext.cpp +++ llvm/lib/IR/LLVMContext.cpp @@ -73,6 +73,11 @@ "preallocated operand bundle id drifted!"); (void)PreallocatedEntry; + auto *GCLiveEntry = pImpl->getOrInsertBundleTag("gc-live"); + assert(GCLiveEntry->second == LLVMContext::OB_gc_live && + "gc-transition operand bundle id drifted!"); + (void)GCLiveEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -3102,7 +3102,7 @@ // and at most one "preallocated" operand bundle. bool FoundDeoptBundle = false, FoundFuncletBundle = false, FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false, - FoundPreallocatedBundle = false; + FoundPreallocatedBundle = false, FoundGCLiveBundle = false;; for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = Call.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); @@ -3139,6 +3139,10 @@ "\"preallocated\" argument must be a token from " "llvm.call.preallocated.setup", Call); + } else if (Tag == LLVMContext::OB_gc_live) { + Assert(!FoundGCLiveBundle, "Multiple gc-live operand bundles", + Call); + FoundGCLiveBundle = true; } } @@ -4746,45 +4750,53 @@ const int BaseIndex = cast(Base)->getZExtValue(); const int DerivedIndex = cast(Derived)->getZExtValue(); + // Check the bounds - Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint base index out of bounds", Call); - Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint derived index out of bounds", Call); - - // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' - // section of the statepoint's argument. - Assert(StatepointCall.arg_size() > 0, - "gc.statepoint: insufficient arguments"); - Assert(isa(StatepointCall.getArgOperand(3)), - "gc.statement: number of call arguments must be constant integer"); - const unsigned NumCallArgs = + if (auto Opt = StatepointCall.getOperandBundle(LLVMContext::OB_gc_live)) { + Assert(0 <= BaseIndex && BaseIndex < (int)Opt->Inputs.size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(0 <= DerivedIndex && DerivedIndex < (int)Opt->Inputs.size(), + "gc.relocate: statepoint derived index out of bounds", Call); + } else { + Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCall.arg_size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCall.arg_size(), + "gc.relocate: statepoint derived index out of bounds", Call); + + // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' + // section of the statepoint's argument. + Assert(StatepointCall.arg_size() > 0, + "gc.statepoint: insufficient arguments"); + Assert(isa(StatepointCall.getArgOperand(3)), + "gc.statement: number of call arguments must be constant integer"); + const unsigned NumCallArgs = cast(StatepointCall.getArgOperand(3))->getZExtValue(); - Assert(StatepointCall.arg_size() > NumCallArgs + 5, - "gc.statepoint: mismatch in number of call arguments"); - Assert(isa(StatepointCall.getArgOperand(NumCallArgs + 5)), - "gc.statepoint: number of transition arguments must be " - "a constant integer"); - const int NumTransitionArgs = - cast(StatepointCall.getArgOperand(NumCallArgs + 5)) - ->getZExtValue(); - const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; - Assert(isa(StatepointCall.getArgOperand(DeoptArgsStart)), - "gc.statepoint: number of deoptimization arguments must be " - "a constant integer"); - const int NumDeoptArgs = - cast(StatepointCall.getArgOperand(DeoptArgsStart)) - ->getZExtValue(); - const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; - const int GCParamArgsEnd = StatepointCall.arg_size(); - Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, - "gc.relocate: statepoint base index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); - Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, - "gc.relocate: statepoint derived index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); + Assert(StatepointCall.arg_size() > NumCallArgs + 5, + "gc.statepoint: mismatch in number of call arguments"); + Assert(isa(StatepointCall.getArgOperand(NumCallArgs + 5)), + "gc.statepoint: number of transition arguments must be " + "a constant integer"); + const int NumTransitionArgs = + cast(StatepointCall.getArgOperand(NumCallArgs + 5)) + ->getZExtValue(); + const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; + Assert(isa(StatepointCall.getArgOperand(DeoptArgsStart)), + "gc.statepoint: number of deoptimization arguments must be " + "a constant integer"); + const int NumDeoptArgs = + cast(StatepointCall.getArgOperand(DeoptArgsStart)) + ->getZExtValue(); + const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; + const int GCParamArgsEnd = StatepointCall.arg_size(); + Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, + "gc.relocate: statepoint base index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, + "gc.relocate: statepoint derived index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + } // Relocated value must be either a pointer type or vector-of-pointer type, // but gc_relocate does not need to return the same pointer type as the Index: llvm/test/Bitcode/operand-bundles-bc-analyzer.ll =================================================================== --- llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -8,6 +8,7 @@ ; CHECK-NEXT: