Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -726,6 +726,75 @@ /// references that need to refer to the last resulting block. void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); + /// Describes a gc.statepoint or a gc.statepoint like thing for the purposes + /// of lowering into a STATEPOINT node. Right now it only abstracts an actual + /// gc.statepoint, but that will change in the future. + struct StatepointInfo { + /// Bases[i] is the base pointer for Ptrs[i]. Together they denote the set + /// of gc pointers this STATEPOINT has to relocate. + ArrayRef Bases; + ArrayRef Ptrs; + + /// The set of gc.relocate calls associated with this gc.statepoint. + ArrayRef GCRelocates; + + /// The full list of gc arguments to the gc.statepoint being lowered. + ArrayRef GCArgs; + + /// The list of gc transition arguments present in the gc.statepoint being + /// lowered. + ArrayRef GCTransitionArgs; + + /// The ID that the resulting STATEPOINT instruction has to report. + unsigned ID; + + /// The actual destination of the gc.statepoint call or invoke. + const Value *CalledValue; + + /// The gc.result associated with this gc.statepoint call, if any. + const Instruction *GCResult; + + /// The deoptimization state associated with this gc.statepoint call, if + /// any. + ArrayRef DeoptState; + + /// The "raw" CallSite for this gc.statepoint call. + ImmutableCallSite CS; + + /// Denotes that this instance of StatepointInfo represents an actual + /// gc.statepoint. Currently always true. + bool IsStatepoint; + + /// Flags associated with the gc.statepoint. + uint64_t StatepointFlags; + + /// The number of patchable bytes the gc.statepoint needs to get lowered + /// into. + unsigned NumPatchBytes; + + /// The call arguments start at this index in the argument list of the raw + /// call site. + unsigned CallArgBeginIdx; + + /// The number of call arguments. + unsigned NumCallArgs; + + /// The exception handling unwind destination, in case this represents an + /// invoke of gc.statepoint. + const BasicBlock *EHPadBB; + + const BasicBlock *getParent() const { return CS.getParent(); } + const Instruction *getInstruction() const { return CS.getInstruction(); } + Type *getReturnType() const { + auto *FTy = cast( + cast(CalledValue->getType())->getElementType()); + return FTy->getReturnType(); + } + }; + + /// Lower \p SI into a STATEPOINT instruction. + void LowerAsStatepoint(const StatepointInfo &SI); + // This function is responsible for the whole statepoint lowering process. // It uniformly handles invoke and call statepoints. void LowerStatepoint(ImmutableStatepoint Statepoint, Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -280,15 +280,12 @@ /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result static SDNode * -lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB, - SelectionDAGBuilder &Builder, - SmallVectorImpl &PendingExports) { - - ImmutableCallSite CS(ISP.getCallSite()); - +lowerCallFromStatepointInfo(const SelectionDAGBuilder::StatepointInfo &SI, + SelectionDAGBuilder &Builder, + SmallVectorImpl &PendingExports) { SDValue ActualCallee; - if (ISP.getNumPatchBytes() > 0) { + if (SI.NumPatchBytes > 0) { // If we've been asked to emit a nop sequence instead of a call instruction // for this statepoint then don't lower the call target, but use a constant // `null` instead. Not lowering the call target lets statepoint clients get @@ -298,24 +295,23 @@ const auto &TLI = Builder.DAG.getTargetLoweringInfo(); const auto &DL = Builder.DAG.getDataLayout(); - unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace(); + unsigned AS = SI.CalledValue->getType()->getPointerAddressSpace(); ActualCallee = Builder.DAG.getConstant(0, Builder.getCurSDLoc(), TLI.getPointerTy(DL, AS)); } else { - ActualCallee = Builder.getValue(ISP.getCalledValue()); + ActualCallee = Builder.getValue(SI.CalledValue); } - assert(CS.getCallingConv() != CallingConv::AnyReg && + assert(SI.CS.getCallingConv() != CallingConv::AnyReg && "anyregcc is not supported on statepoints!"); - Type *DefTy = ISP.getActualReturnType(); + Type *DefTy = SI.getReturnType(); bool HasDef = !DefTy->isVoidTy(); SDValue ReturnValue, CallEndVal; std::tie(ReturnValue, CallEndVal) = Builder.lowerCallOperands( - ISP.getCallSite(), ImmutableStatepoint::CallArgsBeginPos, - ISP.getNumCallArgs(), ActualCallee, DefTy, EHPadBB, - false /* IsPatchPoint */); + SI.CS, SI.CallArgBeginIdx, SI.NumCallArgs, ActualCallee, DefTy, + SI.EHPadBB, false /* IsPatchPoint */); SDNode *CallEnd = CallEndVal.getNode(); @@ -346,9 +342,9 @@ assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!"); // Export the result value if needed - const Instruction *GCResult = ISP.getGCResult(); + const Instruction *GCResult = SI.GCResult; if (HasDef && GCResult) { - if (GCResult->getParent() != CS.getParent()) { + if (GCResult->getParent() != SI.CS.getParent()) { // Result value will be used in a different basic block so we need to // export it now. // Default exporting mechanism will not work here because statepoint call @@ -358,26 +354,26 @@ // manually. // TODO: To eliminate this problem we can remove gc.result intrinsics // completely and make statepoint call to return a tuple. - unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType()); - RegsForValue RFV( - *Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(), - Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType()); + unsigned Reg = Builder.FuncInfo.CreateRegs(SI.getReturnType()); + RegsForValue RFV(*Builder.DAG.getContext(), + Builder.DAG.getTargetLoweringInfo(), + Builder.DAG.getDataLayout(), Reg, SI.getReturnType()); SDValue Chain = Builder.DAG.getEntryNode(); RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain, nullptr); PendingExports.push_back(Chain); - Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg; + Builder.FuncInfo.ValueMap[SI.getInstruction()] = Reg; } else { // Result value will be used in a same basic block. Don't export it or // perform any explicit register copies. // We'll replace the actuall call node shortly. gc_result will grab // this value. - Builder.setValue(CS.getInstruction(), ReturnValue); + Builder.setValue(SI.getInstruction(), ReturnValue); } - } else { + } else if (SI.IsStatepoint) { // The token value is never used from here on, just generate a poison value - Builder.setValue(CS.getInstruction(), + Builder.setValue(SI.getInstruction(), Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc())); } @@ -501,18 +497,13 @@ /// completion, 'Ops' will contain ready to use operands for machine code /// statepoint. The chain nodes will have already been created and the DAG root /// will be set to the last value spilled (if any were). -static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, - ImmutableStatepoint StatepointSite, - SelectionDAGBuilder &Builder) { +static void +lowerStatepointInfoMetaArgs(SmallVectorImpl &Ops, + const SelectionDAGBuilder::StatepointInfo &SI, + SelectionDAGBuilder &Builder) { // Lower the deopt and gc arguments for this statepoint. Layout will // be: deopt argument length, deopt arguments.., gc arguments... - - SmallVector Bases, Ptrs; - SmallVector Relocations; - getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite, - Builder); - #ifndef NDEBUG // Check that each of the gc pointer and bases we've gotten out of the // safepoint is something the strategy thinks might be a pointer (or vector @@ -520,21 +511,21 @@ // errors during statepoint insertion. TODO: This should actually be in the // Verifier, but we can't get to the GCStrategy from there (yet). GCStrategy &S = Builder.GFI->getStrategy(); - for (const Value *V : Bases) { + for (const Value *V : SI.Bases) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed base pointer found in statepoint"); } } - for (const Value *V : Ptrs) { + for (const Value *V : SI.Ptrs) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed derived pointer found in statepoint"); } } - for (const GCRelocateInst *GCR : Relocations) { + for (const GCRelocateInst *GCR : SI.GCRelocates) { auto Opt = S.isGCManagedPointer(GCR->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed pointer relocated"); @@ -547,30 +538,23 @@ // particular value. This is purely an optimization over the code below and // doesn't change semantics at all. It is important for performance that we // reserve slots for both deopt and gc values before lowering either. - for (const Value *V : StatepointSite.vm_state_args()) { + for (const Value *V : SI.DeoptState) { reservePreviousStackSlotForValue(V, Builder); } - for (unsigned i = 0; i < Bases.size(); ++i) { - reservePreviousStackSlotForValue(Bases[i], Builder); - reservePreviousStackSlotForValue(Ptrs[i], Builder); + for (unsigned i = 0; i < SI.Bases.size(); ++i) { + reservePreviousStackSlotForValue(SI.Bases[i], Builder); + reservePreviousStackSlotForValue(SI.Ptrs[i], Builder); } // First, prefix the list with the number of unique values to be // lowered. Note that this is the number of *Values* not the // number of SDValues required to lower them. - const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs(); + const int NumVMSArgs = SI.DeoptState.size(); pushStackMapConstant(Ops, Builder, NumVMSArgs); - assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(), - StatepointSite.vm_state_end())); - - // The vm state arguments are lowered in an opaque manner. We do - // not know what type of values are contained within. We skip the - // first one since that happens to be the total number we lowered - // explicitly just above. We could have left it in the loop and - // not done it explicitly, but it's far easier to understand this - // way. - for (const Value *V : StatepointSite.vm_state_args()) { + // The vm state arguments are lowered in an opaque manner. We do not know + // what type of values are contained within. + for (const Value *V : SI.DeoptState) { SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } @@ -580,11 +564,11 @@ // arrays interwoven with each (lowered) base pointer immediately followed by // it's (lowered) derived pointer. i.e // (base[0], ptr[0], base[1], ptr[1], ...) - for (unsigned i = 0; i < Bases.size(); ++i) { - const Value *Base = Bases[i]; + for (unsigned i = 0; i < SI.Bases.size(); ++i) { + const Value *Base = SI.Bases[i]; lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder); - const Value *Ptr = Ptrs[i]; + const Value *Ptr = SI.Ptrs[i]; lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder); } @@ -593,7 +577,7 @@ // allocas and give control over placement to the consumer. In this case, // it is the contents of the slot which may get updated, not the pointer to // the alloca - for (Value *V : StatepointSite.gc_args()) { + for (Value *V : SI.GCArgs) { SDValue Incoming = Builder.getValue(V); if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { // This handles allocas as arguments to the statepoint @@ -605,11 +589,10 @@ // Record computed locations for all lowered values. // This can not be embedded in lowering loops as we need to record *all* // values, while previous loops account only values with unique SDValues. - const Instruction *StatepointInstr = - StatepointSite.getCallSite().getInstruction(); + const Instruction *StatepointInstr = SI.getInstruction(); auto &SpillMap = Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr]; - for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) { + for (const GCRelocateInst *Relocate : SI.GCRelocates) { const Value *V = Relocate->getDerivedPtr(); SDValue SDV = Builder.getValue(V); SDValue Loc = Builder.StatepointLowering.getLocation(SDV); @@ -645,8 +628,8 @@ LowerStatepoint(ImmutableStatepoint(&CI)); } -void SelectionDAGBuilder::LowerStatepoint( - ImmutableStatepoint ISP, const BasicBlock *EHPadBB /*= nullptr*/) { +void SelectionDAGBuilder::LowerAsStatepoint( + const SelectionDAGBuilder::StatepointInfo &SI) { // The basic scheme here is that information about both the original call and // the safepoint is encoded in the CallInst. We create a temporary call and // lower it, then reverse engineer the calling sequence. @@ -655,36 +638,18 @@ // Clear state StatepointLowering.startNewStatepoint(*this); - ImmutableCallSite CS(ISP.getCallSite()); - #ifndef NDEBUG - // Consistency check. Check only relocates in the same basic block as thier - // statepoint. - for (const User *U : CS->users()) { - const CallInst *Call = cast(U); - if (isa(Call) && Call->getParent() == CS.getParent()) - StatepointLowering.scheduleRelocCall(*Call); - } -#endif - -#ifndef NDEBUG - // If this is a malformed statepoint, report it early to simplify debugging. - // This should catch any IR level mistake that's made when constructing or - // transforming statepoints. - ISP.verify(); - - // Check that the associated GCStrategy expects to encounter statepoints. - assert(GFI->getStrategy().useStatepoints() && - "GCStrategy does not expect to encounter statepoints"); + for (auto *Reloc : SI.GCRelocates) + if (Reloc->getParent() == SI.getParent()) + StatepointLowering.scheduleRelocCall(*Reloc); #endif // Lower statepoint vmstate and gcstate arguments SmallVector LoweredMetaArgs; - lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this); + lowerStatepointInfoMetaArgs(LoweredMetaArgs, SI, *this); // Get call node, we will replace it later with statepoint - SDNode *CallNode = - lowerCallFromStatepoint(ISP, EHPadBB, *this, PendingExports); + SDNode *CallNode = lowerCallFromStatepointInfo(SI, *this, PendingExports); // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END // nodes with all the appropriate arguments and return values. @@ -707,7 +672,8 @@ // followed by a SRCVALUE for the pointer that may be used during lowering // (e.g. to form MachinePointerInfo values for loads/stores). const bool IsGCTransition = - (ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) == + SI.IsStatepoint && + (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) == (uint64_t)StatepointFlags::GCTransition; if (IsGCTransition) { SmallVector TSOps; @@ -716,7 +682,7 @@ TSOps.push_back(Chain); // Add GC transition arguments - for (const Value *V : ISP.gc_transition_args()) { + for (const Value *V : SI.GCTransitionArgs) { TSOps.push_back(getValue(V)); if (V->getType()->isPointerTy()) TSOps.push_back(DAG.getSrcValue(V)); @@ -741,9 +707,9 @@ SmallVector Ops; // Add the and constants. - Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64)); + Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64)); Ops.push_back( - DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32)); + DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32)); // Calculate and push starting position of vmstate arguments // Get number of arguments incoming directly into call node @@ -765,10 +731,10 @@ Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt); // Add a constant argument for the calling convention - pushStackMapConstant(Ops, *this, CS.getCallingConv()); + pushStackMapConstant(Ops, *this, SI.CS.getCallingConv()); // Add a constant argument for the flags - uint64_t Flags = ISP.getFlags(); + uint64_t Flags = SI.StatepointFlags; assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) && "Unknown flag used"); pushStackMapConstant(Ops, *this, Flags); @@ -806,7 +772,7 @@ TEOps.push_back(SDValue(StatepointMCNode, 0)); // Add GC transition arguments - for (const Value *V : ISP.gc_transition_args()) { + for (const Value *V : SI.GCTransitionArgs) { TEOps.push_back(getValue(V)); if (V->getType()->isPointerTy()) TEOps.push_back(DAG.getSrcValue(V)); @@ -838,6 +804,48 @@ // to actually be possible today. } +void +SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, + const BasicBlock *EHPadBB /*= nullptr*/) { + SmallVector Bases; + SmallVector Ptrs; + SmallVector GCRelocates; + + getIncomingStatepointGCValues(Bases, Ptrs, GCRelocates, ISP, *this); + + StatepointInfo SI; + SI.Bases = Bases; + SI.Ptrs = Ptrs; + SI.GCRelocates = GCRelocates; + SI.GCArgs = ArrayRef(ISP.gc_args_begin(), ISP.gc_args_end()); + SI.GCTransitionArgs = + ArrayRef(ISP.gc_args_begin(), ISP.gc_args_end()); + SI.ID = ISP.getID(); + SI.CalledValue = ISP.getCalledValue(); + SI.GCResult = ISP.getGCResult(); + SI.DeoptState = ArrayRef(ISP.vm_state_begin(), ISP.vm_state_end()); + SI.CS = ISP.getCallSite(); + SI.IsStatepoint = true; + SI.StatepointFlags = ISP.getFlags(); + SI.NumPatchBytes = ISP.getNumPatchBytes(); + SI.CallArgBeginIdx = ImmutableStatepoint::CallArgsBeginPos; + SI.NumCallArgs = ISP.getNumCallArgs(); + SI.EHPadBB = EHPadBB; + +#ifndef NDEBUG + // If this is a malformed statepoint, report it early to simplify debugging. + // This should catch any IR level mistake that's made when constructing or + // transforming statepoints. + ISP.verify(); + + // Check that the associated GCStrategy expects to encounter statepoints. + assert(GFI->getStrategy().useStatepoints() && + "GCStrategy does not expect to encounter statepoints"); +#endif + + LowerAsStatepoint(SI); +} + void SelectionDAGBuilder::visitGCResult(const CallInst &CI) { // The result value of the gc_result is simply the result of the actual // call. We've already emitted this, so just grab the value.