Diff 50889

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

Show First 20 Lines • Show All 720 Lines • ▼ Show 20 Lines	public:
std::pair<SDValue, SDValue>		std::pair<SDValue, SDValue>
lowerInvokable(TargetLowering::CallLoweringInfo &CLI,		lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
const BasicBlock *EHPadBB = nullptr);		const BasicBlock *EHPadBB = nullptr);

/// UpdateSplitBlock - When an MBB was split during scheduling, update the		/// UpdateSplitBlock - When an MBB was split during scheduling, update the
/// references that need to refer to the last resulting block.		/// references that need to refer to the last resulting block.
void UpdateSplitBlock(MachineBasicBlock First, MachineBasicBlock Last);		void UpdateSplitBlock(MachineBasicBlock First, MachineBasicBlock Last);

		/// Describes a gc.statepoint or a gc.statepoint like thing for the purposes
		/// of lowering into a STATEPOINT node. Right now it only abstracts an actual
		/// gc.statepoint, but that will change in the future.
		struct StatepointLoweringInfo {
		/// Bases[i] is the base pointer for Ptrs[i]. Together they denote the set
		/// of gc pointers this STATEPOINT has to relocate.
		ArrayRef<const Value *> Bases;
		ArrayRef<const Value *> Ptrs;

		/// The set of gc.relocate calls associated with this gc.statepoint.
		ArrayRef<const GCRelocateInst *> GCRelocates;

		/// The full list of gc arguments to the gc.statepoint being lowered.
		ArrayRef<const Use> GCArgs;

		/// The gc.statepoint instruction.
		const Instruction *StatepointInstr = nullptr;

		/// The list of gc transition arguments present in the gc.statepoint being
		/// lowered.
		ArrayRef<const Use> GCTransitionArgs;

		/// The ID that the resulting STATEPOINT instruction has to report.
		unsigned ID = -1;

		/// Information regarding the underlying call instruction.
		TargetLowering::CallLoweringInfo CLI;

		/// The deoptimization state associated with this gc.statepoint call, if
		/// any.
		ArrayRef<const Use> DeoptState;

		/// Flags associated with the meta arguments being lowered.
		uint64_t StatepointFlags = -1;

		/// The number of patchable bytes the call needs to get lowered into.
		unsigned NumPatchBytes = -1;

		/// The exception handling unwind destination, in case this represents an
		/// invoke of gc.statepoint.
		const BasicBlock *EHPadBB = nullptr;

		explicit StatepointLoweringInfo(SelectionDAG &DAG) : CLI(DAG) {}
		};

		/// Lower \p SLI into a STATEPOINT instruction.
		SDValue LowerAsStatepoint(StatepointLoweringInfo &SLI);

// This function is responsible for the whole statepoint lowering process.		// This function is responsible for the whole statepoint lowering process.
// It uniformly handles invoke and call statepoints.		// It uniformly handles invoke and call statepoints.
void LowerStatepoint(ImmutableStatepoint Statepoint,		void LowerStatepoint(ImmutableStatepoint Statepoint,
const BasicBlock *EHPadBB = nullptr);		const BasicBlock *EHPadBB = nullptr);
private:		private:
// Terminator instructions.		// Terminator instructions.
void visitRet(const ReturnInst &I);		void visitRet(const ReturnInst &I);
void visitBr(const BranchInst &I);		void visitBr(const BranchInst &I);
▲ Show 20 Lines • Show All 230 Lines • Show Last 20 Lines

llvm/trunk/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

Show First 20 Lines • Show All 273 Lines • ▼ Show 20 Lines	removeDuplicatesGCPtrs(SmallVectorImpl<const Value *> &Bases,
Relocs = NewRelocs;		Relocs = NewRelocs;
assert(Ptrs.size() == Bases.size());		assert(Ptrs.size() == Bases.size());
assert(Ptrs.size() == Relocs.size());		assert(Ptrs.size() == Relocs.size());
}		}

/// Extract call from statepoint, lower it and return pointer to the		/// Extract call from statepoint, lower it and return pointer to the
/// call node. Also update NodeMap so that getValue(statepoint) will		/// call node. Also update NodeMap so that getValue(statepoint) will
/// reference lowered call result		/// reference lowered call result
static SDNode *		static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB,		SelectionDAGBuilder::StatepointLoweringInfo &SI,
SelectionDAGBuilder &Builder,		SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
SmallVectorImpl<SDValue> &PendingExports) {

ImmutableCallSite CS(ISP.getCallSite());

SDValue ActualCallee;

if (ISP.getNumPatchBytes() > 0) {
// If we've been asked to emit a nop sequence instead of a call instruction
// for this statepoint then don't lower the call target, but use a constant
// `null` instead. Not lowering the call target lets statepoint clients get
// away without providing a physical address for the symbolic call target at
// link time.

const auto &TLI = Builder.DAG.getTargetLoweringInfo();
const auto &DL = Builder.DAG.getDataLayout();

unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
ActualCallee = Builder.DAG.getConstant(0, Builder.getCurSDLoc(),
TLI.getPointerTy(DL, AS));
} else {
ActualCallee = Builder.getValue(ISP.getCalledValue());
}

assert(CS.getCallingConv() != CallingConv::AnyReg &&
"anyregcc is not supported on statepoints!");

Type *DefTy = ISP.getActualReturnType();
bool HasDef = !DefTy->isVoidTy();

TargetLowering::CallLoweringInfo CLI(Builder.DAG);
Builder.populateCallLoweringInfo(
CLI, ISP.getCallSite(), ImmutableStatepoint::CallArgsBeginPos,
ISP.getNumCallArgs(), ActualCallee, DefTy, false);

SDValue ReturnValue, CallEndVal;		SDValue ReturnValue, CallEndVal;
std::tie(ReturnValue, CallEndVal) = Builder.lowerInvokable(CLI, EHPadBB);		std::tie(ReturnValue, CallEndVal) =
		Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
SDNode *CallEnd = CallEndVal.getNode();		SDNode *CallEnd = CallEndVal.getNode();

// Get a call instruction from the call sequence chain. Tail calls are not		// Get a call instruction from the call sequence chain. Tail calls are not
// allowed. The following code is essentially reverse engineering X86's		// allowed. The following code is essentially reverse engineering X86's
// LowerCallTo.		// LowerCallTo.
//		//
// We are expecting DAG to have the following form:		// We are expecting DAG to have the following form:
//		//
// ch = eh_label (only in case of invoke statepoint)		// ch = eh_label (only in case of invoke statepoint)
// ch, glue = callseq_start ch		// ch, glue = callseq_start ch
// ch, glue = X86::Call ch, glue		// ch, glue = X86::Call ch, glue
// ch, glue = callseq_end ch, glue		// ch, glue = callseq_end ch, glue
// get_return_value ch, glue		// get_return_value ch, glue
//		//
// get_return_value can either be a sequence of CopyFromReg instructions		// get_return_value can either be a sequence of CopyFromReg instructions
// to grab the return value from the return register(s), or it can be a LOAD		// to grab the return value from the return register(s), or it can be a LOAD
// to load a value returned by reference via a stack slot.		// to load a value returned by reference via a stack slot.

		bool HasDef = !SI.CLI.RetTy->isVoidTy();
if (HasDef) {		if (HasDef) {
if (CallEnd->getOpcode() == ISD::LOAD)		if (CallEnd->getOpcode() == ISD::LOAD)
CallEnd = CallEnd->getOperand(0).getNode();		CallEnd = CallEnd->getOperand(0).getNode();
else		else
while (CallEnd->getOpcode() == ISD::CopyFromReg)		while (CallEnd->getOpcode() == ISD::CopyFromReg)
CallEnd = CallEnd->getOperand(0).getNode();		CallEnd = CallEnd->getOperand(0).getNode();
}		}

assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");		assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
		return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
// Export the result value if needed
const Instruction *GCResult = ISP.getGCResult();
if (HasDef && GCResult) {
if (GCResult->getParent() != CS.getParent()) {
// Result value will be used in a different basic block so we need to
// export it now.
// Default exporting mechanism will not work here because statepoint call
// has a different type than the actual call. It means that by default
// llvm will create export register of the wrong type (always i32 in our
// case). So instead we need to create export register with correct type
// manually.
// TODO: To eliminate this problem we can remove gc.result intrinsics
// completely and make statepoint call to return a tuple.
unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType());
RegsForValue RFV(
*Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(),
Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType());
SDValue Chain = Builder.DAG.getEntryNode();

RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain,
nullptr);
PendingExports.push_back(Chain);
Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg;
} else {
// Result value will be used in a same basic block. Don't export it or
// perform any explicit register copies.
// We'll replace the actuall call node shortly. gc_result will grab
// this value.
Builder.setValue(CS.getInstruction(), ReturnValue);
}
} else {
// The token value is never used from here on, just generate a poison value
Builder.setValue(CS.getInstruction(),
Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc()));
}

return CallEnd->getOperand(0).getNode();
}		}

/// Callect all gc pointers coming into statepoint intrinsic, clean them up,		/// Callect all gc pointers coming into statepoint intrinsic, clean them up,
/// and return two arrays:		/// and return two arrays:
/// Bases - base pointers incoming to this statepoint		/// Bases - base pointers incoming to this statepoint
/// Ptrs - derived pointers incoming to this statepoint		/// Ptrs - derived pointers incoming to this statepoint
/// Relocs - the gc_relocate corresponding to each base/ptr pair		/// Relocs - the gc_relocate corresponding to each base/ptr pair
/// Elements of this arrays should be in one-to-one correspondence with each		/// Elements of this arrays should be in one-to-one correspondence with each
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines

/// Lower deopt state and gc pointer arguments of the statepoint. The actual		/// Lower deopt state and gc pointer arguments of the statepoint. The actual
/// lowering is described in lowerIncomingStatepointValue. This function is		/// lowering is described in lowerIncomingStatepointValue. This function is
/// responsible for lowering everything in the right position and playing some		/// responsible for lowering everything in the right position and playing some
/// tricks to avoid redundant stack manipulation where possible. On		/// tricks to avoid redundant stack manipulation where possible. On
/// completion, 'Ops' will contain ready to use operands for machine code		/// completion, 'Ops' will contain ready to use operands for machine code
/// statepoint. The chain nodes will have already been created and the DAG root		/// statepoint. The chain nodes will have already been created and the DAG root
/// will be set to the last value spilled (if any were).		/// will be set to the last value spilled (if any were).
static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,		static void
ImmutableStatepoint StatepointSite,		lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
		SelectionDAGBuilder::StatepointLoweringInfo &SI,
SelectionDAGBuilder &Builder) {		SelectionDAGBuilder &Builder) {

// Lower the deopt and gc arguments for this statepoint. Layout will		// Lower the deopt and gc arguments for this statepoint. Layout will
// be: deopt argument length, deopt arguments.., gc arguments...		// be: deopt argument length, deopt arguments.., gc arguments...

SmallVector<const Value *, 64> Bases, Ptrs;
SmallVector<const GCRelocateInst *, 64> Relocations;
getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
Builder);

#ifndef NDEBUG		#ifndef NDEBUG
// Check that each of the gc pointer and bases we've gotten out of the		// Check that each of the gc pointer and bases we've gotten out of the
// safepoint is something the strategy thinks might be a pointer (or vector		// safepoint is something the strategy thinks might be a pointer (or vector
// of pointers) into the GC heap. This is basically just here to help catch		// of pointers) into the GC heap. This is basically just here to help catch
// errors during statepoint insertion. TODO: This should actually be in the		// errors during statepoint insertion. TODO: This should actually be in the
// Verifier, but we can't get to the GCStrategy from there (yet).		// Verifier, but we can't get to the GCStrategy from there (yet).
GCStrategy &S = Builder.GFI->getStrategy();		GCStrategy &S = Builder.GFI->getStrategy();
for (const Value *V : Bases) {		for (const Value *V : SI.Bases) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());		auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {		if (Opt.hasValue()) {
assert(Opt.getValue() &&		assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");		"non gc managed base pointer found in statepoint");
}		}
}		}
for (const Value *V : Ptrs) {		for (const Value *V : SI.Ptrs) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());		auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {		if (Opt.hasValue()) {
assert(Opt.getValue() &&		assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");		"non gc managed derived pointer found in statepoint");
}		}
}		}
#endif		#endif

// Before we actually start lowering (and allocating spill slots for values),		// Before we actually start lowering (and allocating spill slots for values),
// reserve any stack slots which we judge to be profitable to reuse for a		// reserve any stack slots which we judge to be profitable to reuse for a
// particular value. This is purely an optimization over the code below and		// particular value. This is purely an optimization over the code below and
// doesn't change semantics at all. It is important for performance that we		// doesn't change semantics at all. It is important for performance that we
// reserve slots for both deopt and gc values before lowering either.		// reserve slots for both deopt and gc values before lowering either.
for (const Value *V : StatepointSite.vm_state_args()) {		for (const Value *V : SI.DeoptState) {
reservePreviousStackSlotForValue(V, Builder);		reservePreviousStackSlotForValue(V, Builder);
}		}
for (unsigned i = 0; i < Bases.size(); ++i) {		for (unsigned i = 0; i < SI.Bases.size(); ++i) {
reservePreviousStackSlotForValue(Bases[i], Builder);		reservePreviousStackSlotForValue(SI.Bases[i], Builder);
reservePreviousStackSlotForValue(Ptrs[i], Builder);		reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
}		}

// First, prefix the list with the number of unique values to be		// First, prefix the list with the number of unique values to be
// lowered. Note that this is the number of Values not the		// lowered. Note that this is the number of Values not the
// number of SDValues required to lower them.		// number of SDValues required to lower them.
const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();		const int NumVMSArgs = SI.DeoptState.size();
pushStackMapConstant(Ops, Builder, NumVMSArgs);		pushStackMapConstant(Ops, Builder, NumVMSArgs);

assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),		// The vm state arguments are lowered in an opaque manner. We do not know
StatepointSite.vm_state_end()));		// what type of values are contained within.
		for (const Value *V : SI.DeoptState) {
// The vm state arguments are lowered in an opaque manner. We do
// not know what type of values are contained within. We skip the
// first one since that happens to be the total number we lowered
// explicitly just above. We could have left it in the loop and
// not done it explicitly, but it's far easier to understand this
// way.
for (const Value *V : StatepointSite.vm_state_args()) {
SDValue Incoming = Builder.getValue(V);		SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);		lowerIncomingStatepointValue(Incoming, Ops, Builder);
}		}

// Finally, go ahead and lower all the gc arguments. There's no prefixed		// Finally, go ahead and lower all the gc arguments. There's no prefixed
// length for this one. After lowering, we'll have the base and pointer		// length for this one. After lowering, we'll have the base and pointer
// arrays interwoven with each (lowered) base pointer immediately followed by		// arrays interwoven with each (lowered) base pointer immediately followed by
// it's (lowered) derived pointer. i.e		// it's (lowered) derived pointer. i.e
// (base[0], ptr[0], base[1], ptr[1], ...)		// (base[0], ptr[0], base[1], ptr[1], ...)
for (unsigned i = 0; i < Bases.size(); ++i) {		for (unsigned i = 0; i < SI.Bases.size(); ++i) {
const Value *Base = Bases[i];		const Value *Base = SI.Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);		lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);

const Value *Ptr = Ptrs[i];		const Value *Ptr = SI.Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);		lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
}		}

// If there are any explicit spill slots passed to the statepoint, record		// If there are any explicit spill slots passed to the statepoint, record
// them, but otherwise do not do anything special. These are user provided		// them, but otherwise do not do anything special. These are user provided
// allocas and give control over placement to the consumer. In this case,		// allocas and give control over placement to the consumer. In this case,
// it is the contents of the slot which may get updated, not the pointer to		// it is the contents of the slot which may get updated, not the pointer to
// the alloca		// the alloca
for (Value *V : StatepointSite.gc_args()) {		for (Value *V : SI.GCArgs) {
SDValue Incoming = Builder.getValue(V);		SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {		if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
// This handles allocas as arguments to the statepoint		// This handles allocas as arguments to the statepoint
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),		Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Incoming.getValueType()));		Incoming.getValueType()));
}		}
}		}

// Record computed locations for all lowered values.		// Record computed locations for all lowered values.
// This can not be embedded in lowering loops as we need to record all		// This can not be embedded in lowering loops as we need to record all
// values, while previous loops account only values with unique SDValues.		// values, while previous loops account only values with unique SDValues.
const Instruction *StatepointInstr =		const Instruction *StatepointInstr = SI.StatepointInstr;
StatepointSite.getCallSite().getInstruction();
auto &SpillMap = Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];		auto &SpillMap = Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];

for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) {		for (const GCRelocateInst *Relocate : SI.GCRelocates) {
const Value *V = Relocate->getDerivedPtr();		const Value *V = Relocate->getDerivedPtr();
SDValue SDV = Builder.getValue(V);		SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);		SDValue Loc = Builder.StatepointLowering.getLocation(SDV);

if (Loc.getNode()) {		if (Loc.getNode()) {
SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();		SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {		} else {
// Record value as visited, but not spilled. This is case for allocas		// Record value as visited, but not spilled. This is case for allocas
Show All 19 Lines
void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {		void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
// Check some preconditions for sanity		// Check some preconditions for sanity
assert(isStatepoint(&CI) &&		assert(isStatepoint(&CI) &&
"Function called must be the statepoint function");		"Function called must be the statepoint function");

LowerStatepoint(ImmutableStatepoint(&CI));		LowerStatepoint(ImmutableStatepoint(&CI));
}		}

void SelectionDAGBuilder::LowerStatepoint(		SDValue SelectionDAGBuilder::LowerAsStatepoint(
ImmutableStatepoint ISP, const BasicBlock EHPadBB /= nullptr*/) {		SelectionDAGBuilder::StatepointLoweringInfo &SI) {
// The basic scheme here is that information about both the original call and		// The basic scheme here is that information about both the original call and
// the safepoint is encoded in the CallInst. We create a temporary call and		// the safepoint is encoded in the CallInst. We create a temporary call and
// lower it, then reverse engineer the calling sequence.		// lower it, then reverse engineer the calling sequence.

NumOfStatepoints++;		NumOfStatepoints++;
// Clear state		// Clear state
StatepointLowering.startNewStatepoint(*this);		StatepointLowering.startNewStatepoint(*this);

ImmutableCallSite CS(ISP.getCallSite());

#ifndef NDEBUG		#ifndef NDEBUG
// Consistency check. Check only relocates in the same basic block as thier		for (auto *Reloc : SI.GCRelocates)
// statepoint.		if (Reloc->getParent() == SI.StatepointInstr->getParent())
for (const User *U : CS->users()) {		StatepointLowering.scheduleRelocCall(*Reloc);
const CallInst *Call = cast<CallInst>(U);
if (isa<GCRelocateInst>(Call) && Call->getParent() == CS.getParent())
StatepointLowering.scheduleRelocCall(*Call);
}
#endif

#ifndef NDEBUG
// If this is a malformed statepoint, report it early to simplify debugging.
// This should catch any IR level mistake that's made when constructing or
// transforming statepoints.
ISP.verify();

// Check that the associated GCStrategy expects to encounter statepoints.
assert(GFI->getStrategy().useStatepoints() &&
"GCStrategy does not expect to encounter statepoints");
#endif		#endif

// Lower statepoint vmstate and gcstate arguments		// Lower statepoint vmstate and gcstate arguments
SmallVector<SDValue, 10> LoweredMetaArgs;		SmallVector<SDValue, 10> LoweredMetaArgs;
lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this);		lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);

		// Now that we've emitted the spills, we need to update the root so that the
		// call sequence is ordered correctly.
		SI.CLI.setChain(getRoot());

// Get call node, we will replace it later with statepoint		// Get call node, we will replace it later with statepoint
SDNode *CallNode =		SDValue ReturnVal;
lowerCallFromStatepoint(ISP, EHPadBB, *this, PendingExports);		SDNode *CallNode;
		std::tie(ReturnVal, CallNode) =
		lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);

// Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END		// Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
// nodes with all the appropriate arguments and return values.		// nodes with all the appropriate arguments and return values.

// Call Node: Chain, Target, {Args}, RegMask, [Glue]		// Call Node: Chain, Target, {Args}, RegMask, [Glue]
SDValue Chain = CallNode->getOperand(0);		SDValue Chain = CallNode->getOperand(0);

SDValue Glue;		SDValue Glue;
bool CallHasIncomingGlue = CallNode->getGluedNode();		bool CallHasIncomingGlue = CallNode->getGluedNode();
if (CallHasIncomingGlue) {		if (CallHasIncomingGlue) {
// Glue is always last operand		// Glue is always last operand
Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);		Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
}		}

// Build the GC_TRANSITION_START node if necessary.		// Build the GC_TRANSITION_START node if necessary.
//		//
// The operands to the GC_TRANSITION_{START,END} nodes are laid out in the		// The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
// order in which they appear in the call to the statepoint intrinsic. If		// order in which they appear in the call to the statepoint intrinsic. If
// any of the operands is a pointer-typed, that operand is immediately		// any of the operands is a pointer-typed, that operand is immediately
// followed by a SRCVALUE for the pointer that may be used during lowering		// followed by a SRCVALUE for the pointer that may be used during lowering
// (e.g. to form MachinePointerInfo values for loads/stores).		// (e.g. to form MachinePointerInfo values for loads/stores).
const bool IsGCTransition =		const bool IsGCTransition =
(ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) ==		(SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
(uint64_t)StatepointFlags::GCTransition;		(uint64_t)StatepointFlags::GCTransition;
if (IsGCTransition) {		if (IsGCTransition) {
SmallVector<SDValue, 8> TSOps;		SmallVector<SDValue, 8> TSOps;

// Add chain		// Add chain
TSOps.push_back(Chain);		TSOps.push_back(Chain);

// Add GC transition arguments		// Add GC transition arguments
for (const Value *V : ISP.gc_transition_args()) {		for (const Value *V : SI.GCTransitionArgs) {
TSOps.push_back(getValue(V));		TSOps.push_back(getValue(V));
if (V->getType()->isPointerTy())		if (V->getType()->isPointerTy())
TSOps.push_back(DAG.getSrcValue(V));		TSOps.push_back(DAG.getSrcValue(V));
}		}

// Add glue if necessary		// Add glue if necessary
if (CallHasIncomingGlue)		if (CallHasIncomingGlue)
TSOps.push_back(Glue);		TSOps.push_back(Glue);

SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);		SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);

SDValue GCTransitionStart =		SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);		DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);

Chain = GCTransitionStart.getValue(0);		Chain = GCTransitionStart.getValue(0);
Glue = GCTransitionStart.getValue(1);		Glue = GCTransitionStart.getValue(1);
}		}

// TODO: Currently, all of these operands are being marked as read/write in		// TODO: Currently, all of these operands are being marked as read/write in
// PrologEpilougeInserter.cpp, we should special case the VMState arguments		// PrologEpilougeInserter.cpp, we should special case the VMState arguments
// and flags to be read-only.		// and flags to be read-only.
SmallVector<SDValue, 40> Ops;		SmallVector<SDValue, 40> Ops;

// Add the <id> and <numBytes> constants.		// Add the <id> and <numBytes> constants.
Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));		Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
Ops.push_back(		Ops.push_back(
DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));		DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));

// Calculate and push starting position of vmstate arguments		// Calculate and push starting position of vmstate arguments
// Get number of arguments incoming directly into call node		// Get number of arguments incoming directly into call node
unsigned NumCallRegArgs =		unsigned NumCallRegArgs =
CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);		CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));		Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));

// Add call target		// Add call target
SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);		SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
Ops.push_back(CallTarget);		Ops.push_back(CallTarget);

// Add call arguments		// Add call arguments
// Get position of register mask in the call		// Get position of register mask in the call
SDNode::op_iterator RegMaskIt;		SDNode::op_iterator RegMaskIt;
if (CallHasIncomingGlue)		if (CallHasIncomingGlue)
RegMaskIt = CallNode->op_end() - 2;		RegMaskIt = CallNode->op_end() - 2;
else		else
RegMaskIt = CallNode->op_end() - 1;		RegMaskIt = CallNode->op_end() - 1;
Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);		Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);

// Add a constant argument for the calling convention		// Add a constant argument for the calling convention
pushStackMapConstant(Ops, *this, CS.getCallingConv());		pushStackMapConstant(Ops, *this, SI.CLI.CallConv);

// Add a constant argument for the flags		// Add a constant argument for the flags
uint64_t Flags = ISP.getFlags();		uint64_t Flags = SI.StatepointFlags;
assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&		assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
"Unknown flag used");		"Unknown flag used");
pushStackMapConstant(Ops, *this, Flags);		pushStackMapConstant(Ops, *this, Flags);

// Insert all vmstate and gcstate arguments		// Insert all vmstate and gcstate arguments
Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());		Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());

// Add register mask from call node		// Add register mask from call node
Show All 21 Lines	#endif
// the operands to the GC_TRANSITION_END node.		// the operands to the GC_TRANSITION_END node.
if (IsGCTransition) {		if (IsGCTransition) {
SmallVector<SDValue, 8> TEOps;		SmallVector<SDValue, 8> TEOps;

// Add chain		// Add chain
TEOps.push_back(SDValue(StatepointMCNode, 0));		TEOps.push_back(SDValue(StatepointMCNode, 0));

// Add GC transition arguments		// Add GC transition arguments
for (const Value *V : ISP.gc_transition_args()) {		for (const Value *V : SI.GCTransitionArgs) {
TEOps.push_back(getValue(V));		TEOps.push_back(getValue(V));
if (V->getType()->isPointerTy())		if (V->getType()->isPointerTy())
TEOps.push_back(DAG.getSrcValue(V));		TEOps.push_back(DAG.getSrcValue(V));
}		}

// Add glue		// Add glue
TEOps.push_back(SDValue(StatepointMCNode, 1));		TEOps.push_back(SDValue(StatepointMCNode, 1));

Show All 13 Lines	#endif
// DON'T set the root - under the assumption that it's already set past the		// DON'T set the root - under the assumption that it's already set past the
// inserted node we created.		// inserted node we created.

// TODO: A better future implementation would be to emit a single variable		// TODO: A better future implementation would be to emit a single variable
// argument, variable return value STATEPOINT node here and then hookup the		// argument, variable return value STATEPOINT node here and then hookup the
// return value of each gc.relocate to the respective output of the		// return value of each gc.relocate to the respective output of the
// previously emitted STATEPOINT value. Unfortunately, this doesn't appear		// previously emitted STATEPOINT value. Unfortunately, this doesn't appear
// to actually be possible today.		// to actually be possible today.

		return ReturnVal;
		}

		void
		SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
		const BasicBlock EHPadBB /= nullptr*/) {
		SmallVector<const Value *, 16> Bases;
		SmallVector<const Value *, 16> Ptrs;
		SmallVector<const GCRelocateInst *, 16> GCRelocates;

		getIncomingStatepointGCValues(Bases, Ptrs, GCRelocates, ISP, *this);

		assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
		"anyregcc is not supported on statepoints!");

		#ifndef NDEBUG
		// If this is a malformed statepoint, report it early to simplify debugging.
		// This should catch any IR level mistake that's made when constructing or
		// transforming statepoints.
		ISP.verify();

		// Check that the associated GCStrategy expects to encounter statepoints.
		assert(GFI->getStrategy().useStatepoints() &&
		"GCStrategy does not expect to encounter statepoints");
		#endif

		SDValue ActualCallee;

		if (ISP.getNumPatchBytes() > 0) {
		// If we've been asked to emit a nop sequence instead of a call instruction
		// for this statepoint then don't lower the call target, but use a constant
		// `null` instead. Not lowering the call target lets statepoint clients get
		// away without providing a physical address for the symbolic call target at
		// link time.

		const auto &TLI = DAG.getTargetLoweringInfo();
		const auto &DL = DAG.getDataLayout();

		unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
		ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
		} else {
		ActualCallee = getValue(ISP.getCalledValue());
		}

		StatepointLoweringInfo SI(DAG);
		populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
		ImmutableStatepoint::CallArgsBeginPos,
		ISP.getNumCallArgs(), ActualCallee,
		ISP.getActualReturnType(), false /* IsPatchPoint */);

		SI.Bases = Bases;
		SI.Ptrs = Ptrs;
		SI.GCRelocates = GCRelocates;
		SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
		SI.StatepointInstr = ISP.getInstruction();
		SI.GCTransitionArgs =
		ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
		SI.ID = ISP.getID();
		SI.DeoptState = ArrayRef<const Use>(ISP.vm_state_begin(), ISP.vm_state_end());
		SI.StatepointFlags = ISP.getFlags();
		SI.NumPatchBytes = ISP.getNumPatchBytes();
		SI.EHPadBB = EHPadBB;

		SDValue ReturnValue = LowerAsStatepoint(SI);

		// Export the result value if needed
		const Instruction *GCResult = ISP.getGCResult();
		Type *RetTy = ISP.getActualReturnType();
		if (!RetTy->isVoidTy() && GCResult) {
		if (GCResult->getParent() != ISP.getCallSite().getParent()) {
		// Result value will be used in a different basic block so we need to
		// export it now. Default exporting mechanism will not work here because
		// statepoint call has a different type than the actual call. It means
		// that by default llvm will create export register of the wrong type
		// (always i32 in our case). So instead we need to create export register
		// with correct type manually.
		// TODO: To eliminate this problem we can remove gc.result intrinsics
		// completely and make statepoint call to return a tuple.
		unsigned Reg = FuncInfo.CreateRegs(RetTy);
		RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
		DAG.getDataLayout(), Reg, RetTy);
		SDValue Chain = DAG.getEntryNode();

		RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
		PendingExports.push_back(Chain);
		FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
		} else {
		// Result value will be used in a same basic block. Don't export it or
		// perform any explicit register copies.
		// We'll replace the actuall call node shortly. gc_result will grab
		// this value.
		setValue(ISP.getInstruction(), ReturnValue);
		}
		} else {
		// The token value is never used from here on, just generate a poison value
		setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
		}
}		}

void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {		void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
// The result value of the gc_result is simply the result of the actual		// The result value of the gc_result is simply the result of the actual
// call. We've already emitted this, so just grab the value.		// call. We've already emitted this, so just grab the value.
Instruction *I = cast<Instruction>(CI.getArgOperand(0));		Instruction *I = cast<Instruction>(CI.getArgOperand(0));
assert(isStatepoint(I) && "first argument must be a statepoint token");		assert(isStatepoint(I) && "first argument must be a statepoint token");

▲ Show 20 Lines • Show All 71 Lines • Show Last 20 Lines

This is an archive of the discontinued LLVM Phabricator instance.

Extract out a SelectionDAGBuilder::LowerAsStatepoint; NFC
ClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents

Diff 50889

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

llvm/trunk/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

This is an archive of the discontinued LLVM Phabricator instance.

Extract out a SelectionDAGBuilder::LowerAsStatepoint; NFCClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents

Diff 50889

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

llvm/trunk/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

Extract out a SelectionDAGBuilder::LowerAsStatepoint; NFC
ClosedPublic