Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -606,6 +606,10 @@ void visit(unsigned Opcode, const User &I); + /// getCopyFromRegs - If there was virtual register allocated for the value V + /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. + SDValue getCopyFromRegs(const Value *V, Type *Ty); + // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, // generate the debug data structures now that we've seen its definition. void resolveDanglingDebugInfo(const Value *V, SDValue Val); @@ -662,7 +666,9 @@ void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); // This function is responsible for the whole statepoint lowering process. - void LowerStatepoint(ImmutableStatepoint Statepoint); + // It uniformly handles invoke and call statepoints. + void LowerStatepoint(ImmutableStatepoint Statepoint, + MachineBasicBlock *LandingPad = nullptr); private: std::pair lowerInvokable( TargetLowering::CallLoweringInfo &CLI, Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1016,6 +1016,24 @@ } } +/// getCopyFromRegs - If there was virtual register allocated for the value V +/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. +SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { + DenseMap::iterator It = FuncInfo.ValueMap.find(V); + SDValue res; + + if (It != FuncInfo.ValueMap.end()) { + unsigned InReg = It->second; + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg, + Ty); + SDValue Chain = DAG.getEntryNode(); + res = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); + resolveDanglingDebugInfo(V, res); + } + + return res; +} + /// getValue - Return an SDValue for the given Value. SDValue SelectionDAGBuilder::getValue(const Value *V) { // If we already have an SDValue for this value, use it. It's important @@ -1026,15 +1044,9 @@ // If there's a virtual register allocated and initialized for this // value, use it. - DenseMap::iterator It = FuncInfo.ValueMap.find(V); - if (It != FuncInfo.ValueMap.end()) { - unsigned InReg = It->second; - RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg, - V->getType()); - SDValue Chain = DAG.getEntryNode(); - N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); - resolveDanglingDebugInfo(V, N); - return N; + SDValue copyFromReg = getCopyFromRegs(V, V->getType()); + if (copyFromReg.getNode()) { + return copyFromReg; } // Otherwise create a new SDValue and remember it. Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -224,6 +224,7 @@ /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, + MachineBasicBlock *LandingPad, SelectionDAGBuilder &Builder) { ImmutableCallSite CS(StatepointSite.getCallSite()); @@ -245,15 +246,24 @@ Tmp->setTailCall(CS.isTailCall()); Tmp->setCallingConv(CS.getCallingConv()); Tmp->setAttributes(CS.getAttributes()); - Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false); + Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false, LandingPad); // Handle the return value of the call iff any. const bool HasDef = !Tmp->getType()->isVoidTy(); if (HasDef) { - // The value of the statepoint itself will be the value of call itself. - // We'll replace the actually call node shortly. gc_result will grab - // this value. - Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp)); + if (CS.isInvoke()) { + // Result value will be used in different basic block for invokes + // so we need to export it now. + unsigned reg = Builder.FuncInfo.CreateRegs(Tmp->getType()); + Builder.CopyValueToVirtualRegister(Tmp, reg); + Builder.FuncInfo.ValueMap[CS.getInstruction()] = reg; + } + else { + // The value of the statepoint itself will be the value of call itself. + // We'll replace the actually call node shortly. gc_result will grab + // this value. + Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp)); + } } else { // The token value is never used from here on, just generate a poison value Builder.setValue(CS.getInstruction(), Builder.DAG.getIntPtrConstant(-1)); @@ -267,6 +277,15 @@ // Search for the call node // The following code is essentially reverse engineering X86's // LowerCallTo. + // We are expecting DAG to have the following form: + // ch = eh_label (only in case of invoke statepoint) + // ch, glue = callseq_start ch + // ch, glue = X86::Call ch, glue + // ch, glue = callseq_end ch, glue + // ch = eh_label ch (only in case of invoke statepoint) + // + // DAG root will be either last eh_label or callseq_end. + SDNode *CallNode = nullptr; // We just emitted a call, so it should be last thing generated @@ -276,8 +295,11 @@ SDNode *CallEnd = Chain.getNode(); int Sanity = 0; while (CallEnd->getOpcode() != ISD::CALLSEQ_END) { - CallEnd = CallEnd->getGluedNode(); - assert(CallEnd && "Can not find call node"); + assert(CallEnd->getNumOperands() >= 1 && + CallEnd->getOperand(0).getValueType() == MVT::Other); + + CallEnd = CallEnd->getOperand(0).getNode(); + assert(Sanity < 20 && "should have found call end already"); Sanity++; } @@ -506,7 +528,9 @@ LowerStatepoint(ImmutableStatepoint(&CI)); } -void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP) { +void +SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, + MachineBasicBlock *LandingPad/*=nullptr*/) { // The basic scheme here is that information about both the original call and // the safepoint is encoded in the CallInst. We create a temporary call and // lower it, then reverse engineer the calling sequence. @@ -542,13 +566,12 @@ } #endif - // Lower statepoint vmstate and gcstate arguments SmallVector LoweredArgs; lowerStatepointMetaArgs(LoweredArgs, ISP, *this); // Get call node, we will replace it later with statepoint - SDNode *CallNode = lowerCallFromStatepoint(ISP, *this); + SDNode *CallNode = lowerCallFromStatepoint(ISP, LandingPad, *this); // Construct the actual STATEPOINT node with all the appropriate arguments // and return values. @@ -638,7 +661,23 @@ assert(isStatepoint(I) && "first argument must be a statepoint token"); - setValue(&CI, getValue(I)); + if (isa(I)) { + // For invokes we should have stored call result in a virtual register. + // We can not use default getValue() functionality to copy value from this register + // because statepoint and actuall call return types can be different, + // and getValue() will use CopyFromReg of the wrong type (always i32 in our case) + PointerType *calleeType = cast( + ImmutableStatepoint(I).actualCallee()->getType()); + Type *retTy = cast( + calleeType->getElementType())->getReturnType(); + SDValue copyFromReg = getCopyFromRegs(I, retTy); + + assert(copyFromReg.getNode()); + setValue(&CI, copyFromReg); + } + else { + setValue(&CI, getValue(I)); + } } void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {