diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h @@ -103,9 +103,9 @@ return AllocatedStackSlots.test(Offset); } - /// For each statepoint keep mapping from original derived pointer to - /// the statepoint node result defining its new value. - DenseMap DerivedPtrMap; + /// Maps pre-relocated value to virtual register holding it's relocation if + /// vreg lowering was used. + DenseMap VirtRegs; private: /// Maps pre-relocation value (gc pointer directly incoming into statepoint) diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -90,13 +90,13 @@ // FunctionLoweringInfo. Also need to ensure used bits get cleared. AllocatedStackSlots.clear(); AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size()); - DerivedPtrMap.clear(); + VirtRegs.clear(); } void StatepointLoweringState::clear() { Locations.clear(); AllocatedStackSlots.clear(); - DerivedPtrMap.clear(); + VirtRegs.clear(); assert(PendingGCRelocateCalls.empty() && "cleared before statepoint sequence completed"); } @@ -691,8 +691,9 @@ // Conservatively export all values used by gc.relocates outside this // block. This is currently only needed for expressions which don't need - // relocation, but will likely be extended for vreg case shortly. - if (Relocate->getParent() != StatepointInstr->getParent()) { + // relocation (such as constants and allocas). + if (!LowerAsVReg.count(SDV) && + Relocate->getParent() != StatepointInstr->getParent()) { Builder.ExportFromCurrentBlock(V); assert(!LowerAsVReg.count(SDV)); } @@ -857,15 +858,31 @@ SDNode *SinkNode = StatepointMCNode; - // Fill mapping from derived pointer to statepoint result denoting its - // relocated value. - auto &DPtrMap = StatepointLowering.DerivedPtrMap; + // Remember the mapping between values relocated in registers and the virtual + // register holding the relocation. Note that for simplicity, we *always* + // create a vreg even within a single block. + auto &VirtRegs = StatepointLowering.VirtRegs; for (const auto *Relocate : SI.GCRelocates) { Value *Derived = Relocate->getDerivedPtr(); SDValue SD = getValue(Derived); if (!LowerAsVReg.count(SD)) continue; - DPtrMap[Derived] = SDValue(StatepointMCNode, LowerAsVReg[SD]); + + // Handle multiple gc.relocates of the same input efficiently. + if (VirtRegs.count(Derived)) + continue; + + SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]); + + auto *RetTy = Relocate->getType(); + Register Reg = FuncInfo.CreateRegs(RetTy); + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), + DAG.getDataLayout(), Reg, RetTy, None); + SDValue Chain = DAG.getEntryNode(); + RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr); + PendingExports.push_back(Chain); + + VirtRegs[Derived] = Reg; } // Build the GC_TRANSITION_END node if necessary. @@ -1101,6 +1118,22 @@ #endif const Value *DerivedPtr = Relocate.getDerivedPtr(); + + // If relocation was done via virtual register.. + auto &VirtRegs = StatepointLowering.VirtRegs; + auto It = VirtRegs.find(DerivedPtr); + if (It != VirtRegs.end()) { + Register InReg = It->second; + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), + DAG.getDataLayout(), InReg, Relocate.getType(), + None); // This is not an ABI copy. + SDValue Chain = DAG.getEntryNode(); + SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), + Chain, nullptr, nullptr); + setValue(&Relocate, Relocation); + return; + } + SDValue SD = getValue(DerivedPtr); if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) { @@ -1110,17 +1143,6 @@ return; } - // Relocate is local to statepoint block and its pointer was assigned - // to VReg. Use corresponding statepoint result. - auto &DPtrMap = StatepointLowering.DerivedPtrMap; - auto It = DPtrMap.find(DerivedPtr); - if (It != DPtrMap.end()) { - setValue(&Relocate, It->second); - assert(Relocate.getParent() == Relocate.getStatepoint()->getParent() && - "unexpected DPtrMap entry"); - return; - } - auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()]; auto SlotIt = SpillMap.find(DerivedPtr); assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value"); diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.ll b/llvm/test/CodeGen/X86/statepoint-vreg.ll --- a/llvm/test/CodeGen/X86/statepoint-vreg.ll +++ b/llvm/test/CodeGen/X86/statepoint-vreg.ll @@ -310,10 +310,11 @@ ; CHECK-VREG-LABEL: name: test_gcptr_uniqueing ; CHECK-VREG: %0:gr64 = COPY $rdi ; CHECK-VREG: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp -; CHECK-VREG: %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp +; CHECK-VREG: %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp ; CHECK-VREG: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp +; CHECK-VREG: %1:gr64 = COPY %2 ; CHECK-VREG: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp -; CHECK-VREG: $rdi = COPY %1 +; CHECK-VREG: $rdi = COPY %2 ; CHECK-VREG: $rsi = COPY %1 ; CHECK-VREG: CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp