Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -35,6 +35,7 @@ class ConstantRange; class DataLayout; class LLVMContext; +template class UnwindDestIterator; enum AtomicOrdering { NotAtomic = 0, @@ -3706,8 +3707,12 @@ void setNormalDest(BasicBlock *B) { Op<-2>() = reinterpret_cast(B); } - void setUnwindDest(BasicBlock *B) { - Op<-1>() = reinterpret_cast(B); + void setUnwindDest(BasicBlock *B) { Op<-1>() = reinterpret_cast(B); } + + template + iterator_range> + getTransitiveUnwindDests() const { + return UnwindDestIterator::range(getUnwindDest()); } /// getLandingPadInst - Get the landingpad instruction from the landing pad @@ -3915,6 +3920,12 @@ setOperand(1, UnwindDest); } + template + iterator_range> + getTransitiveUnwindDests() const { + return UnwindDestIterator::range(getUnwindDest()); + } + /// getNumHandlers - return the number of 'handlers' in this catchswitch /// instruction, except the default handler unsigned getNumHandlers() const { @@ -4244,6 +4255,12 @@ Op<1>() = NewDest; } + template + iterator_range> + getTransitiveUnwindDests() const { + return UnwindDestIterator::range(getUnwindDest()); + } + // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return (I->getOpcode() == Instruction::CleanupRet); @@ -4822,6 +4839,72 @@ } }; +// TODO: Figure out where this should live +template class UnwindDestIterator { +public: + BasicBlock *operator*() { return CurrentInstr->getParent(); } + UnwindDestIterator &operator++() { + if (CurrentHandler) { + CatchSwitchInst *CatchSwitch = + cast(CurrentInstr)->getCatchSwitch(); + if (++(*CurrentHandler) != CatchSwitch->handler_end()) { + CurrentInstr = (**CurrentHandler)->getFirstNonPHI(); + return *this; + } + CurrentHandler.reset(); + visitUnwindDest(CatchSwitch->getUnwindDest()); + return *this; + } + if (auto *CatchSwitch = dyn_cast(CurrentInstr)) { + visitFirstHandler(CatchSwitch); + return *this; + } + CurrentInstr = nullptr; + return *this; + } + bool operator==(const UnwindDestIterator &Other) const { + if (CurrentInstr != Other.CurrentInstr) + return false; + if (CurrentHandler.hasValue() != Other.CurrentHandler.hasValue()) + return false; + if (CurrentHandler && + (CurrentHandler.getValue() != Other.CurrentHandler.getValue())) + return false; + return true; + } + bool operator!=(const UnwindDestIterator &Other) const { + return !operator==(Other); + } + static iterator_range range(BasicBlock *UnwindDest) { + return iterator_range(UnwindDestIterator(UnwindDest), + UnwindDestIterator()); + } + +private: + explicit UnwindDestIterator() {} + explicit UnwindDestIterator(BasicBlock *UnwindDest) { + visitUnwindDest(UnwindDest); + } + Instruction *CurrentInstr = nullptr; + llvm::Optional CurrentHandler; + + void visitUnwindDest(BasicBlock *UnwindDest) { + assert(!CurrentHandler); + if (!UnwindDest) { + CurrentInstr = nullptr; + return; + } + CurrentInstr = UnwindDest->getFirstNonPHI(); + if (SkipUnsplittable) + if (auto *CatchSwitch = dyn_cast(CurrentInstr)) + visitFirstHandler(CatchSwitch); + } + void visitFirstHandler(CatchSwitchInst *CatchSwitch) { + CurrentHandler = CatchSwitch->handler_begin(); + CurrentInstr = (**CurrentHandler)->getFirstNonPHI(); + } +}; + } // End llvm namespace #endif Index: include/llvm/IR/Statepoint.h =================================================================== --- include/llvm/IR/Statepoint.h +++ include/llvm/IR/Statepoint.h @@ -330,6 +330,8 @@ // This takes care both of relocates for call statepoints and relocates // on normal path of invoke statepoint. if (!isa(Token)) { + assert(!cast(Token)->isEHPad() && + "Funclet pads don't support 1:1 relocate:statepoint mapping"); return cast(Token); } @@ -390,13 +392,21 @@ return Result; // We need to scan thorough exceptional relocations if it is invoke statepoint - LandingPadInst *LandingPad = - cast(getInstruction())->getLandingPadInst(); + const InvokeInst *Invoke = cast(getInstruction()); + + for (BasicBlock *UnwindDest : Invoke->getTransitiveUnwindDests()) { + if (!UnwindDest->isLandingPad()) { + assert(llvm::all_of(UnwindDest->users(), + [](User *U) { return !isa(U); }) && + "Relocates on funclet EH not supported"); + continue; + } - // Search for gc relocates that are attached to this landingpad. - for (const User *LandingPadUser : LandingPad->users()) { - if (auto *Relocate = dyn_cast(LandingPadUser)) - Result.push_back(Relocate); + // Search for gc relocates that are attached to this landingpad. + for (const User *LandingPadUser : UnwindDest->getFirstNonPHI()->users()) { + if (auto *Relocate = dyn_cast(LandingPadUser)) + Result.push_back(Relocate); + } } return Result; } Index: lib/Transforms/Scalar/RewriteStatepointsForGC.cpp =================================================================== --- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -72,6 +72,11 @@ cl::location(ClobberNonLive), cl::Hidden); +static cl::opt SpillOnExceptionPath("rs4gc-spill-on-exception-path", + cl::Hidden, cl::init(false)); +static cl::opt SpillOnNormalPath("rs4gc-spill-on-normal-path", cl::Hidden, + cl::init(false)); + static cl::opt AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true)); @@ -169,17 +174,21 @@ // Generally, after the execution of a full findBasePointer call, only the // base relation will remain. Internally, we add a mixture of the two // types, then update all the second type to the first type +typedef DenseMap BaseMapTy; typedef DenseMap DefiningValueMapTy; typedef DenseSet StatepointLiveSetTy; typedef DenseMap, AssertingVH> - RematerializedValueMapTy; + RematerializedValueMapTy; +typedef DenseMap SpillMapTy; +typedef std::set> FillSetTy; +typedef DenseMap BaseMapMapTy; struct PartiallyConstructedSafepointRecord { /// The set of values known to be live across this safepoint StatepointLiveSetTy LiveSet; /// Mapping from live pointers to a base-defining-value - DenseMap PointerToBase; + BaseMapTy PointerToBase; /// The *new* gc.statepoint instruction itself. This produces the token /// that normal path gc.relocates and the gc.result are tied to. @@ -193,6 +202,28 @@ /// They are not included into 'LiveSet' field. /// Maps rematerialized copy to it's original value. RematerializedValueMapTy RematerializedValues; + + /// Record which values get spilled, and which allocas they get + /// spilled to + SpillMapTy &SpillMap; + + /// Record fills where we spilled a live value instead of relocating. + RematerializedValueMapTy FillMap; + + /// Record which (EH pad, value) pairs we've inserted fills for + FillSetTy &FillSet; + + PartiallyConstructedSafepointRecord(SpillMapTy &SpillMap, FillSetTy &FillSet) + : SpillMap(SpillMap), FillSet(FillSet) {} +}; + +struct PartiallyConstructedSafepointRecordSet { + SmallVector Records; + SpillMapTy SpillMap; + FillSetTy FillSet; + + PartiallyConstructedSafepointRecordSet(size_t size) + : Records(size, PartiallyConstructedSafepointRecord(SpillMap, FillSet)) {} }; } @@ -211,12 +242,15 @@ /// Compute the live-in set for every basic block in the function static void computeLiveInValues(DominatorTree &DT, Function &F, - GCPtrLivenessData &Data); + GCPtrLivenessData &Data, + const BaseMapMapTy *BaseMaps = nullptr); /// Given results from the dataflow liveness computation, find the set of live /// Values at a particular instruction. -static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, - StatepointLiveSetTy &out); +static void findLiveSetAtStatepoint(CallSite Statepoint, + GCPtrLivenessData &Data, + const BaseMapMapTy *BaseMaps, + StatepointLiveSetTy &out); // TODO: Once we can get to the GCStrategy, this becomes // Optional isGCManagedPointer(const Type *Ty) const override { @@ -291,16 +325,17 @@ } // Conservatively identifies any definitions which might be live at the -// given instruction. The analysis is performed immediately before the -// given instruction. Values defined by that instruction are not considered -// live. Values used by that instruction are considered live. +// given instruction. Deopt arguments are treated specially, and considered +// live at the given parse point even though they appear in its argument +// list, to ensure they are reported/relocated. static void analyzeParsePointLiveness( DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, const CallSite &CS, PartiallyConstructedSafepointRecord &result) { - Instruction *inst = CS.getInstruction(); StatepointLiveSetTy LiveSet; - findLiveSetAtInst(inst, OriginalLivenessData, LiveSet); + assert(result.PointerToBase.empty() && + "Not expecting bases to be computed yet"); + findLiveSetAtStatepoint(CS, OriginalLivenessData, nullptr, LiveSet); if (PrintLiveSet) { // Note: This output is used by several of the test cases @@ -1191,8 +1226,9 @@ /// Given an updated version of the dataflow liveness results, update the /// liveset and base pointer maps for the call site CS. static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, + const BaseMapMapTy &BaseMaps, const CallSite &CS, - PartiallyConstructedSafepointRecord &result); + PartiallyConstructedSafepointRecord &Info); static void recomputeLiveInValues( Function &F, DominatorTree &DT, ArrayRef toUpdate, @@ -1200,11 +1236,16 @@ // TODO-PERF: reuse the original liveness, then simply run the dataflow // again. The old values are still live and will help it stabilize quickly. GCPtrLivenessData RevisedLivenessData; - computeLiveInValues(DT, F, RevisedLivenessData); + BaseMapMapTy BaseMaps; + for (size_t I = 0; I < records.size(); I++) { + Instruction *Statepoint = toUpdate[I].getInstruction(); + BaseMaps[Statepoint] = &records[I].PointerToBase; + } + computeLiveInValues(DT, F, RevisedLivenessData, &BaseMaps); for (size_t i = 0; i < records.size(); i++) { struct PartiallyConstructedSafepointRecord &info = records[i]; const CallSite &CS = toUpdate[i]; - recomputeLiveInValues(RevisedLivenessData, CS, info); + recomputeLiveInValues(RevisedLivenessData, BaseMaps, CS, info); } } @@ -1385,7 +1426,6 @@ Instruction *InsertBefore = CS.getInstruction(); IRBuilder<> Builder(InsertBefore); - ArrayRef GCArgs(LiveVariables); uint64_t StatepointID = 0xABCDEF00; uint32_t NumPatchBytes = 0; uint32_t Flags = uint32_t(StatepointFlags::None); @@ -1393,6 +1433,81 @@ ArrayRef CallArgs(CS.arg_begin(), CS.arg_end()); ArrayRef DeoptArgs = GetDeoptBundleOperands(CS); ArrayRef TransitionArgs; + + // Insert any necessary spills, and bulid up the GCArgs which consist of + // the LiveVariables and/or spill slots. + SpillMapTy &SpillMap = Result.SpillMap; + MapVector IncomingPHIValueMap; + ArrayRef GCArgs; + SmallVector ArgBacking; + if (SpillOnNormalPath || CS.isInvoke() && SpillOnExceptionPath) { + // We need to generate spill slots. + + bool SpillOnAllPaths = + SpillOnNormalPath && (SpillOnExceptionPath || !CS.isInvoke()); + + if (SpillOnAllPaths) + GCArgs = None; + + // If not spilling on all paths, the spill slots will need to be + // appended to the GC args, so copy the LiveVariables into ArgBacking; + if (!SpillOnAllPaths) + ArgBacking.append(LiveVariables.begin(), LiveVariables.end()); + + BasicBlock *CallBlock = InsertBefore->getParent(); + Function *F = CallBlock->getParent(); + BasicBlock *EntryBlock = &F->getEntryBlock(); + Instruction *AllocaInsertBefore = &EntryBlock->front(); + + for (Value *Var : LiveVariables) { + AllocaInst *&Alloca = SpillMap[Var]; + if (!Alloca) + Alloca = + new AllocaInst(Var->getType(), suffixed_name_or(Var, ".spill", ""), + AllocaInsertBefore); + ArgBacking.push_back(Alloca); + Builder.CreateStore(Var, Alloca); + } + + if (CS.isInvoke() && SpillOnExceptionPath) { + // We also need to spill the incoming values to any PHIs on the exception + // path + BasicBlock *Pred = CS.getParent(); + for (BasicBlock *UnwindDest : cast(CS.getInstruction()) + ->getTransitiveUnwindDests()) { + for (Instruction &I : *UnwindDest) { + auto *PHI = dyn_cast(&I); + if (!PHI) + break; + // Find/create the Alloca for this PHI + AllocaInst *&Alloca = SpillMap[PHI]; + if (!Alloca) + Alloca = new AllocaInst(PHI->getType(), + suffixed_name_or(PHI, ".spill", ""), + AllocaInsertBefore); + ArgBacking.push_back(Alloca); + // Find/create the value to store for this PHI + Value *IncomingValue = PHI->getIncomingValueForBlock(Pred); + // If the incoming value was itself a PHI we've walked over, + // recurse to that incoming value + auto MapIter = IncomingPHIValueMap.find(IncomingValue); + if (MapIter != IncomingPHIValueMap.end()) + IncomingValue = MapIter->second; + IncomingPHIValueMap[PHI] = IncomingValue; + // Store the value into the spill slot. + Builder.CreateStore(IncomingValue, Alloca); + } + // Update pred if we're going to visit this block's successors, + // which we'll do iff it's a catchswitch. + if (isa(UnwindDest->getFirstNonPHI())) + Pred = UnwindDest; + } + } + + GCArgs = ArrayRef(ArgBacking); + } else { + GCArgs = ArrayRef(LiveVariables); + } if (auto TransitionBundle = CS.getOperandBundle(LLVMContext::OB_gc_transition)) { Flags |= uint32_t(StatepointFlags::GCTransition); @@ -1462,21 +1577,41 @@ Token = Invoke; // Generate gc relocates in exceptional path - BasicBlock *UnwindBlock = ToReplace->getUnwindDest(); - assert(!isa(UnwindBlock->begin()) && - UnwindBlock->getUniquePredecessor() && - "can't safely insert in this block!"); - - Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt()); - Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); - - // Attach exceptional gc relocates to the landingpad. - Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst(); - Result.UnwindToken = ExceptionalToken; + Result.UnwindToken = nullptr; + for (BasicBlock *UnwindBlock : ToReplace->getTransitiveUnwindDests()) { + Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt()); + Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); + + if (SpillOnExceptionPath) { + auto insertSpill = [&, UnwindBlock](Value *Var) { + AllocaInst *SpillSlot = SpillMap[Var]; + if (Result.FillSet.insert({UnwindBlock, SpillSlot}).second) { + LoadInst *Fill = Builder.CreateLoad( + SpillSlot, suffixed_name_or(Var, ".reload", "")); + Result.FillMap[Fill] = Var; + } + }; - const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); - CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, ExceptionalToken, - Builder); + for (Value *LiveVar : LiveVariables) + insertSpill(LiveVar); + for (auto &Pair : IncomingPHIValueMap) + insertSpill(Pair.first); + } else { + assert(!isa(UnwindBlock->begin()) && + UnwindBlock->getUniquePredecessor() && + "can't safely insert in this block!"); + + // Attach exceptional gc relocates to the landingpad. + Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst(); + assert(Result.UnwindToken == nullptr && + "Cannot report multiple unwind tokens"); + Result.UnwindToken = ExceptionalToken; + + const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); + CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, + ExceptionalToken, Builder); + } + } // Generate gc relocates and returns for normal block BasicBlock *NormalDest = ToReplace->getNormalDest(); @@ -1512,8 +1647,16 @@ Result.StatepointToken = Token; // Second, create a gc.relocate for every live variable - const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); - CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, Token, Builder); + if (SpillOnNormalPath) { + for (Value *LiveVar : LiveVariables) { + LoadInst *Fill = Builder.CreateLoad( + SpillMap[LiveVar], suffixed_name_or(LiveVar, ".reload", "")); + Result.FillMap[Fill] = LiveVar; + } + } else { + const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); + CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, Token, Builder); + } } static void StabilizeOrder(SmallVectorImpl &BaseVec, @@ -1616,12 +1759,12 @@ } // Helper function for the "relocationViaAlloca". Similar to the -// "insertRelocationStores" but works for rematerialized values. -static void -insertRematerializationStores( - RematerializedValueMapTy RematerializedValues, - DenseMap &AllocaMap, - DenseSet &VisitedLiveValues) { +// "insertRelocationStores" but works for rematerialized values +// and reloads of spilled values. +static void insertRematerializationStores( + const RematerializedValueMapTy &RematerializedValues, + DenseMap &AllocaMap, + DenseSet &VisitedLiveValues) { for (auto RematerializedValuePair: RematerializedValues) { Instruction *RematerializedValue = RematerializedValuePair.first; @@ -1659,6 +1802,7 @@ SmallVector PromotableAllocas; // Used later to chack that we have enough allocas to store all values std::size_t NumRematerializedValues = 0; + std::size_t NumSpilledValues = 0; PromotableAllocas.reserve(Live.size()); // Emit alloca for "LiveValue" and record it in "allocaMap" and @@ -1675,7 +1819,7 @@ emitAllocaFor(V); // Emit allocas for rematerialized values - for (const auto &Info : Records) + for (const auto &Info : Records) { for (auto RematerializedValuePair : Info.RematerializedValues) { Value *OriginalValue = RematerializedValuePair.second; if (AllocaMap.count(OriginalValue) != 0) @@ -1684,6 +1828,15 @@ emitAllocaFor(OriginalValue); ++NumRematerializedValues; } + for (auto SpilledValuePair : Info.FillMap) { + Value *OriginalValue = SpilledValuePair.second; + if (AllocaMap.count(OriginalValue) != 0) + continue; + + emitAllocaFor(OriginalValue); + ++NumSpilledValues; + } + } // The next two loops are part of the same conceptual operation. We need to // insert a store to the alloca after the original def and at each @@ -1705,7 +1858,7 @@ // In case if it was invoke statepoint // we will insert stores for exceptional path gc relocates. - if (isa(Statepoint)) { + if (isa(Statepoint) && Info.UnwindToken) { insertRelocationStores(Info.UnwindToken->users(), AllocaMap, VisitedLiveValues); } @@ -1714,6 +1867,9 @@ insertRematerializationStores(Info.RematerializedValues, AllocaMap, VisitedLiveValues); + // And with reloads + insertRematerializationStores(Info.FillMap, AllocaMap, VisitedLiveValues); + if (ClobberNonLive) { // As a debugging aid, pretend that an unrelocated pointer becomes null at // the gc.statepoint. This will turn some subtle GC problems into @@ -1745,7 +1901,8 @@ // gc.results and gc.relocates, but that's fine. if (auto II = dyn_cast(Statepoint)) { InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt()); - InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt()); + for (BasicBlock *UnwindDest : II->getTransitiveUnwindDests()) + InsertClobbersAt(&*UnwindDest->getFirstInsertionPt()); } else { InsertClobbersAt(cast(Statepoint)->getNextNode()); } @@ -1816,7 +1973,8 @@ } } - assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues && + assert(PromotableAllocas.size() == + Live.size() + NumRematerializedValues + NumSpilledValues && "we must have the same allocas with lives"); if (!PromotableAllocas.empty()) { // Apply mem2reg to promote alloca to SSA @@ -1841,33 +1999,6 @@ }), Vec.end()); } -/// Insert holders so that each Value is obviously live through the entire -/// lifetime of the call. -static void insertUseHolderAfter(CallSite &CS, const ArrayRef Values, - SmallVectorImpl &Holders) { - if (Values.empty()) - // No values to hold live, might as well not insert the empty holder - return; - - Module *M = CS.getInstruction()->getModule(); - // Use a dummy vararg function to actually hold the values live - Function *Func = cast(M->getOrInsertFunction( - "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true))); - if (CS.isCall()) { - // For call safepoints insert dummy calls right after safepoint - Holders.push_back(CallInst::Create(Func, Values, "", - &*++CS.getInstruction()->getIterator())); - return; - } - // For invoke safepooints insert dummy calls both in normal and - // exceptional destination blocks - auto *II = cast(CS.getInstruction()); - Holders.push_back(CallInst::Create( - Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt())); - Holders.push_back(CallInst::Create( - Func, Values, "", &*II->getUnwindDest()->getFirstInsertionPt())); -} - static void findLiveReferences( Function &F, DominatorTree &DT, ArrayRef toUpdate, MutableArrayRef records) { @@ -2185,16 +2316,16 @@ Instruction *NormalInsertBefore = &*Invoke->getNormalDest()->getFirstInsertionPt(); - Instruction *UnwindInsertBefore = - &*Invoke->getUnwindDest()->getFirstInsertionPt(); - Instruction *NormalRematerializedValue = rematerializeChain(NormalInsertBefore); - Instruction *UnwindRematerializedValue = - rematerializeChain(UnwindInsertBefore); - Info.RematerializedValues[NormalRematerializedValue] = LiveValue; - Info.RematerializedValues[UnwindRematerializedValue] = LiveValue; + + for (BasicBlock *UnwindDest : Invoke->getTransitiveUnwindDests()) { + Instruction *UnwindInsertBefore = &*UnwindDest->getFirstInsertionPt(); + Instruction *UnwindRematerializedValue = + rematerializeChain(UnwindInsertBefore); + Info.RematerializedValues[UnwindRematerializedValue] = LiveValue; + } } } @@ -2219,38 +2350,21 @@ // When inserting gc.relocates for invokes, we need to be able to insert at // the top of the successor blocks. See the comment on - // normalForInvokeSafepoint on exactly what is needed. Note that this step + // normalizeForInvokeSafepoint on exactly what is needed. Note that this step // may restructure the CFG. for (CallSite CS : ToUpdate) { if (!CS.isInvoke()) continue; auto *II = cast(CS.getInstruction()); normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT); - normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT); + if (!SpillOnExceptionPath) + for (BasicBlock *UnwindDest : II->getTransitiveUnwindDests()) + normalizeForInvokeSafepoint(UnwindDest, II->getParent(), DT); } - // A list of dummy calls added to the IR to keep various values obviously - // live in the IR. We'll remove all of these when done. - SmallVector Holders; - - // Insert a dummy call with all of the arguments to the vm_state we'll need - // for the actual safepoint insertion. This ensures reference arguments in - // the deopt argument list are considered live through the safepoint (and - // thus makes sure they get relocated.) - for (CallSite CS : ToUpdate) { - SmallVector DeoptValues; - - for (Value *Arg : GetDeoptBundleOperands(CS)) { - assert(!isUnhandledGCPointerType(Arg->getType()) && - "support for FCA unimplemented"); - if (isHandledGCPointerType(Arg->getType())) - DeoptValues.push_back(Arg); - } - - insertUseHolderAfter(CS, DeoptValues, Holders); - } - - SmallVector Records(ToUpdate.size()); + PartiallyConstructedSafepointRecordSet RecordSet(ToUpdate.size()); + MutableArrayRef Records = + RecordSet.Records; // A) Identify all gc pointers which are statically live at the given call // site. @@ -2269,7 +2383,13 @@ } } // end of cache scope - // The base phi insertion logic (for any safepoint) may have inserted new + // We insert some dummy calls after each safepoint to definitely hold live + // the base pointers which were identified for that safepoint. We'll then + // ask liveness for _every_ base inserted to see what is now live. Then we + // remove the dummy calls. + + // By selecting base pointers, we've effectively inserted new uses, because + // the base phi insertion logic (for any safepoint) may have inserted new // instructions which are now live at some safepoint. The simplest such // example is: // loop: @@ -2278,24 +2398,8 @@ // gep a + 1 // safepoint 2 // br loop - // We insert some dummy calls after each safepoint to definitely hold live - // the base pointers which were identified for that safepoint. We'll then - // ask liveness for _every_ base inserted to see what is now live. Then we - // remove the dummy calls. - Holders.reserve(Holders.size() + Records.size()); - for (size_t i = 0; i < Records.size(); i++) { - PartiallyConstructedSafepointRecord &Info = Records[i]; - - SmallVector Bases; - for (auto Pair : Info.PointerToBase) - Bases.push_back(Pair.second); - - insertUseHolderAfter(ToUpdate[i], Bases, Holders); - } - - // By selecting base pointers, we've effectively inserted new uses. Thus, we - // need to rerun liveness. We may *also* have inserted new defs, but that's - // not the key issue. + // Thus, we need to rerun liveness. We may *also* have inserted new defs, + // but that's not the key issue. recomputeLiveInValues(F, DT, ToUpdate, Records); if (PrintBasePointers) { @@ -2324,11 +2428,6 @@ if (isa(BasePair.second)) Info.LiveSet.erase(BasePair.first); - for (CallInst *CI : Holders) - CI->eraseFromParent(); - - Holders.clear(); - // Do a limited scalarization of any live at safepoint vector values which // contain pointers. This enables this pass to run after vectorization at // the cost of some possible performance loss. Note: This is known to not @@ -2422,8 +2521,12 @@ #ifndef NDEBUG // sanity check for (auto *Ptr : Live) - assert(isHandledGCPointerType(Ptr->getType()) && - "must be a gc pointer type"); + if (auto *Alloca = dyn_cast(Ptr)) + assert(isHandledGCPointerType(Alloca->getAllocatedType()) && + "spill must be a gc pointer type"); + else + assert(isHandledGCPointerType(Ptr->getType()) && + "ptr value must be a gc pointer type"); #endif relocationViaAlloca(F, DT, Live, Records); @@ -2628,7 +2731,8 @@ /// the live-out set of the basic block static void computeLiveInValues(BasicBlock::reverse_iterator rbegin, BasicBlock::reverse_iterator rend, - DenseSet &LiveTmp) { + const BaseMapMapTy *BaseMaps, + StatepointLiveSetTy &LiveTmp) { for (BasicBlock::reverse_iterator ritr = rbegin; ritr != rend; ritr++) { Instruction *I = &*ritr; @@ -2659,6 +2763,15 @@ LiveTmp.insert(V); } } + + // If we've computed live references and the current instruction happens + // to be a statepoint, add its bases to the live set. + if (BaseMaps && CallSite(I)) { + auto BaseMapIter = BaseMaps->find(I); + if (BaseMapIter != BaseMaps->end()) + for (auto &Pair : *BaseMapIter->second) + LiveTmp.insert(Pair.second); + } } } @@ -2716,7 +2829,8 @@ #endif static void computeLiveInValues(DominatorTree &DT, Function &F, - GCPtrLivenessData &Data) { + GCPtrLivenessData &Data, + const BaseMapMapTy *BaseMaps) { SmallSetVector Worklist; auto AddPredsToWorklist = [&](BasicBlock *BB) { @@ -2733,7 +2847,7 @@ for (BasicBlock &BB : F) { Data.KillSet[&BB] = computeKillSet(&BB); Data.LiveSet[&BB].clear(); - computeLiveInValues(BB.rbegin(), BB.rend(), Data.LiveSet[&BB]); + computeLiveInValues(BB.rbegin(), BB.rend(), BaseMaps, Data.LiveSet[&BB]); #ifndef NDEBUG for (Value *Kill : Data.KillSet[&BB]) @@ -2793,31 +2907,42 @@ #endif } -static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data, - StatepointLiveSetTy &Out) { - +static void findLiveSetAtStatepoint(CallSite Statepoint, + GCPtrLivenessData &Data, + const BaseMapMapTy *BaseMaps, + StatepointLiveSetTy &Out) { + Instruction *Inst = Statepoint.getInstruction(); BasicBlock *BB = Inst->getParent(); // Note: The copy is intentional and required assert(Data.LiveOut.count(BB)); DenseSet LiveOut = Data.LiveOut[BB]; - // We want to handle the statepoint itself oddly. It's - // call result is not live (normal), nor are it's arguments - // (unless they're used again later). This adjustment is - // specifically what we need to relocate + // We want to handle the statepoint itself oddly. Its + // call result is not live (normal), nor are its non-deopt + // arguments (unless they're used again later). This + // adjustment is specifically what we need to relocate. BasicBlock::reverse_iterator rend(Inst->getIterator()); - computeLiveInValues(BB->rbegin(), rend, LiveOut); + computeLiveInValues(BB->rbegin(), rend, BaseMaps, LiveOut); LiveOut.erase(Inst); Out.insert(LiveOut.begin(), LiveOut.end()); + + // Ensures reference arguments in the deopt argument list are considered live + // through the safepoint (and thus make sure they get relocated.) + for (Value *Arg : GetDeoptBundleOperands(Statepoint)) { + assert(!isUnhandledGCPointerType(Arg->getType()) && + "support for FCA unimplemented"); + if (isHandledGCPointerType(Arg->getType())) + Out.insert(Arg); + } } static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, + const BaseMapMapTy &BaseMaps, const CallSite &CS, PartiallyConstructedSafepointRecord &Info) { - Instruction *Inst = CS.getInstruction(); StatepointLiveSetTy Updated; - findLiveSetAtInst(Inst, RevisedLivenessData, Updated); + findLiveSetAtStatepoint(CS, RevisedLivenessData, &BaseMaps, Updated); #ifndef NDEBUG DenseSet Bases;