Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -864,12 +864,16 @@ /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before /// giving up. The TopologicalPrune flag signals that positive NodeIds are /// topologically ordered (Operands have strictly smaller node id) and search - /// can be pruned leveraging this. + /// can be pruned leveraging this. If BailOut is not nullptr, it will return + /// whether the searching bails out. static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl &Visited, SmallVectorImpl &Worklist, unsigned int MaxSteps = 0, - bool TopologicalPrune = false) { + bool TopologicalPrune = false, + bool *BailOut = nullptr) { + if (BailOut) + *BailOut = false; SmallVector DeferredNodes; if (Visited.count(N)) return true; @@ -914,8 +918,11 @@ // Push deferred nodes back on worklist. Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); // If we bailed early, conservatively return found. - if (MaxSteps != 0 && Visited.size() >= MaxSteps) + if (MaxSteps != 0 && Visited.size() >= MaxSteps) { + if (BailOut) + *BailOut = true; return true; + } return Found; } Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -115,6 +115,11 @@ "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors")); +static cl::opt StoreMergeDependenceLimit( + "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), + cl::desc("Limit the number of times for the same StoreNode and RootNode " + "to bail out in store merging dependence check")); + namespace { class DAGCombiner { @@ -152,6 +157,14 @@ /// which have not yet been combined to the worklist. SmallPtrSet CombinedNodes; + /// Map from candidate StoreNode to the pair of RootNode and count. + /// The count is used to track how many times we have seen the StoreNode + /// with the same RootNode bail out in dependence check. If we have seen + /// the bail out for the same pair many times over a limit, we won't + /// consider the StoreNode with the same RootNode as store merging + /// candidate again. + DenseMap> StoreRootCountMap; + // AA - Used for DAG load/store alias analysis. AliasAnalysis *AA; @@ -236,6 +249,7 @@ void removeFromWorklist(SDNode *N) { CombinedNodes.erase(N); PruningList.remove(N); + StoreRootCountMap.erase(N); auto It = WorklistMap.find(N); if (It == WorklistMap.end()) @@ -15425,6 +15439,19 @@ return (BasePtr.equalBaseIndex(Ptr, DAG, Offset)); }; + // Check if the pair of StoreNode and the RootNode already bail out many + // times which is over the limit in dependence check. + auto OverLimitInDependenceCheck = [&](SDNode *StoreNode, + SDNode *RootNode) -> bool { + return false; + auto RootCount = StoreRootCountMap.find(StoreNode); + if (RootCount != StoreRootCountMap.end() && + RootCount->second.first == RootNode && + RootCount->second.second > StoreMergeDependenceLimit) + return true; + return false; + }; + // We looking for a root node which is an ancestor to all mergable // stores. We search up through a load, to our root and then down // through all children. For instance we will find Store{1,2,3} if @@ -15454,7 +15481,8 @@ if (StoreSDNode *OtherST = dyn_cast(*I2)) { BaseIndexOffset Ptr; int64_t PtrDiff; - if (CandidateMatch(OtherST, Ptr, PtrDiff)) + if (CandidateMatch(OtherST, Ptr, PtrDiff) && + !OverLimitInDependenceCheck(OtherST, RootNode)) StoreNodes.push_back(MemOpLink(OtherST, PtrDiff)); } } else @@ -15464,7 +15492,8 @@ if (StoreSDNode *OtherST = dyn_cast(*I)) { BaseIndexOffset Ptr; int64_t PtrDiff; - if (CandidateMatch(OtherST, Ptr, PtrDiff)) + if (CandidateMatch(OtherST, Ptr, PtrDiff) && + !OverLimitInDependenceCheck(OtherST, RootNode)) StoreNodes.push_back(MemOpLink(OtherST, PtrDiff)); } } @@ -15520,10 +15549,19 @@ Worklist.push_back(N->getOperand(j).getNode()); } // Search through DAG. We can stop early if we find a store node. + bool BailOut; for (unsigned i = 0; i < NumStores; ++i) if (SDNode::hasPredecessorHelper(StoreNodes[i].MemNode, Visited, Worklist, - Max)) + Max, false, &BailOut)) { + // If BailOut is seen, record the StoreNode and RootNode in the + // StoreRootCountMap. If we have seen the pair many times over a limit, + // we won't add the StoreNode into StoreNodes set again. + if (BailOut) { + auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode]; + RootCount = {RootNode, RootCount.second + 1}; + } return false; + } return true; }