Index: llvm/include/llvm/CodeGen/SelectionDAG.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAG.h +++ llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1154,6 +1154,12 @@ SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef Ops); + /// If \p Vals contains 64k values or more, move values into new + /// TokenFactors in 64k-1 blocks, until \p Vals contains less than 64k values. + /// Use this function for operand lists exceeding the maximum number of + /// operands for SDValue nodes. + void splitAcrossTokenFactors(const SDLoc &DL, SmallVector &Vals); + /// *Mutate* the specified machine node's memory references to the provided /// list. void setNodeMemRefs(MachineSDNode *N, Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9287,6 +9287,18 @@ checkForCycles(Node); } +void SelectionDAG::splitAcrossTokenFactors(const SDLoc &DL, + SmallVector &Vals) { + size_t Limit = std::numeric_limits::max(); + while (Vals.size() > Limit) { + unsigned SliceIdx = Vals.size() - Limit; + auto ExtractedTFs = ArrayRef(Vals).slice(SliceIdx, Limit); + SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); + Vals.erase(Vals.begin() + SliceIdx, Vals.end()); + Vals.emplace_back(NewTF); + } +} + #ifndef NDEBUG static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl &Visited, Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1032,19 +1032,9 @@ } // Otherwise, we have to make a token factor node. - // If we have >= 2^16 loads then split across multiple token factors as - // there's a 64k limit on the number of SDNode operands. - SDValue Root; - size_t Limit = (1 << 16) - 1; - while (PendingLoads.size() > Limit) { - unsigned SliceIdx = PendingLoads.size() - Limit; - auto ExtractedTFs = ArrayRef(PendingLoads).slice(SliceIdx, Limit); - SDValue NewTF = - DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, ExtractedTFs); - PendingLoads.erase(PendingLoads.begin() + SliceIdx, PendingLoads.end()); - PendingLoads.emplace_back(NewTF); - } - Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads); + DAG.splitAcrossTokenFactors(getCurSDLoc(), PendingLoads); + SDValue Root = + DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads); PendingLoads.clear(); DAG.setRoot(Root); return Root;