diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h b/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h --- a/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h @@ -140,8 +140,9 @@ // Applies available reductions on Heads, appending resulting heads to the list. // // Exposed for testing only. -void glrReduce(std::vector &Heads, SymbolID Lookahead, - const ParseParams &Params, const Language &Lang); +void glrReduce(std::vector &Heads, + Token::Index LookaheadIndex, const ParseParams &Params, + const Language &Lang); // Heuristically recover from a state where no further parsing is possible. // diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Language.h b/clang-tools-extra/pseudo/include/clang-pseudo/Language.h --- a/clang-tools-extra/pseudo/include/clang-pseudo/Language.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/Language.h @@ -22,8 +22,7 @@ struct GuardParams { llvm::ArrayRef RHS; const TokenStream &Tokens; - // FIXME: use the index of Tokens. - SymbolID Lookahead; + Token::Index LookaheadIndex = 0; }; // A guard restricts when a grammar rule can be used. // diff --git a/clang-tools-extra/pseudo/lib/GLR.cpp b/clang-tools-extra/pseudo/lib/GLR.cpp --- a/clang-tools-extra/pseudo/lib/GLR.cpp +++ b/clang-tools-extra/pseudo/lib/GLR.cpp @@ -392,19 +392,24 @@ // PoppedHeads is our position within it. std::vector *Heads; unsigned NextPopHead; + SymbolID Lookahead; + Token::Index LookaheadIndex; Sequence TempSequence; public: GLRReduce(const ParseParams &Params, const Language &Lang) : Params(Params), Lang(Lang) {} - void operator()(std::vector &Heads, SymbolID Lookahead) { - assert(isToken(Lookahead)); - + void operator()(std::vector &Heads, + Token::Index LookaheadPos) { NextPopHead = 0; this->Heads = &Heads; - this->Lookahead = Lookahead; + this->LookaheadIndex = LookaheadPos; + this->Lookahead = + tokenSymbol(LookaheadPos >= Params.Code.tokens().size() + ? tok::eof + : Params.Code.tokens()[LookaheadPos].Kind); assert(Sequences.empty()); SequenceStorageCount = 0; @@ -421,7 +426,7 @@ if (!R.Guarded) return true; if (auto Guard = Lang.Guards.lookup(RID)) - return Guard({RHS, Params.Code, Lookahead}); + return Guard({RHS, Params.Code, LookaheadIndex}); LLVM_DEBUG(llvm::dbgs() << llvm::formatv("missing guard implementation for rule {0}\n", Lang.G.dumpRule(RID))); @@ -630,9 +635,7 @@ ++I; // Form nonterminals containing the token we just consumed. - SymbolID Lookahead = - I == Terminals.size() ? tokenSymbol(tok::eof) : Terminals[I].symbol(); - Reduce(NextHeads, Lookahead); + Reduce(NextHeads, I); // Prepare for the next token. std::swap(Heads, NextHeads); NextHeads.clear(); @@ -664,7 +667,7 @@ // token. unsigned I = Terminals.size(); glrRecover(Heads, I, Params, Lang, NextHeads); - Reduce(NextHeads, tokenSymbol(tok::eof)); + Reduce(NextHeads, I); if (auto *Result = SearchForAccept(NextHeads)) return *Result; @@ -673,10 +676,10 @@ return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0); } -void glrReduce(std::vector &Heads, SymbolID Lookahead, +void glrReduce(std::vector &Heads, Token::Index LookaheadIndex, const ParseParams &Params, const Language &Lang) { // Create a new GLRReduce each time for tests, performance doesn't matter. - GLRReduce{Params, Lang}(Heads, Lookahead); + GLRReduce{Params, Lang}(Heads, LookaheadIndex); } const GSS::Node *GSS::addNode(LRTable::StateID State, const ForestNode *Symbol, diff --git a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp --- a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp +++ b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp @@ -159,7 +159,9 @@ } bool guardNextTokenNotElse(const GuardParams &P) { - return symbolToToken(P.Lookahead) != tok::kw_else; + if (P.LookaheadIndex >= P.Tokens.tokens().size()) + return true; + return P.Tokens.tokens()[P.LookaheadIndex].Kind != tok::kw_else; } // Whether this e.g. decl-specifier contains an "exclusive" type such as a class diff --git a/clang-tools-extra/pseudo/unittests/GLRTest.cpp b/clang-tools-extra/pseudo/unittests/GLRTest.cpp --- a/clang-tools-extra/pseudo/unittests/GLRTest.cpp +++ b/clang-tools-extra/pseudo/unittests/GLRTest.cpp @@ -71,7 +71,15 @@ Empty.finalize(); return Empty; } - + TokenStream singleTokenStream(tok::TokenKind Kind) { + TokenStream TS; + Token T; + T.Kind = Kind; + TS.push(std::move(T)); + TS.finalize(); + return TS; + } + void buildGrammar(std::vector Nonterminals, std::vector Rules) { Nonterminals.push_back("_"); @@ -185,7 +193,7 @@ GSStack.addNode(1, &Arena.createTerminal(tok::identifier, 0), {GSSNode0}); std::vector Heads = {GSSNode1}; - glrReduce(Heads, tokenSymbol(tok::eof), + glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack}, TestLang); EXPECT_THAT(Heads, UnorderedElementsAre( GSSNode1, @@ -224,7 +232,7 @@ TestLang.Table = std::move(B).build(); std::vector Heads = {GSSNode4}; - glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack}, + glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack}, TestLang); EXPECT_THAT(Heads, UnorderedElementsAre( @@ -277,7 +285,7 @@ TestLang.Table = std::move(B).build(); std::vector Heads = {GSSNode3, GSSNode4}; - glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack}, + glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack}, TestLang); // Verify that the stack heads are joint at state 5 after reduces. @@ -332,7 +340,7 @@ TestLang.Table = std::move(B).build(); std::vector Heads = {GSSNode3, GSSNode4}; - glrReduce(Heads, tokenSymbol(tok::eof), + glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack}, TestLang); EXPECT_THAT( @@ -367,7 +375,7 @@ // When the lookahead is +, reduce is performed. std::vector Heads = {GSSNode1}; - glrReduce(Heads, tokenSymbol(tok::plus), {emptyTokenStream(), Arena, GSStack}, + glrReduce(Heads, 0, {singleTokenStream(tok::plus), Arena, GSStack}, TestLang); EXPECT_THAT(Heads, ElementsAre(GSSNode1, AllOf(state(2), parsedSymbolID(id("term")), @@ -375,8 +383,8 @@ // When the lookahead is -, reduce is not performed. Heads = {GSSNode1}; - glrReduce(Heads, tokenSymbol(tok::minus), - {emptyTokenStream(), Arena, GSStack}, TestLang); + glrReduce(Heads, 0, + {singleTokenStream(tok::minus), Arena, GSStack}, TestLang); EXPECT_THAT(Heads, ElementsAre(GSSNode1)); }