diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h b/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h
--- a/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h
+++ b/clang-tools-extra/pseudo/include/clang-pseudo/GLR.h
@@ -140,8 +140,9 @@
// Applies available reductions on Heads, appending resulting heads to the list.
//
// Exposed for testing only.
-void glrReduce(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead,
- const ParseParams &Params, const Language &Lang);
+void glrReduce(std::vector<const GSS::Node *> &Heads,
+ Token::Index LookaheadIndex, const ParseParams &Params,
+ const Language &Lang);
// Heuristically recover from a state where no further parsing is possible.
//
diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Language.h b/clang-tools-extra/pseudo/include/clang-pseudo/Language.h
--- a/clang-tools-extra/pseudo/include/clang-pseudo/Language.h
+++ b/clang-tools-extra/pseudo/include/clang-pseudo/Language.h
@@ -22,8 +22,7 @@
struct GuardParams {
llvm::ArrayRef<const ForestNode *> RHS;
const TokenStream &Tokens;
- // FIXME: use the index of Tokens.
- SymbolID Lookahead;
+ Token::Index LookaheadIndex = 0;
};
// A guard restricts when a grammar rule can be used.
//
diff --git a/clang-tools-extra/pseudo/lib/GLR.cpp b/clang-tools-extra/pseudo/lib/GLR.cpp
--- a/clang-tools-extra/pseudo/lib/GLR.cpp
+++ b/clang-tools-extra/pseudo/lib/GLR.cpp
@@ -392,19 +392,24 @@
// PoppedHeads is our position within it.
std::vector<const GSS::Node *> *Heads;
unsigned NextPopHead;
+
SymbolID Lookahead;
+ Token::Index LookaheadIndex;
Sequence TempSequence;
public:
GLRReduce(const ParseParams &Params, const Language &Lang)
: Params(Params), Lang(Lang) {}
- void operator()(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead) {
- assert(isToken(Lookahead));
-
+ void operator()(std::vector<const GSS::Node *> &Heads,
+ Token::Index LookaheadPos) {
NextPopHead = 0;
this->Heads = &Heads;
- this->Lookahead = Lookahead;
+ this->LookaheadIndex = LookaheadPos;
+ this->Lookahead =
+ tokenSymbol(LookaheadPos >= Params.Code.tokens().size()
+ ? tok::eof
+ : Params.Code.tokens()[LookaheadPos].Kind);
assert(Sequences.empty());
SequenceStorageCount = 0;
@@ -421,7 +426,7 @@
if (!R.Guarded)
return true;
if (auto Guard = Lang.Guards.lookup(RID))
- return Guard({RHS, Params.Code, Lookahead});
+ return Guard({RHS, Params.Code, LookaheadIndex});
LLVM_DEBUG(llvm::dbgs()
<< llvm::formatv("missing guard implementation for rule {0}\n",
Lang.G.dumpRule(RID)));
@@ -630,9 +635,7 @@
++I;
// Form nonterminals containing the token we just consumed.
- SymbolID Lookahead =
- I == Terminals.size() ? tokenSymbol(tok::eof) : Terminals[I].symbol();
- Reduce(NextHeads, Lookahead);
+ Reduce(NextHeads, I);
// Prepare for the next token.
std::swap(Heads, NextHeads);
NextHeads.clear();
@@ -664,7 +667,7 @@
// token.
unsigned I = Terminals.size();
glrRecover(Heads, I, Params, Lang, NextHeads);
- Reduce(NextHeads, tokenSymbol(tok::eof));
+ Reduce(NextHeads, I);
if (auto *Result = SearchForAccept(NextHeads))
return *Result;
@@ -673,10 +676,10 @@
return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0);
}
-void glrReduce(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead,
+void glrReduce(std::vector<const GSS::Node *> &Heads, Token::Index LookaheadIndex,
const ParseParams &Params, const Language &Lang) {
// Create a new GLRReduce each time for tests, performance doesn't matter.
- GLRReduce{Params, Lang}(Heads, Lookahead);
+ GLRReduce{Params, Lang}(Heads, LookaheadIndex);
}
const GSS::Node *GSS::addNode(LRTable::StateID State, const ForestNode *Symbol,
diff --git a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
--- a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
+++ b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
@@ -159,7 +159,9 @@
}
bool guardNextTokenNotElse(const GuardParams &P) {
- return symbolToToken(P.Lookahead) != tok::kw_else;
+ if (P.LookaheadIndex >= P.Tokens.tokens().size())
+ return true;
+ return P.Tokens.tokens()[P.LookaheadIndex].Kind != tok::kw_else;
}
// Whether this e.g. decl-specifier contains an "exclusive" type such as a class
diff --git a/clang-tools-extra/pseudo/unittests/GLRTest.cpp b/clang-tools-extra/pseudo/unittests/GLRTest.cpp
--- a/clang-tools-extra/pseudo/unittests/GLRTest.cpp
+++ b/clang-tools-extra/pseudo/unittests/GLRTest.cpp
@@ -71,7 +71,15 @@
Empty.finalize();
return Empty;
}
-
+ TokenStream singleTokenStream(tok::TokenKind Kind) {
+ TokenStream TS;
+ Token T;
+ T.Kind = Kind;
+ TS.push(std::move(T));
+ TS.finalize();
+ return TS;
+ }
+
void buildGrammar(std::vector<std::string> Nonterminals,
std::vector<std::string> Rules) {
Nonterminals.push_back("_");
@@ -185,7 +193,7 @@
GSStack.addNode(1, &Arena.createTerminal(tok::identifier, 0), {GSSNode0});
std::vector<const GSS::Node *> Heads = {GSSNode1};
- glrReduce(Heads, tokenSymbol(tok::eof),
+ glrReduce(Heads, /*eof*/0,
{emptyTokenStream(), Arena, GSStack}, TestLang);
EXPECT_THAT(Heads, UnorderedElementsAre(
GSSNode1,
@@ -224,7 +232,7 @@
TestLang.Table = std::move(B).build();
std::vector<const GSS::Node *> Heads = {GSSNode4};
- glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack},
+ glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack},
TestLang);
EXPECT_THAT(Heads, UnorderedElementsAre(
@@ -277,7 +285,7 @@
TestLang.Table = std::move(B).build();
std::vector<const GSS::Node *> Heads = {GSSNode3, GSSNode4};
- glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack},
+ glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack},
TestLang);
// Verify that the stack heads are joint at state 5 after reduces.
@@ -332,7 +340,7 @@
TestLang.Table = std::move(B).build();
std::vector<const GSS::Node *> Heads = {GSSNode3, GSSNode4};
- glrReduce(Heads, tokenSymbol(tok::eof),
+ glrReduce(Heads, /*eof*/0,
{emptyTokenStream(), Arena, GSStack}, TestLang);
EXPECT_THAT(
@@ -367,7 +375,7 @@
// When the lookahead is +, reduce is performed.
std::vector<const GSS::Node *> Heads = {GSSNode1};
- glrReduce(Heads, tokenSymbol(tok::plus), {emptyTokenStream(), Arena, GSStack},
+ glrReduce(Heads, 0, {singleTokenStream(tok::plus), Arena, GSStack},
TestLang);
EXPECT_THAT(Heads,
ElementsAre(GSSNode1, AllOf(state(2), parsedSymbolID(id("term")),
@@ -375,8 +383,8 @@
// When the lookahead is -, reduce is not performed.
Heads = {GSSNode1};
- glrReduce(Heads, tokenSymbol(tok::minus),
- {emptyTokenStream(), Arena, GSStack}, TestLang);
+ glrReduce(Heads, 0,
+ {singleTokenStream(tok::minus), Arena, GSStack}, TestLang);
EXPECT_THAT(Heads, ElementsAre(GSSNode1));
}