[pseudo] Use the TokenIndex for the lookahead in GuardParams

clang-tools-extra/pseudo/include/clang-pseudo/GLR.h

 //
 // Exposed for testing only.
 void glrShift(llvm::ArrayRef<const GSS::Node *> OldHeads,
               const ForestNode &NextTok, const ParseParams &Params,
               const Language &Lang, std::vector<const GSS::Node *> &NewHeads);
 // Applies available reductions on Heads, appending resulting heads to the list.
 //
 // Exposed for testing only.
-void glrReduce(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead,
-               const ParseParams &Params, const Language &Lang);
+void glrReduce(std::vector<const GSS::Node *> &Heads,
+               Token::Index LookaheadIndex, const ParseParams &Params,
+               const Language &Lang);
 
 // Heuristically recover from a state where no further parsing is possible.
 //
 // OldHeads is the parse state at TokenIndex.
 // This function consumes zero or more tokens by advancing TokenIndex,
 // and places any recovery states created in NewHeads.
 //
 // On failure, NewHeads is empty and TokenIndex is unchanged.

clang-tools-extra/pseudo/include/clang-pseudo/Language.h

 namespace pseudo {
 class ForestNode;
 class TokenStream;
 class LRTable;
 
 struct GuardParams {
   llvm::ArrayRef<const ForestNode *> RHS;
   const TokenStream &Tokens;
-  // FIXME: use the index of Tokens.
-  SymbolID Lookahead;
+  Token::Index LookaheadIndex = 0;
 };
 // A guard restricts when a grammar rule can be used.
 //
 // The GLR parser will use the guard to determine whether a rule reduction will
 // be conducted. For example, e.g. a guard may allow the rule
 // `virt-specifier := IDENTIFIER` only if the identifier's text is 'override`.
 //
 // Return true if the guard is satisfied.

clang-tools-extra/pseudo/lib/GLR.cpp

struct PushSpec {
     Sequence *Seq = nullptr;
   };
   KeyedQueue<Family, PushSpec> Sequences; // FIXME: rename => PendingPushes?
 
   // We treat Heads as a queue of Pop operations still to be performed.
   // PoppedHeads is our position within it.
   std::vector<const GSS::Node *> *Heads;
   unsigned NextPopHead;
+
   SymbolID Lookahead;
+  Token::Index LookaheadIndex;
 
   Sequence TempSequence;
 public:
   GLRReduce(const ParseParams &Params, const Language &Lang)
       : Params(Params), Lang(Lang) {}
 
-  void operator()(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead) {
-    assert(isToken(Lookahead));
-
+  void operator()(std::vector<const GSS::Node *> &Heads,
+                  Token::Index LookaheadPos) {
     NextPopHead = 0;
     this->Heads = &Heads;
-    this->Lookahead = Lookahead;
+    this->LookaheadIndex = LookaheadPos;
+    this->Lookahead =
+        tokenSymbol(LookaheadPos >= Params.Code.tokens().size()
+                        ? tok::eof
+                        : Params.Code.tokens()[LookaheadPos].Kind);
     assert(Sequences.empty());
     SequenceStorageCount = 0;
 
     popPending();
     while (!Sequences.empty()) {
       pushNext();
       popPending();
     }
   }
 
 private:
   bool canReduce(const Rule &R, RuleID RID,
                  llvm::ArrayRef<const ForestNode *> RHS) const {
     if (!R.Guarded)
       return true;
     if (auto Guard = Lang.Guards.lookup(RID))
-      return Guard({RHS, Params.Code, Lookahead});
+      return Guard({RHS, Params.Code, LookaheadIndex});
     LLVM_DEBUG(llvm::dbgs()
                << llvm::formatv("missing guard implementation for rule {0}\n",
                                 Lang.G.dumpRule(RID)));
     return true;
   }
   // pop walks up the parent chain(s) for a reduction from Head by to Rule.
   // Once we reach the end, record the bases and sequences.
   void pop(const GSS::Node *Head, RuleID RID, const Rule &Rule) {
if (NextHeads.empty()) {
       if (NextHeads.empty())
         // FIXME: Ensure the `_ := start-symbol` rules have a fallback
         // error-recovery strategy attached. Then this condition can't happen.
         return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0);
     } else
       ++I;
 
     // Form nonterminals containing the token we just consumed.
-    SymbolID Lookahead =
-        I == Terminals.size() ? tokenSymbol(tok::eof) : Terminals[I].symbol();
-    Reduce(NextHeads, Lookahead);
+    Reduce(NextHeads, I);
     // Prepare for the next token.
     std::swap(Heads, NextHeads);
     NextHeads.clear();
     MaybeGC();
   }
   LLVM_DEBUG(llvm::dbgs() << llvm::formatv("Reached eof\n"));
 
   // The parse was successful if we're in state `_ := start-symbol .`
const ForestNode &glrParse(const ParseParams &Params, SymbolID StartSymbol,
   // Failed to parse the input, attempt to run recovery.
   // FIXME: this awkwardly repeats the recovery in the loop, when shift fails.
   // More elegant is to include EOF in the token stream, and make the
   // augmented rule: `_ := translation-unit EOF`. In this way recovery at EOF
   // would not be a special case: it show up as a failure to shift the EOF
   // token.
   unsigned I = Terminals.size();
   glrRecover(Heads, I, Params, Lang, NextHeads);
-  Reduce(NextHeads, tokenSymbol(tok::eof));
+  Reduce(NextHeads, I);
   if (auto *Result = SearchForAccept(NextHeads))
     return *Result;
 
   // We failed to parse the input, returning an opaque forest node for recovery.
   // FIXME: as above, we can add fallback error handling so this is impossible.
   return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0);
 }
 
-void glrReduce(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead,
+void glrReduce(std::vector<const GSS::Node *> &Heads, Token::Index LookaheadIndex,
                const ParseParams &Params, const Language &Lang) {
   // Create a new GLRReduce each time for tests, performance doesn't matter.
-  GLRReduce{Params, Lang}(Heads, Lookahead);
+  GLRReduce{Params, Lang}(Heads, LookaheadIndex);
 }
 
 const GSS::Node *GSS::addNode(LRTable::StateID State, const ForestNode *Symbol,
 
                               llvm::ArrayRef<const Node *> Parents) {
   Node *Result = new (allocate(Parents.size()))
       Node({State, GCParity, static_cast<uint16_t>(Parents.size())});
   Alive.push_back(Result);

clang-tools-extra/pseudo/lib/cxx/CXX.cpp

default:
       assert(false && "unhandled declarator for IsFunction");
       return IsFunction;
     }
   }
   llvm_unreachable("unreachable");
 }
 
 bool guardNextTokenNotElse(const GuardParams &P) {
-  return symbolToToken(P.Lookahead) != tok::kw_else;
+  if (P.LookaheadIndex >= P.Tokens.tokens().size())
+    return true;
+  return P.Tokens.tokens()[P.LookaheadIndex].Kind != tok::kw_else;
 }
 
 // Whether this e.g. decl-specifier contains an "exclusive" type such as a class
 // name, and thus can't combine with a second exclusive type.
 //
 // Returns false for
 //  - non-types
 //  - "unsigned" etc that may suffice as types but may modify others

clang-tools-extra/pseudo/unittests/GLRTest.cpp

void build(llvm::StringRef GrammarBNF) {
     TestLang.G = Grammar::parseBNF(GrammarBNF, Diags);
   }
 
   TokenStream emptyTokenStream() {
     TokenStream Empty;
     Empty.finalize();
     return Empty;
   }
- 
+  TokenStream singleTokenStream(tok::TokenKind Kind) {
+    TokenStream TS;
+    Token T;
+    T.Kind = Kind;
+    TS.push(std::move(T));
+    TS.finalize();
+    return TS;
+  }
   void buildGrammar(std::vector<std::string> Nonterminals,
                     std::vector<std::string> Rules) {
     Nonterminals.push_back("_");
     llvm::sort(Nonterminals);
     Nonterminals.erase(std::unique(Nonterminals.begin(), Nonterminals.end()),
                        Nonterminals.end());
     std::string FakeTestBNF;
     for (const auto &NT : Nonterminals)
TEST_F(GLRTest, ReduceConflictsSplitting) {
   TestLang.Table = std::move(B).build();
 
   const auto *GSSNode0 =
       GSStack.addNode(/*State=*/0, /*ForestNode=*/nullptr, /*Parents=*/{});
   const auto *GSSNode1 =
       GSStack.addNode(1, &Arena.createTerminal(tok::identifier, 0), {GSSNode0});
 
   std::vector<const GSS::Node *> Heads = {GSSNode1};
-  glrReduce(Heads, tokenSymbol(tok::eof),
+  glrReduce(Heads, /*eof*/0,
             {emptyTokenStream(), Arena, GSStack}, TestLang);
   EXPECT_THAT(Heads, UnorderedElementsAre(
                          GSSNode1,
                          AllOf(state(2), parsedSymbolID(id("class-name")),
                                parents({GSSNode0})),
                          AllOf(state(3), parsedSymbolID(id("enum-name")),
                                parents({GSSNode0}))))
       << Heads;
TEST_F(GLRTest, ReduceSplittingDueToMultipleBases) {
 
   LRTable::Builder B(TestLang.G);
   B.Transition[{StateID{2}, id("ptr-operator")}] = StateID{5};
   B.Transition[{StateID{3}, id("ptr-operator")}] = StateID{6};
   B.Reduce[StateID{4}].insert(ruleFor("ptr-operator"));
   TestLang.Table = std::move(B).build();
 
   std::vector<const GSS::Node *> Heads = {GSSNode4};
-  glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack},
+  glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack},
             TestLang);
 
   EXPECT_THAT(Heads, UnorderedElementsAre(
                          GSSNode4,
                          AllOf(state(5), parsedSymbolID(id("ptr-operator")),
                                parents({GSSNode2})),
                          AllOf(state(6), parsedSymbolID(id("ptr-operator")),
                                parents({GSSNode3}))))
TEST_F(GLRTest, ReduceJoiningWithMultipleBases) {
   LRTable::Builder B(TestLang.G);
   B.Transition[{StateID{1}, id("type-name")}] = StateID{5};
   B.Transition[{StateID{2}, id("type-name")}] = StateID{5};
   B.Reduce[StateID{3}].insert(/* type-name := class-name */ RuleID{0});
   B.Reduce[StateID{4}].insert(/* type-name := enum-name */ RuleID{1});
   TestLang.Table = std::move(B).build();
 
   std::vector<const GSS::Node *> Heads = {GSSNode3, GSSNode4};
-  glrReduce(Heads, tokenSymbol(tok::eof), {emptyTokenStream(), Arena, GSStack},
+  glrReduce(Heads, /*eof*/0, {emptyTokenStream(), Arena, GSStack},
             TestLang);
 
   // Verify that the stack heads are joint at state 5 after reduces.
   EXPECT_THAT(Heads, UnorderedElementsAre(GSSNode3, GSSNode4,
                                           AllOf(state(5),
                                                 parsedSymbolID(id("type-name")),
                                                 parents({GSSNode1, GSSNode2}))))
       << Heads;
TEST_F(GLRTest, ReduceJoiningWithSameBase) {
   // FIXME: figure out a way to get rid of the hard-coded reduce RuleID!
   LRTable::Builder B(TestLang.G);
   B.Transition[{StateID{0}, id("pointer")}] = StateID{5};
   B.Reduce[StateID{3}].insert(/* pointer := class-name */ RuleID{0});
   B.Reduce[StateID{4}].insert(/* pointer := enum-name */ RuleID{1});
   TestLang.Table = std::move(B).build();
 
   std::vector<const GSS::Node *> Heads = {GSSNode3, GSSNode4};
-  glrReduce(Heads, tokenSymbol(tok::eof),
+  glrReduce(Heads, /*eof*/0,
             {emptyTokenStream(), Arena, GSStack}, TestLang);
 
   EXPECT_THAT(
       Heads, UnorderedElementsAre(GSSNode3, GSSNode4,
                                   AllOf(state(5), parsedSymbolID(id("pointer")),
                                         parents({GSSNode0}))))
       << Heads;
   EXPECT_EQ(Heads.back()->Payload->dumpRecursive(TestLang.G),
TEST_F(GLRTest, ReduceLookahead) {
 
   const auto *Root =
       GSStack.addNode(/*State=*/0, /*ForestNode=*/nullptr, /*Parents=*/{});
   const auto *GSSNode1 =
       GSStack.addNode(/*State=*/1, /*ForestNode=*/Identifier, {Root});
 
   // When the lookahead is +, reduce is performed.
   std::vector<const GSS::Node *> Heads = {GSSNode1};
-  glrReduce(Heads, tokenSymbol(tok::plus), {emptyTokenStream(), Arena, GSStack},
+  glrReduce(Heads, 0, {singleTokenStream(tok::plus), Arena, GSStack},
             TestLang);
   EXPECT_THAT(Heads,
               ElementsAre(GSSNode1, AllOf(state(2), parsedSymbolID(id("term")),
                                           parents(Root))));
 
   // When the lookahead is -, reduce is not performed.
   Heads = {GSSNode1};
-  glrReduce(Heads, tokenSymbol(tok::minus),
-            {emptyTokenStream(), Arena, GSStack}, TestLang);
+  glrReduce(Heads, 0,
+            {singleTokenStream(tok::minus), Arena, GSStack}, TestLang);
   EXPECT_THAT(Heads, ElementsAre(GSSNode1));
 }
 
 TEST_F(GLRTest, Recover) {
   // Recovery while parsing "word" inside braces.
   //  Before:
   //    0--1({)--2(?)
   //  After recovering a `word` at state 1: