[pseudo] Allow opaque nodes to represent terminals

[sammccall]

Summary

This allows incomplete code such as namespace foo { to be modeled as a
normal sequence with the missing } represented by an empty opaque node.

[hokein]

The change looks good, are you planning to use it in the cxx.bnf?

[sammccall]

In D130551#3679021, @hokein wrote:

The change looks good, are you planning to use it in the cxx.bnf?

I want to land a recovery rule that completes scopes (e.g. namespaces) rather than discarding them.

But i have to fix several deficiencies, at least this one, D130550, D130523. and at some point we have to allow rules to have more than one recovery point.

I like having the motivating case in the patch itself but in this case there's just too much of this stuff

clang-tools-extra/pseudo/lib/GLR.cpp

void glrRecover(llvm::ArrayRef<const GSS::Node *> OldHeads,
   // and we risk creating redundant Forest and GSS nodes.
   // We also may inadvertently set up the next glrReduce to create a sequence
   // node duplicating an opaque node that we're creating here.
   // There are various options, including simply breaking ties between options.
   // For now it's obscure enough to ignore.
   for (const PlaceholderRecovery *Option : BestOptions) {
     const ForestNode &Placeholder =
         Params.Forest.createOpaque(Option->Symbol, RecoveryRange->Begin);
-    const GSS::Node *NewHead = Params.GSStack.addNode(
-        *Lang.Table.getGoToState(Option->RecoveryNode->State, Option->Symbol),
-        &Placeholder, {Option->RecoveryNode});
+    LRTable::StateID OldState = Option->RecoveryNode->State;
+    LRTable::StateID NewState =
+        isToken(Option->Symbol)
+            ? *Lang.Table.getShiftState(OldState, Option->Symbol)
+            : *Lang.Table.getGoToState(OldState, Option->Symbol);
+    const GSS::Node *NewHead =
+        Params.GSStack.addNode(NewState, &Placeholder, {Option->RecoveryNode});
     NewHeads.push_back(NewHead);
   }
   TokenIndex = RecoveryRange->End;
 }
 
 using StateID = LRTable::StateID;
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const GSS::Node &N) {

clang-tools-extra/pseudo/unittests/GLRTest.cpp

EXPECT_EQ(Parsed.dumpRecursive(TestLang.G),
             "[  4,   5) │ │ └─} := tok[4]\n"
             "[  1,   5) │ └─block := { numbers [recover=Braces] }\n"
             "[  1,   2) │   ├─{ := tok[1]\n"
             "[  2,   4) │   ├─numbers := <opaque>\n"
             "[  4,   5) │   └─} := tok[4]\n"
             "[  5, end) └─} := tok[5]\n");
 }
 
+TEST_F(GLRTest, RecoverTerminal) {
+  build(R"bnf(
+    _ := stmt
+
+    stmt := IDENTIFIER ; [recover=Skip]
+  )bnf");
+  TestLang.Table = LRTable::buildSLR(TestLang.G);
+  TestLang.RecoveryStrategies.try_emplace(
+      extensionID("Skip"),
+      [](Token::Index Start, const TokenStream &) { return Start + 1; });
+  clang::LangOptions LOptions;
+  TokenStream Tokens = cook(lex("foo", LOptions), LOptions);
+
+  const ForestNode &Parsed =
+      glrParse({Tokens, Arena, GSStack}, id("stmt"), TestLang);
+  EXPECT_EQ(Parsed.dumpRecursive(TestLang.G),
+            "[  0, end) stmt := IDENTIFIER ; [recover=Skip]\n"
+            "[  0,   1) ├─IDENTIFIER := tok[0]\n"
+            "[  1, end) └─; := <opaque>\n");
+}
+
+
 TEST_F(GLRTest, NoExplicitAccept) {
   build(R"bnf(
     _ := test
 
     test := IDENTIFIER test
     test := IDENTIFIER
   )bnf");
   clang::LangOptions LOptions;