[pseudo] Apply the EOF fallback error-recovery strategy by default.

[hokein]

Summary

Address 2 FIXMEs.

clang-tools-extra/pseudo/lib/GLR.cpp

if (NextHeads.empty()) {
       // We discard all heads formed by reduction, and recreate them without
       // this constraint. This may duplicate some nodes, but it's rare.
       LLVM_DEBUG(llvm::dbgs() << "Shift failed, will attempt recovery. "
                                  "Re-reducing without lookahead.\n");
       Heads.resize(HeadsPartition);
       Reduce(Heads, /*allow all reductions*/ tokenSymbol(tok::unknown));
 
       glrRecover(Heads, I, Params, Lang, NextHeads);
-      if (NextHeads.empty())
-        // FIXME: Ensure the `_ := start-symbol` rules have a fallback
-        // error-recovery strategy attached. Then this condition can't happen.
-        return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0);
+      assert(!NextHeads.empty() &&  "no fallback recovery strategy!" );
     } else
       ++I;
 
     // Form nonterminals containing the token we just consumed.
     SymbolID Lookahead =
         I == Terminals.size() ? tokenSymbol(tok::eof) : Terminals[I].symbol();
     HeadsPartition = NextHeads.size();
     Reduce(NextHeads, Lookahead);
for (const auto *Head : Heads) {
         assert(Head->Payload->symbol() == tokenSymbol(tok::eof));
         assert(Result == nullptr && "multiple results!");
         Result = Head->parents().front()->Payload;
         assert(Result->symbol() == StartSymbol);
       }
     }
     return Result;
   };
-  if (auto *Result = SearchForAccept(Heads))
-    return *Result;
-  // We failed to parse the input, returning an opaque forest node for recovery.
-  // FIXME: as above, we can add fallback error handling so this is impossible.
-  return Params.Forest.createOpaque(StartSymbol, /*Token::Index=*/0);
+  auto *Result = SearchForAccept(Heads);
+  assert(Result);
 }
 
 void glrReduce(std::vector<const GSS::Node *> &Heads, SymbolID Lookahead,
                const ParseParams &Params, const Language &Lang) {
   // Create a new GLRReduce each time for tests, performance doesn't matter.
   GLRReduce{Params, Lang}(Heads, Lookahead);
 }
 

clang-tools-extra/pseudo/lib/cxx/CXX.cpp

assert(Left.Kind == tok::l_brace || Left.Kind == tok::l_paren ||
          Left.Kind == tok::l_square);
   if (const Token *Right = Left.pair()) {
     assert(Tokens.index(*Right) > Begin - 1);
     return Tokens.index(*Right);
   }
   return Token::Invalid;
 }
 
+Token::Index recoverEof(Token::Index, const TokenStream &Tokens) {
+  return Tokens.tokens().size();
+}
+
 llvm::DenseMap<ExtensionID, RecoveryStrategy> buildRecoveryStrategies() {
   return {
       {Extension::Brackets, recoverBrackets},
+      {Extension::Eof, recoverEof},
   };
 }
 
 } // namespace
 
 const Language &getLanguage() {
   static const auto &CXXLanguage = []() -> const Language & {
     std::vector<std::string> Diags;

clang-tools-extra/pseudo/lib/grammar/GrammarBNF.cpp

Grammar build(llvm::StringRef BNF) {
     assert(T->Nonterminals.size() < (1 << (SymbolBits - 1)) &&
            "Too many nonterminals to fit in SymbolID bits!");
     llvm::sort(T->Nonterminals, [](const GrammarTable::Nonterminal &L,
                                    const GrammarTable::Nonterminal &R) {
       return L.Name < R.Name;
     });
     // Add an empty string for the corresponding sentinel unset attribute.
     T->AttributeValues.push_back("");
+    // Add the fallback EOF error recovery strategy.
+    UniqueAttributeValues.insert("Eof");
     UniqueAttributeValues.erase("");
     for (llvm::StringRef Name : UniqueAttributeValues) {
       T->AttributeValues.emplace_back();
       T->AttributeValues.back() = Name.str();
     }
     llvm::sort(T->AttributeValues);
     assert(T->AttributeValues.front() == "");
 
for (unsigned I = 0; I < Spec.Sequence.size(); ++I) {
           R.Recovery = LookupExtensionID(KV.second);
           R.RecoveryIndex = I;
         } else {
           Diagnostics.push_back(
               llvm::formatv("Unknown attribute '{0}'", KV.first).str());
         }
       }
     }
+    if (Spec.Target == StartSymbol) {
+      R.Recovery = LookupExtensionID("Eof");
+      R.RecoveryIndex = 0;
+    }
   }
 
   // Inlines all _opt symbols.
   // For example, a rule E := id +_opt id, after elimination, we have two
   // equivalent rules:
   //   1) E := id + id
   //   2) E := id id
   std::vector<RuleSpec> eliminateOptional(llvm::ArrayRef<RuleSpec> Input) {

clang-tools-extra/pseudo/unittests/GLRTest.cpp

EXPECT_EQ(Parsed.dumpRecursive(TestLang.G),
             "[  0, end) sentence := word word [recover=AcceptAnyTokenInstead]\n"
             "[  0,   1) ├─word := IDENTIFIER\n"
             "[  0,   1) │ └─IDENTIFIER := tok[0]\n"
             "[  1, end) └─word := <opaque>\n");
 }
 
 TEST_F(GLRTest, RecoveryFromStartOfInput) {
   build(R"bnf(
-    _ := start [recover=Fallback] EOF
+    _ := start EOF
 
     start := IDENTIFIER
   )bnf");
   TestLang.Table = LRTable::buildSLR(TestLang.G);
   bool fallback_recovered = false;
-  auto fallback = [&](Token::Index Start, const TokenStream & Code) {
+  auto fallback = [&](Token::Index Start, const TokenStream &Code) {
     fallback_recovered = true;
     return Code.tokens().size();
   };
-  TestLang.RecoveryStrategies.try_emplace(
-      extensionID("Fallback"),
-      fallback);
+  TestLang.RecoveryStrategies.try_emplace(extensionID("Eof"), fallback);
   clang::LangOptions LOptions;
   TokenStream Tokens = cook(lex("?", LOptions), LOptions);
 
   const ForestNode &Parsed =
       glrParse({Tokens, Arena, GSStack}, id("start"), TestLang);
   EXPECT_TRUE(fallback_recovered);
-  EXPECT_EQ(Parsed.dumpRecursive(TestLang.G),
-            "[  0, end) start := <opaque>\n");
+  EXPECT_EQ(Parsed.dumpRecursive(TestLang.G), "[  0, end) start := <opaque>\n");
 }
 
 TEST_F(GLRTest, RepeatedRecovery) {
   // We require multiple steps of recovery at eof and then a reduction in order
   // to successfully parse.
   build(R"bnf(
     _ := function EOF
     # FIXME: this forces EOF to be in follow(signature).
TEST_F(GLRTest, GuardExtension) {
   std::string Input = "test";
   const TokenStream &Succeeded = cook(lex(Input, LOptions), LOptions);
   EXPECT_EQ(glrParse({Succeeded, Arena, GSStack}, id("start"), TestLang)
                 .dumpRecursive(TestLang.G),
             "[  0, end) start := IDENTIFIER [guard]\n"
             "[  0, end) └─IDENTIFIER := tok[0]\n");
 
   Input = "notest";
+  TestLang.RecoveryStrategies.try_emplace(
+      extensionID("Eof"), [](Token::Index Begin, const TokenStream &Code) {
+        return Code.tokens().size();
+      });
   const TokenStream &Failed = cook(lex(Input, LOptions), LOptions);
   EXPECT_EQ(glrParse({Failed, Arena, GSStack}, id("start"), TestLang)
                 .dumpRecursive(TestLang.G),
             "[  0, end) start := <opaque>\n");
 }
 
 TEST(GSSTest, GC) {
   //      ┌-A-┬-AB