[pseudo] Generate an enum type for identifying grammar rules.

[hokein]

Summary

The Rule enum type enables us to identify a grammar rule within C++'s
type system.

[sammccall]

clang-tools-extra/pseudo/include/clang-pseudo/grammar/Grammar.h

172

I hope you mean kw_int? I'd like this to compile :-)

Can you also give the example for IDENTIFIER?

clang-tools-extra/pseudo/lib/grammar/Grammar.cpp

48

I'm not sure exposing these from Grammar is an improvement, i think even in principle we'd only want to use these for codegen.

The need for testing is real, but i guess you can add a test that uses the CXX generated grammar, assert the elements of Rule::whatever, and the name of Symbol::whatever?

[hokein]

update.

clang-tools-extra/pseudo/include/clang-pseudo/grammar/Grammar.h

172

As discussed offline, we stick to the current version without kw_ prefix. Added some comments for this decision.

clang-tools-extra/pseudo/lib/grammar/Grammar.cpp

48

Yeah, these functions are only used in the codegen. I'm inlined to the put it into the Grammar(it also has a symbolName definition, I'd prefer to have all naming functions in a single place).

[sammccall]

clang-tools-extra/pseudo/lib/grammar/Grammar.cpp

48

That's exactly my point though: that symbolname is general purpose and should be used everywhere, which is why it belongs on grammar.

Vs the mangled name that literally nobody should be using for anything.

clang-tools-extra/pseudo/gen/Main.cpp

int main(int argc, char *argv[]) {
   }
 
   switch (Emit) {
   case EmitSymbolList:
     Out.os() << R"cpp(
 #ifndef NONTERMINAL
 #define NONTERMINAL(X, Y)
 #endif
+#ifndef RULE
+#define RULE(X, Y)
+#endif
 #ifndef EXTENSION
 #define EXTENSION(X, Y)
 #endif
-    )cpp";
     for (clang::pseudo::SymbolID ID = 0; ID < G.table().Nonterminals.size();
-         ++ID) {
-      std::string Name = G.symbolName(ID).str();
-      // translation-unit -> translation_unit
-      std::replace(Name.begin(), Name.end(), '-', '_');
-      Out.os() << llvm::formatv("NONTERMINAL({0}, {1})\n", Name, ID);
-    }
+         ++ID)
+      Out.os() << llvm::formatv("NONTERMINAL({0}, {1})\n", G.mangleSymbol(ID),
+                                ID);
+    for (clang::pseudo::RuleID RID = 0; RID < G.table().Rules.size(); ++RID)
+      Out.os() << llvm::formatv("RULE({0}, {1})\n", G.mangleRule(RID), RID);
     for (clang::pseudo::ExtensionID EID = 1 /*skip the sentinel 0 value*/;
          EID < G.table().AttributeValues.size(); ++EID) {
       llvm::StringRef Name = G.table().AttributeValues[EID];
       assert(!Name.empty());
       Out.os() << llvm::formatv("EXTENSION({0}, {1})\n", Name, EID);
     }
     Out.os() << R"cpp(
 #undef NONTERMINAL
+#undef RULE
 #undef EXTENSION
-    )cpp";
     break;
   case EmitGrammarContent:
     for (llvm::StringRef Line : llvm::split(GrammarText, '\n')) {
       Out.os() << '"';
       Out.os().write_escaped((Line + "\n").str());
       Out.os() << "\"\n";
     }
     break;
   }
 
   Out.keep();
 
   return 0;
 }

clang-tools-extra/pseudo/include/clang-pseudo/cxx/CXX.h

 // Symbol represents nonterminal symbols in the C++ grammar.
 // It provides a simple uniform way to access a particular nonterminal.
 enum class Symbol : SymbolID {
 #define NONTERMINAL(X, Y) X = Y,
 #include "CXXSymbols.inc"
 #undef NONTERMINAL
 };
 
+enum class Rule : RuleID {
+#define RULE(X, Y) X = Y,
+#include "CXXSymbols.inc"
+#undef RULE
+};
+
 enum class Extension : ExtensionID {
 #define EXTENSION(X, Y) X = Y,
 #include "CXXSymbols.inc"
 #undef EXTENSION
 };
 
 // Returns the Language for the cxx.bnf grammar.
 const Language &getLanguage();
 
 } // namespace cxx
 
 } // namespace pseudo
 } // namespace clang
 
 #endif // CLANG_PSEUDO_CXX_CXX_H

clang-tools-extra/pseudo/include/clang-pseudo/grammar/Grammar.h

public:
   // Returns all rules of the given nonterminal symbol.
   llvm::ArrayRef<Rule> rulesFor(SymbolID SID) const;
   const Rule &lookupRule(RuleID RID) const;
 
   // Gets symbol (terminal or nonterminal) name.
   // Terminals have names like "," (kw_comma) or "OPERATOR" (kw_operator).
   llvm::StringRef symbolName(SymbolID) const;
 
+  // Gets the mangled name for a terminal/nonterminal.
+  // Compared to names in the grammar,
+  //   nonterminals `ptr-declartor` becomes `ptr_declarator`;
+  //   terminal `,` becomes `comma`;
+  //   terminal `IDENTIFIER` becomes `identifier`;
+  //   terminal `INT` becomes `int`;
+  // NOTE: for nonterminals, the mangled name is the same as the cxx::Symbol
+  // enum class; for terminals, we deliberately stripped the `kw_` prefix in
+  // favor of the simplicity.
+  std::string mangleSymbol(SymbolID) const;
+  // Gets the mangled name for the rule.
+  // E.g. for the grammar rule `ptr-declarator := ptr-operator ptr-declarator`,
+  // it is `ptr_declarator_0ptr_operator_1ptr_declarator`.
+  std::string mangleRule(RuleID) const;
+
   // Lookup the SymbolID of the nonterminal symbol by Name.
   llvm::Optional<SymbolID> findNonterminal(llvm::StringRef Name) const;
 
   // Dumps the whole grammar.
   std::string dump() const;
   // Dumps a particular rule.
   std::string dumpRule(RuleID) const;
   // Dumps all rules of the given nonterminal symbol.

clang-tools-extra/pseudo/lib/grammar/Grammar.cpp

 }
 
 llvm::StringRef Grammar::symbolName(SymbolID SID) const {
   if (isToken(SID))
     return T->Terminals[symbolToToken(SID)];
   return T->Nonterminals[SID].Name;
 }
 
+std::string Grammar::mangleSymbol(SymbolID SID) const {
+  static const char *const TokNames[] = {
+#define TOK(X) #X,
+#define KEYWORD(X, Y) #X,
+#include "clang/Basic/TokenKinds.def"
+      nullptr};
+  if (clang::pseudo::isToken(SID))
+    return TokNames[clang::pseudo::symbolToToken(SID)];
+  std::string Name = symbolName(SID).str();
+  // translation-unit -> translation_unit
+  std::replace(Name.begin(), Name.end(), '-', '_');
+  return Name;
+}
+
+std::string Grammar::mangleRule(RuleID RID) const {
+  const auto &R = lookupRule(RID);
+  std::string MangleName = mangleSymbol(R.Target);
+  for (size_t I = 0; I < R.seq().size(); ++I)
+    MangleName += llvm::formatv("_{0}{1}", I, mangleSymbol(R.seq()[I]));
+  return MangleName;
+}
+
 llvm::Optional<SymbolID> Grammar::findNonterminal(llvm::StringRef Name) const {
   auto It = llvm::partition_point(
       T->Nonterminals,
       [&](const GrammarTable::Nonterminal &X) { return X.Name < Name; });
   if (It != T->Nonterminals.end() && It->Name == Name)
     return It - T->Nonterminals.begin();
   return llvm::None;
 }

clang-tools-extra/pseudo/unittests/GrammarTest.cpp

TEST_F(GrammarTest, Annotation) {
   )bnf");
   ASSERT_TRUE(Diags.empty());
   EXPECT_EQ(G.lookupRule(ruleFor("_")).Guard, 0);
   EXPECT_GT(G.lookupRule(ruleFor("x")).Guard, 0);
   EXPECT_GT(G.lookupRule(ruleFor("y")).Guard, 0);
   EXPECT_NE(G.lookupRule(ruleFor("x")).Guard, G.lookupRule(ruleFor("y")).Guard);
 }
 
+TEST_F(GrammarTest, MangleName) {
+  build(R"bnf(
+    _ := declaration
+
+    declaration := ptr-declarator ;
+    ptr-declarator := * IDENTIFIER
+
+  )bnf");
+  ASSERT_TRUE(Diags.empty());
+  EXPECT_EQ(G.mangleRule(ruleFor("declaration")),
+            "declaration_0ptr_declarator_1semi");
+  EXPECT_EQ(G.mangleRule(ruleFor("ptr-declarator")),
+            "ptr_declarator_0star_1identifier");
+}
+
 TEST_F(GrammarTest, Diagnostics) {
   build(R"cpp(
     _ := ,_opt
     _ := undefined-sym
     null :=
     _ := IDENFIFIE # a typo of the terminal IDENFITIER
 
     invalid