[Tooling] Add stdlib::Symbol::all() and stdlib::Symbol::qualified_name()

[sammccall]

Summary

These address some remaining reasons to #include StdSymbolMap.inc directly.

[sammccall]

oops, helps if I run tests on the right branch

[hokein]

clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp

19

nit: we group five symbols together, it is a bit hard to follow.

Can you move the HeaderNames just before the HeaderIDs and mention the value of the HeaderIDs is the index of HeaderNames?

113

should be SymbolCount.

[sammccall]

clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp

19

Rearranged these and added comments to each.

113

Oops, thanks!

clang/include/clang/Tooling/Inclusions/StandardLibrary.h

 class Symbol;
 
 // A standard library header, such as <iostream>
 // Lightweight class, in fact just an index into a table.
 // C++ and C Library compatibility headers are considered different: e.g.
 // "<cstdio>" and "<stdio.h>" (and their symbols) are treated differently.
 class Header {
 public:
+  static std::vector<Header> all();
   // Name should contain the angle brackets, e.g. "<vector>".
   static std::optional<Header> named(llvm::StringRef Name);
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Header &H) {
     return OS << H.name();
   }
   llvm::StringRef name() const;
 

 // A top-level standard library symbol, such as std::vector
 // Lightweight class, in fact just an index into a table.
 // C++ and C Standard Library symbols are considered distinct: e.g. std::printf
 // and ::printf are not treated as the same symbol.
 // The symbols do not contain macros right now, we don't have a reliable index
 // for them.
 class Symbol {
 public:
+  static std::vector<Symbol> all();
   /// \p Scope should have the trailing "::", for example:
   /// named("std::chrono::", "system_clock")
   static std::optional<Symbol> named(llvm::StringRef Scope,
                                       llvm::StringRef Name);
 
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Symbol &S) {
-    return OS << S.scope() << S.name();
+    return OS << S.qualified_name();
   }
   llvm::StringRef scope() const;
   llvm::StringRef name() const;
+  llvm::StringRef qualified_name() const;
   // The preferred header for this symbol (e.g. the suggested insertion).
   Header header() const;
   // Some symbols may be provided by multiple headers.
   llvm::SmallVector<Header> headers() const;
 
 private:
   Symbol(unsigned ID) : ID(ID) {}
   unsigned ID;

clang/lib/Sema/SemaDecl.cpp

if (TagDecl *PrevTagDecl = dyn_cast<TagDecl>(PrevDecl)) {
                   // Postpone making the old definition visible until after we
                   // complete parsing the new one and do the structural
                   // comparison.
                   SkipBody->CheckSameAsPrevious = true;
                   SkipBody->New = createTagFromNewDecl();
                   SkipBody->Previous = Def;
                   return Def;
                 } else {
+                  llvm::errs() << "got here\n";
+                  Def->dump();
+                  Hidden->dump();
+                  NameLoc.dump(getSourceManager());
                   SkipBody->ShouldSkip = true;
                   SkipBody->Previous = Def;
                   makeMergedDefinitionVisible(Hidden);
                   // Carry on and handle it like a normal definition. We'll
                   // skip starting the definitiion later.
                 }
               } else if (!IsExplicitSpecializationAfterInstantiation) {
                 // A redeclaration in function prototype scope in C isn't

clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp

 #include "clang/AST/Decl.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 
 namespace clang {
 namespace tooling {
 namespace stdlib {
 
+// Header::ID => header name
 static llvm::StringRef *HeaderNames;
-static std::pair<llvm::StringRef, llvm::StringRef> *SymbolNames;
-static unsigned *SymbolHeaderIDs;
+// Header name => Header::ID
 static llvm::DenseMap<llvm::StringRef, unsigned> *HeaderIDs;
-// Maps symbol name -> Symbol::ID, within a namespace.
+
+static unsigned SymbolCount = 0;
+// Symbol::ID => symbol qualified_name/name/scope
+static struct SymbolName {
+  const char *Data;  // std::vector
+  unsigned ScopeLen; // ~~~~~
+  unsigned NameLen;  //      ~~~~~~
+} *SymbolNames;
+// Symbol name -> Symbol::ID, within a namespace.
 using NSSymbolMap = llvm::DenseMap<llvm::StringRef, unsigned>;
 static llvm::DenseMap<llvm::StringRef, NSSymbolMap *> *NamespaceSymbols;
+// Symbol::ID => Header::ID
+static unsigned *SymbolHeaderIDs;
 
 static int initialize() {
-  unsigned SymCount = 0;
-#define SYMBOL(Name, NS, Header) ++SymCount;
+  SymbolCount = 0;
+#define SYMBOL(Name, NS, Header) ++SymbolCount;
 #include "clang/Tooling/Inclusions/CSymbolMap.inc"
 #include "clang/Tooling/Inclusions/StdSymbolMap.inc"
 #undef SYMBOL
-  SymbolNames = new std::remove_reference_t<decltype(*SymbolNames)>[SymCount];
+  SymbolNames =
+      new std::remove_reference_t<decltype(*SymbolNames)>[SymbolCount];
   SymbolHeaderIDs =
-      new std::remove_reference_t<decltype(*SymbolHeaderIDs)>[SymCount];
+      new std::remove_reference_t<decltype(*SymbolHeaderIDs)>[SymbolCount];
   NamespaceSymbols = new std::remove_reference_t<decltype(*NamespaceSymbols)>;
   HeaderIDs = new std::remove_reference_t<decltype(*HeaderIDs)>;
 
   auto AddNS = [&](llvm::StringRef NS) -> NSSymbolMap & {
     auto R = NamespaceSymbols->try_emplace(NS, nullptr);
     if (R.second)
       R.first->second = new NSSymbolMap();
     return *R.first->second;
   };
 
   auto AddHeader = [&](llvm::StringRef Header) -> unsigned {
     return HeaderIDs->try_emplace(Header, HeaderIDs->size()).first->second;
   };
 
-  auto Add = [&, SymIndex(0)](llvm::StringRef Name, llvm::StringRef NS,
+  auto Add = [&, SymIndex(0)](llvm::StringRef QName, unsigned NSLen,
                               llvm::StringRef HeaderName) mutable {
-    if (NS == "None")
-      NS = "";
+    // Correct "Nonefoo" => foo.
+    // FIXME: get rid of "None" from the generated mapping files.
+    if (QName.take_front(NSLen) == "None") {
+      QName = QName.drop_front(NSLen);
+      NSLen = 0;
+    }
 
-    SymbolNames[SymIndex] = {NS, Name};
+    SymbolNames[SymIndex] = {QName.data(), NSLen,
+                             static_cast<unsigned int>(QName.size() - NSLen)};
     SymbolHeaderIDs[SymIndex] = AddHeader(HeaderName);
 
-    NSSymbolMap &NSSymbols = AddNS(NS);
-    NSSymbols.try_emplace(Name, SymIndex);
+    NSSymbolMap &NSSymbols = AddNS(QName.take_front(NSLen));
+    NSSymbols.try_emplace(QName.drop_front(NSLen), SymIndex);
 
     ++SymIndex;
   };
-#define SYMBOL(Name, NS, Header) Add(#Name, #NS, #Header);
+#define SYMBOL(Name, NS, Header) Add(#NS #Name, strlen(#NS), #Header);
 #include "clang/Tooling/Inclusions/CSymbolMap.inc"
 #include "clang/Tooling/Inclusions/StdSymbolMap.inc"
 #undef SYMBOL
 
   HeaderNames = new llvm::StringRef[HeaderIDs->size()];
   for (const auto &E : *HeaderIDs)
     HeaderNames[E.second] = E.first;
 
   return 0;
 }
 
 static void ensureInitialized() {
   static int Dummy = initialize();
   (void)Dummy;
 }
 
+std::vector<Header> Header::all() {
+  std::vector<Header> Result;
+  Result.reserve(HeaderIDs->size());
+  for (unsigned I = 0, E = HeaderIDs->size(); I < E; ++I)
+    Result.push_back(Header(I));
+  return Result;
+}
 std::optional<Header> Header::named(llvm::StringRef Name) {
   ensureInitialized();
   auto It = HeaderIDs->find(Name);
   if (It == HeaderIDs->end())
     return std::nullopt;
   return Header(It->second);
 }
 llvm::StringRef Header::name() const { return HeaderNames[ID]; }
-llvm::StringRef Symbol::scope() const { return SymbolNames[ID].first; }
-llvm::StringRef Symbol::name() const { return SymbolNames[ID].second; }
+
+std::vector<Symbol> Symbol::all() {
+  std::vector<Symbol> Result;
+  Result.reserve(SymbolCount);
+  for (unsigned I = 0, E = SymbolCount; I < E; ++I)
+    Result.push_back(Symbol(I));
+  return Result;
+}
+llvm::StringRef Symbol::scope() const {
+  SymbolName &S = SymbolNames[ID];
+  return StringRef(S.Data, S.ScopeLen);
+}
+llvm::StringRef Symbol::name() const {
+  SymbolName &S = SymbolNames[ID];
+  return StringRef(S.Data + S.ScopeLen, S.NameLen);
+}
+llvm::StringRef Symbol::qualified_name() const {
+  SymbolName &S = SymbolNames[ID];
+  return StringRef(S.Data, S.ScopeLen + S.NameLen);
+}
 std::optional<Symbol> Symbol::named(llvm::StringRef Scope,
                                      llvm::StringRef Name) {
   ensureInitialized();
   if (NSSymbolMap *NSSymbols = NamespaceSymbols->lookup(Scope)) {
     auto It = NSSymbols->find(Name);
     if (It != NSSymbols->end())
       return Symbol(It->second);
   }

clang/test/AST/dump.cpp

 // CHECK-NEXT: | | |-DeclRefExpr {{.+}} <col:47> 'char' lvalue Var {{.+}} 'omp_out' 'char'
 // CHECK-NEXT: | | `-ImplicitCastExpr {{.+}} <col:58> 'int' <IntegralCast>
 // CHECK-NEXT: | |   `-ImplicitCastExpr {{.+}} <col:58> 'char' <LValueToRValue>
 // CHECK-NEXT: | |     `-DeclRefExpr {{.+}} <col:58> 'char' lvalue Var {{.+}} 'omp_in' 'char'
 // CHECK-NEXT: | |-VarDecl {{.+}} <col:40> col:40 implicit used omp_in 'char'
 // CHECK-NEXT: | `-VarDecl {{.+}} <col:40> col:40 implicit used omp_out 'char'
 // CHECK-NEXT: |-OMPDeclareReductionDecl {{.+}} <line:[[@LINE-17]]:37> col:37 fun 'float' combiner 0x{{.+}} initializer 0x{{.+}}
 // CHECK-NEXT: | |-CompoundAssignOperator {{.+}} <col:45, col:56> 'float' lvalue '+=' ComputeLHSTy='float' ComputeResultTy='float'
-// CHECK-NEXT: | | |-DeclRefExpr {{.+}} <col:45> 'float' lvalue Var {{.+}} 'omp_out' 'float'
+// CHECK-NEXT: | | |-BOOM{{.+}} <col:45> 'float' lvalue Var {{.+}} 'omp_out' 'float'
 // CHECK-NEXT: | | `-ImplicitCastExpr {{.+}} <col:56> 'float' <LValueToRValue>
 // CHECK-NEXT: | |   `-DeclRefExpr {{.+}} <col:56> 'float' lvalue Var {{.+}} 'omp_in' 'float'
 // CHECK-NEXT: | |-BinaryOperator {{.+}} <col:87, col:98> 'float' '+'
 // CHECK-NEXT: | | |-ImplicitCastExpr {{.+}} <col:87> 'float' <LValueToRValue>
 // CHECK-NEXT: | | | `-DeclRefExpr {{.+}} <col:87> 'float' lvalue Var {{.+}} 'omp_orig' 'float'
 // CHECK-NEXT: | | `-ImplicitCastExpr {{.+}} <col:98> 'float' <IntegralToFloating>
 // CHECK-NEXT: | |   `-IntegerLiteral {{.+}} <col:98> 'int' 15
 

clang/unittests/Tooling/StandardLibraryTest.cpp

 #include "clang/Testing/TestAST.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+using ::testing::Contains;
 using ::testing::ElementsAre;
 
 namespace clang {
 namespace tooling {
 namespace {
 
 const NamedDecl &lookup(TestAST &AST, llvm::StringRef Name) {
   TranslationUnitDecl *TU = AST.context().getTranslationUnitDecl();
   auto Result = TU->lookup(DeclarationName(&AST.context().Idents.get(Name)));
   assert(!Result.empty() && "Lookup failed");
   assert(Result.isSingleResult() && "Lookup returned multiple results");
   return *Result.front();
 }
 
 TEST(StdlibTest, All) {
   auto VectorH = stdlib::Header::named("<vector>");
   EXPECT_TRUE(VectorH);
+  EXPECT_EQ(VectorH->name(), "<vector>");
   EXPECT_EQ(llvm::to_string(*VectorH), "<vector>");
   EXPECT_FALSE(stdlib::Header::named("HeadersTests.cpp"));
 
   auto Vector = stdlib::Symbol::named("std::", "vector");
   EXPECT_TRUE(Vector);
+  EXPECT_EQ(Vector->scope(), "std::");
+  EXPECT_EQ(Vector->name(), "vector");
+  EXPECT_EQ(Vector->qualified_name(), "std::vector");
   EXPECT_EQ(llvm::to_string(*Vector), "std::vector");
   EXPECT_FALSE(stdlib::Symbol::named("std::", "dongle"));
   EXPECT_FALSE(stdlib::Symbol::named("clang::", "ASTContext"));
 
   EXPECT_EQ(Vector->header(), *VectorH);
   EXPECT_THAT(Vector->headers(), ElementsAre(*VectorH));
+
+  EXPECT_THAT(stdlib::Header::all(), Contains(*VectorH));
+  EXPECT_THAT(stdlib::Symbol::all(), Contains(*Vector));
 }
 
 TEST(StdlibTest, Recognizer) {
   TestAST AST(R"cpp(
     namespace std {
     inline namespace inl {
 
     template <typename>