[clang][Tooling] Move STL recognizer to its own library

[kadircet]

Summary

As pointed out in https://reviews.llvm.org/D119130#3829816, this
introduces a clang AST dependency to the clangToolingInclusions, which is used
by clang-format.

Since rest of the inclusion tooling doesn't depend on clang ast, moving this
into a separate library.

[sammccall]

clang-tools-extra/clangd/CMakeLists.txt

163

StandardLibrary or Stdlib?

STL isn't accurate or consistent with the names in the code.

clang/lib/Tooling/Inclusions/STL/CMakeLists.txt

2 ↗

(On Diff #465306)

This means the implementation files and the header files have a different directory structure, which may be confusing to people trying to work out which library to link against based on the headers they included.

On the other hand, I think the cascading effect of dependencies => libraries => directory structure => header structure is pretty unfortunate leaking of llvm's sad cmake structure. Up to you

[kadircet]

clang-tools-extra/clangd/CMakeLists.txt

163

changing to Stdlib

clang/lib/Tooling/Inclusions/STL/CMakeLists.txt

2 ↗

(On Diff #465306)

right, i was also torn between moving the headers around vs not. but i finally managed to convince myself that the implementation being in a different subdirectory is actually an unfortunate detail about the way LLVM is build (I didn't want to have PARTIAL_SOURCES_INTENDED, either) and shouldn't matter for the applications that want to use it.

[kadircet]

• Rename STL to Stdlib

[thakis]

Thanks!

clang/lib/Tooling/Inclusions/STL/CMakeLists.txt

2 ↗

(On Diff #465306)

FWIW, I'd move the header.

FWIW, I also don't think llvm's cmake setup is particularly unfortunate here either. It makes it easy to see where library boundaries are.

[thakis]

(Alternatively, it looks like the only client is clangd, so maybe the file could just live in there instead of being in a dedicated library with a single client.)

[kamaub]

This change causes a linking failure during the check-all testing of 'clang-tools-extra' on the clang-ppc64le-rhel #22596 and clang-ppc64le-linux-multistage #23864 build bots, please address this failure as soon as possible.

[kadircet]

sorry for that, bb4f0af97dba600c50a0f727b25a760e8185c578 should fix it

clang-tools-extra/clangd/CMakeLists.txt

clang_target_link_libraries(clangDaemon
   clangFrontend
   clangIndex
   clangLex
   clangSema
   clangSerialization
   clangTooling
   clangToolingCore
   clangToolingInclusions
+  clangToolingInclusionsStdlib
   clangToolingSyntax
   )
 
 target_link_libraries(clangDaemon
   PRIVATE
   ${LLVM_PTHREAD_LIB}
 
   clangTidy

clang/lib/Tooling/Inclusions/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangToolingInclusions
   HeaderIncludes.cpp
   IncludeStyle.cpp
-  StandardLibrary.cpp
 
   LINK_LIBS
-  clangAST
   clangBasic
   clangLex
   clangRewrite
   clangToolingCore
   )
+
+  add_subdirectory(Stdlib)

clang/lib/Tooling/Inclusions/Stdlib/CMakeLists.txt

+add_clang_library(clangToolingInclusionsStdlib
+  StandardLibrary.cpp
+
+  LINK_LIBS
+  clangAST
+  )

clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp

 //===--- StandardLibrary.cpp ------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/Inclusions/StandardLibrary.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 
 namespace clang {
 namespace tooling {
 namespace stdlib {
 
 static llvm::StringRef *HeaderNames;
 static std::pair<llvm::StringRef, llvm::StringRef> *SymbolNames;
 static unsigned *SymbolHeaderIDs;
 static llvm::DenseMap<llvm::StringRef, unsigned> *HeaderIDs;
 // Maps symbol name -> Symbol::ID, within a namespace.
 using NSSymbolMap = llvm::DenseMap<llvm::StringRef, unsigned>;
 static llvm::DenseMap<llvm::StringRef, NSSymbolMap *> *NamespaceSymbols;
 
 static int initialize() {
   unsigned SymCount = 0;
 #define SYMBOL(Name, NS, Header) ++SymCount;
 #include "clang/Tooling/Inclusions/CSymbolMap.inc"
 #include "clang/Tooling/Inclusions/StdSymbolMap.inc"
 #undef SYMBOL
   SymbolNames = new std::remove_reference_t<decltype(*SymbolNames)>[SymCount];
   SymbolHeaderIDs =
       new std::remove_reference_t<decltype(*SymbolHeaderIDs)>[SymCount];
   NamespaceSymbols = new std::remove_reference_t<decltype(*NamespaceSymbols)>;
   HeaderIDs = new std::remove_reference_t<decltype(*HeaderIDs)>;
 
   auto AddNS = [&](llvm::StringRef NS) -> NSSymbolMap & {
     auto R = NamespaceSymbols->try_emplace(NS, nullptr);
     if (R.second)
       R.first->second = new NSSymbolMap();
     return *R.first->second;
   };
 
   auto AddHeader = [&](llvm::StringRef Header) -> unsigned {
     return HeaderIDs->try_emplace(Header, HeaderIDs->size()).first->second;
   };
 
   auto Add = [&, SymIndex(0)](llvm::StringRef Name, llvm::StringRef NS,
                               llvm::StringRef HeaderName) mutable {
     if (NS == "None")
       NS = "";
 
     SymbolNames[SymIndex] = {NS, Name};
     SymbolHeaderIDs[SymIndex] = AddHeader(HeaderName);
 
     NSSymbolMap &NSSymbols = AddNS(NS);
     NSSymbols.try_emplace(Name, SymIndex);
 
     ++SymIndex;
   };
 #define SYMBOL(Name, NS, Header) Add(#Name, #NS, #Header);
 #include "clang/Tooling/Inclusions/CSymbolMap.inc"
 #include "clang/Tooling/Inclusions/StdSymbolMap.inc"
 #undef SYMBOL
 
   HeaderNames = new llvm::StringRef[HeaderIDs->size()];
   for (const auto &E : *HeaderIDs)
     HeaderNames[E.second] = E.first;
 
   return 0;
 }
 
 static void ensureInitialized() {
   static int Dummy = initialize();
   (void)Dummy;
 }
 
 llvm::Optional<Header> Header::named(llvm::StringRef Name) {
   ensureInitialized();
   auto It = HeaderIDs->find(Name);
   if (It == HeaderIDs->end())
     return llvm::None;
   return Header(It->second);
 }
 llvm::StringRef Header::name() const { return HeaderNames[ID]; }
 llvm::StringRef Symbol::scope() const { return SymbolNames[ID].first; }
 llvm::StringRef Symbol::name() const { return SymbolNames[ID].second; }
 llvm::Optional<Symbol> Symbol::named(llvm::StringRef Scope,
                                      llvm::StringRef Name) {
   ensureInitialized();
   if (NSSymbolMap *NSSymbols = NamespaceSymbols->lookup(Scope)) {
     auto It = NSSymbols->find(Name);
     if (It != NSSymbols->end())
       return Symbol(It->second);
   }
   return llvm::None;
 }
 Header Symbol::header() const { return Header(SymbolHeaderIDs[ID]); }
 llvm::SmallVector<Header> Symbol::headers() const {
   return {header()}; // FIXME: multiple in case of ambiguity
 }
 
 Recognizer::Recognizer() { ensureInitialized(); }
 
 NSSymbolMap *Recognizer::namespaceSymbols(const NamespaceDecl *D) {
   auto It = NamespaceCache.find(D);
   if (It != NamespaceCache.end())
     return It->second;
 
   NSSymbolMap *Result = [&]() -> NSSymbolMap * {
     if (D && D->isAnonymousNamespace())
       return nullptr;
     // Print the namespace and its parents ommitting inline scopes.
     std::string Scope;
     for (const auto *ND = D; ND;
          ND = llvm::dyn_cast_or_null<NamespaceDecl>(ND->getParent()))
       if (!ND->isInlineNamespace() && !ND->isAnonymousNamespace())
         Scope = ND->getName().str() + "::" + Scope;
     return NamespaceSymbols->lookup(Scope);
   }();
   NamespaceCache.try_emplace(D, Result);
   return Result;
 }
 
 llvm::Optional<Symbol> Recognizer::operator()(const Decl *D) {
   // If D is std::vector::iterator, `vector` is the outer symbol to look up.
   // We keep all the candidate DCs as some may turn out to be anon enums.
   // Do this resolution lazily as we may turn out not to have a std namespace.
   llvm::SmallVector<const DeclContext *> IntermediateDecl;
   const DeclContext *DC = D->getDeclContext();
   while (DC && !DC->isNamespace()) {
     if (NamedDecl::classofKind(DC->getDeclKind()))
       IntermediateDecl.push_back(DC);
     DC = DC->getParent();
   }
   NSSymbolMap *Symbols = namespaceSymbols(cast_or_null<NamespaceDecl>(DC));
   if (!Symbols)
     return llvm::None;
 
   llvm::StringRef Name = [&]() -> llvm::StringRef {
     for (const auto *SymDC : llvm::reverse(IntermediateDecl)) {
       DeclarationName N = cast<NamedDecl>(SymDC)->getDeclName();
       if (const auto *II = N.getAsIdentifierInfo())
         return II->getName();
       if (!N.isEmpty())
         return ""; // e.g. operator<: give up
     }
     if (const auto *ND = llvm::dyn_cast<NamedDecl>(D))
       if (const auto *II = ND->getIdentifier())
         return II->getName();
     return "";
   }();
   if (Name.empty())
     return llvm::None;
 
   auto It = Symbols->find(Name);
   if (It == Symbols->end())
     return llvm::None;
   return Symbol(It->second);
 }
 
 } // namespace stdlib
 } // namespace tooling
 } // namespace clang

clang/unittests/Tooling/CMakeLists.txt

clang_target_link_libraries(ToolingTests
   clangFormat
   clangFrontend
   clangLex
   clangRewrite
   clangSerialization
   clangTooling
   clangToolingCore
   clangToolingInclusions
+  clangToolingInclusionsSTL
   clangToolingRefactoring
   clangTransformer
   )
 
 target_link_libraries(ToolingTests
   PRIVATE
   LLVMTestingSupport
   clangTesting
 )
 
 add_subdirectory(Syntax)