[clangd] Allow using experimental Dex index

[kbobyrev]

Summary

This patch adds hidden Clangd flag which replaces (currently) default MemIndex with DexIndex for the static index.

[kbobyrev]

Fix anonymous namespace beginning placement in Clangd driver.

[ioeric]

clang-tools-extra/clangd/index/Index.h

360

I'd try to avoid this pattern as it implicitly requires T::build(std::shared_ptr<std::vector<const Symbol *>>) to be implemented. Alternatively, you could pull out a helper function that turns a SymbolSlab into std::shared_ptr<std::vector<const Symbol *>>, which can be shared by different indexes. It can probably live in MemIndex.h.

[ioeric]

clang-tools-extra/clangd/index/MemIndex.h

47

Please add documentation.

clang-tools-extra/clangd/tool/ClangdMain.cpp

99

Why the move? A chunk of code in the middle of flags seems strange.

[kbobyrev]

Aww, the previous diff was the wrong one and didn't contain docs.

The move of the code to the middle of Clangd driver was justified by the assumption that it might be better to move the code that uses UseDex flag closer to the flag itself, but I agree: it looks strange. I moved it back.

[ioeric]

lg

clang-tools-extra/clangd/index/Index.h

struct OccurrencesRequest {
   llvm::DenseSet<SymbolID> IDs;
   SymbolOccurrenceKind Filter;
 };
 
 /// \brief Interface for symbol indexes that can be used for searching or
 /// matching symbols among a set of symbols based on names or unique IDs.
 class SymbolIndex {
 public:
+ /// \brief Build index from a symbol slab.
+ template <typename T>
+ static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab) {
+   struct Snapshot {
+     SymbolSlab Slab;
+     std::vector<const Symbol *> Pointers;
+   };
+   auto Snap = std::make_shared<Snapshot>();
+   Snap->Slab = std::move(Slab);
+   for (auto &Sym : Snap->Slab) Snap->Pointers.push_back(&Sym);
+   auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
+                                                         &Snap->Pointers);
+   auto Idx = llvm::make_unique<T>();
+   Idx->build(std::move(S));
+   return std::move(Idx);
+ }
+
   virtual ~SymbolIndex() = default;
 
   /// \brief Matches symbols in the index fuzzily and applies \p Callback on
   /// each matched symbol before returning.
   /// If returned Symbols are used outside Callback, they must be deep-copied!
   ///
   /// Returns true if there may be more results (limited by MaxCandidateCount).
   virtual bool

clang-tools-extra/clangd/index/MemIndex.h

 /// \brief This implements an index for a (relatively small) set of symbols that
 /// can be easily managed in memory.
 class MemIndex : public SymbolIndex {
 public:
   /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain
   /// accessible as long as `Symbols` is kept alive.
   void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
 
-  /// \brief Build index from a symbol slab.
-  static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab);
-
   bool
   fuzzyFind(const FuzzyFindRequest &Req,
             llvm::function_ref<void(const Symbol &)> Callback) const override;
 
   void
   lookup(const LookupRequest &Req,
          llvm::function_ref<void(const Symbol &)> Callback) const override;
 
   void findOccurrences(const OccurrencesRequest &Req,
                        llvm::function_ref<void(const SymbolOccurrence &)>
                            Callback) const override;
 
 private:
   std::shared_ptr<std::vector<const Symbol *>> Symbols;
   // Index is a set of symbols that are deduplicated by symbol IDs.
   // FIXME: build smarter index structure.
   llvm::DenseMap<SymbolID, const Symbol *> Index;
   mutable std::mutex Mutex;
 };
 
 } // namespace clangd
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H

clang-tools-extra/clangd/index/MemIndex.cpp

void MemIndex::lookup(const LookupRequest &Req,
                       llvm::function_ref<void(const Symbol &)> Callback) const {
   for (const auto &ID : Req.IDs) {
     auto I = Index.find(ID);
     if (I != Index.end())
       Callback(*I->second);
   }
 }
 
-std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
-  struct Snapshot {
-    SymbolSlab Slab;
-    std::vector<const Symbol *> Pointers;
-  };
-  auto Snap = std::make_shared<Snapshot>();
-  Snap->Slab = std::move(Slab);
-  for (auto &Sym : Snap->Slab)
-    Snap->Pointers.push_back(&Sym);
-  auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
-                                                        &Snap->Pointers);
-  auto MemIdx = llvm::make_unique<MemIndex>();
-  MemIdx->build(std::move(S));
-  return std::move(MemIdx);
-}
-
 void MemIndex::findOccurrences(
     const OccurrencesRequest &Req,
     llvm::function_ref<void(const SymbolOccurrence &)> Callback) const {
   log("findOccurrences is not implemented.");
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <thread>
 #include "ClangdLSPServer.h"
 #include "JSONRPCDispatcher.h"
 #include "Path.h"
 #include "Trace.h"
-#include "index/SymbolYAML.h"
 #include "clang/Basic/Version.h"
+#include "index/SymbolYAML.h"
+#include "index/dex/DexIndex.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
-#include <cstdlib>
-#include <iostream>
-#include <memory>
-#include <string>
-#include <thread>
 
 using namespace clang;
 using namespace clang::clangd;
 
 namespace {
 enum class PCHStorageFlag { Disk, Memory };
 
-// Build an in-memory static index for global symbols from a YAML-format file.
-// The size of global symbols should be relatively small, so that all symbols
-// can be managed in memory.
-std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile) {
-  auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
-  if (!Buffer) {
-    llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
-    return nullptr;
-  }
-  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
-  SymbolSlab::Builder SymsBuilder;
-  for (auto Sym : Slab)
-    SymsBuilder.insert(Sym);
-
-  return MemIndex::build(std::move(SymsBuilder).build());
-}
-} // namespace
-
 static llvm::cl::opt<Path> CompileCommandsDir(
     "compile-commands-dir",
     llvm::cl::desc("Specify a path to look for compile_commands.json. If path "
                    "is invalid, clangd will look in the current directory and "
                    "parent paths of each source file."));
 
 static llvm::cl::opt<unsigned>
     WorkerThreadsCount("j",

 
 static llvm::cl::opt<bool> Test(
     "lit-test",
     llvm::cl::desc(
         "Abbreviation for -input-style=delimited -pretty -run-synchronously. "
         "Intended to simplify lit tests."),
     llvm::cl::init(false), llvm::cl::Hidden);
 
 static llvm::cl::opt<PCHStorageFlag> PCHStorage(
     "pch-storage",
     llvm::cl::desc("Storing PCHs in memory increases memory usages, but may "
                    "improve performance"),
     llvm::cl::values(
         clEnumValN(PCHStorageFlag::Disk, "disk", "store PCHs on disk"),
         clEnumValN(PCHStorageFlag::Memory, "memory", "store PCHs in memory")),
     llvm::cl::init(PCHStorageFlag::Disk));
 
static llvm::cl::opt<CompileArgsFrom> CompileArgsFrom(
     llvm::cl::values(clEnumValN(LSPCompileArgs, "lsp",
                                 "All compile commands come from LSP and "
                                 "'compile_commands.json' files are ignored"),
                      clEnumValN(FilesystemCompileArgs, "filesystem",
                                 "All compile commands come from the "
                                 "'compile_commands.json' files")),
     llvm::cl::init(FilesystemCompileArgs), llvm::cl::Hidden);
 
+static llvm::cl::opt<bool> UseDex(
+    "use-dex-index", llvm::cl::desc("Use experimental Dex static index."),
+    llvm::cl::init(false), llvm::cl::Hidden);
+
+// Build an in-memory static index for global symbols from a YAML-format file.
+// The size of global symbols should be relatively small, so that all symbols
+// can be managed in memory.
+std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile) {
+  auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
+  if (!Buffer) {
+    llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
+    return nullptr;
+  }
+  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
+  SymbolSlab::Builder SymsBuilder;
+  for (auto Sym : Slab) SymsBuilder.insert(Sym);
+
+  return UseDex
+             ? SymbolIndex::build<dex::DexIndex>(std::move(SymsBuilder).build())
+             : SymbolIndex::build<MemIndex>(std::move(SymsBuilder).build());
+}
+}  // namespace
+
 int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
   llvm::cl::SetVersionPrinter([](llvm::raw_ostream &OS) {
     OS << clang::getClangToolFullVersion("clangd") << "\n";
   });
   llvm::cl::ParseCommandLineOptions(
       argc, argv,
       "clangd is a language server that provides IDE-like features to editors. "

clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp

Matcher<const std::vector<CodeCompletion> &> Has(std::string Name,
   return Contains(AllOf(Named(std::move(Name)), Kind(K)));
 }
 MATCHER(IsDocumented, "") { return !arg.Documentation.empty(); }
 
 std::unique_ptr<SymbolIndex> memIndex(std::vector<Symbol> Symbols) {
   SymbolSlab::Builder Slab;
   for (const auto &Sym : Symbols)
     Slab.insert(Sym);
-  return MemIndex::build(std::move(Slab).build());
+  return SymbolIndex::build<MemIndex>(std::move(Slab).build());
 }
 
 CodeCompleteResult completions(ClangdServer &Server, StringRef TestCode,
                                Position point,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {}) {
   std::unique_ptr<SymbolIndex> OverrideIndex;
   if (!IndexSymbols.empty()) {

clang-tools-extra/unittests/clangd/TestTU.cpp

 }
 
 SymbolSlab TestTU::headerSymbols() const {
   auto AST = build();
   return indexAST(AST.getASTContext(), AST.getPreprocessorPtr());
 }
 
 std::unique_ptr<SymbolIndex> TestTU::index() const {
-  return MemIndex::build(headerSymbols());
+  return SymbolIndex::build<MemIndex>(headerSymbols());
 }
 
 const Symbol &findSymbol(const SymbolSlab &Slab, llvm::StringRef QName) {
   const Symbol *Result = nullptr;
   for (const Symbol &S : Slab) {
     if (QName != (S.Scope + S.Name).str())
       continue;
     if (Result) {