[clangd] Allow using experimental Dex index

[kbobyrev]

Summary

This patch adds hidden Clangd flag which replaces (currently) default MemIndex with DexIndex for the static index.

[kbobyrev]

Fix anonymous namespace beginning placement in Clangd driver.

[ioeric]

clang-tools-extra/clangd/index/Index.h

360 ↗

(On Diff #161239)

I'd try to avoid this pattern as it implicitly requires T::build(std::shared_ptr<std::vector<const Symbol *>>) to be implemented. Alternatively, you could pull out a helper function that turns a SymbolSlab into std::shared_ptr<std::vector<const Symbol *>>, which can be shared by different indexes. It can probably live in MemIndex.h.

[ioeric]

clang-tools-extra/clangd/index/MemIndex.h

50 ↗

(On Diff #161487)

Please add documentation.

clang-tools-extra/clangd/tool/ClangdMain.cpp

96 ↗

(On Diff #161487)

Why the move? A chunk of code in the middle of flags seems strange.

[kbobyrev]

Aww, the previous diff was the wrong one and didn't contain docs.

The move of the code to the middle of Clangd driver was justified by the assumption that it might be better to move the code that uses UseDex flag closer to the flag itself, but I agree: it looks strange. I moved it back.

[ioeric]

lg

clang-tools-extra/trunk/clangd/index/MemIndex.h

 private:
   std::shared_ptr<std::vector<const Symbol *>> Symbols;
   // Index is a set of symbols that are deduplicated by symbol IDs.
   // FIXME: build smarter index structure.
   llvm::DenseMap<SymbolID, const Symbol *> Index;
   mutable std::mutex Mutex;
 };
 
+// Returns pointers to the symbols in given slab and bundles slab lifetime with
+// returned symbol pointers so that the pointers are never invalid.
+std::shared_ptr<std::vector<const Symbol *>>
+getSymbolsFromSlab(SymbolSlab Slab);
+
 } // namespace clangd
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H

clang-tools-extra/trunk/clangd/index/MemIndex.cpp

void MemIndex::build(std::shared_ptr<std::vector<const Symbol *>> Syms) {
   // Swap out the old symbols and index.
   {
     std::lock_guard<std::mutex> Lock(Mutex);
     Index = std::move(TempIndex);
     Symbols = std::move(Syms); // Relase old symbols.
   }
 }
 
+std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
+  auto Idx = llvm::make_unique<MemIndex>();
+  Idx->build(getSymbolsFromSlab(std::move(Slab)));
+  return std::move(Idx);
+}
+
 bool MemIndex::fuzzyFind(
     const FuzzyFindRequest &Req,
     llvm::function_ref<void(const Symbol &)> Callback) const {
   assert(!StringRef(Req.Query).contains("::") &&
          "There must be no :: in query.");
 
   std::priority_queue<std::pair<float, const Symbol *>> Top;
   FuzzyMatcher Filter(Req.Query);
void MemIndex::lookup(const LookupRequest &Req,
   std::lock_guard<std::mutex> Lock(Mutex);
   for (const auto &ID : Req.IDs) {
     auto I = Index.find(ID);
     if (I != Index.end())
       Callback(*I->second);
   }
 }
 
-std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
+void MemIndex::findOccurrences(
+    const OccurrencesRequest &Req,
+    llvm::function_ref<void(const SymbolOccurrence &)> Callback) const {
+  log("findOccurrences is not implemented.");
+}
+
+std::shared_ptr<std::vector<const Symbol *>>
+getSymbolsFromSlab(SymbolSlab Slab) {
   struct Snapshot {
     SymbolSlab Slab;
     std::vector<const Symbol *> Pointers;
   };
   auto Snap = std::make_shared<Snapshot>();
   Snap->Slab = std::move(Slab);
   for (auto &Sym : Snap->Slab)
     Snap->Pointers.push_back(&Sym);
-  auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
-                                                        &Snap->Pointers);
-  auto MemIdx = llvm::make_unique<MemIndex>();
-  MemIdx->build(std::move(S));
-  return std::move(MemIdx);
-}
-
-void MemIndex::findOccurrences(
-    const OccurrencesRequest &Req,
-    llvm::function_ref<void(const SymbolOccurrence &)> Callback) const {
-  log("findOccurrences is not implemented.");
+  return std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/trunk/clangd/index/dex/DexIndex.h

 // (when the clangd process starts). Therefore, it can be easier to store built
 // index on disk and then load it if available.
 class DexIndex : public SymbolIndex {
 public:
   /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain
   /// accessible as long as `Symbols` is kept alive.
   void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
 
+  /// \brief Build index from a symbol slab.
+  static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab);
+
   bool
   fuzzyFind(const FuzzyFindRequest &Req,
             llvm::function_ref<void(const Symbol &)> Callback) const override;
 
   void lookup(const LookupRequest &Req,
               llvm::function_ref<void(const Symbol &)> Callback) const override;
 
   void findOccurrences(const OccurrencesRequest &Req,

clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp

for (DocID SymbolRank = 0; SymbolRank < Syms->size(); ++SymbolRank) {
     // Replace outdated index with the new one.
     LookupTable = std::move(TempLookupTable);
     Symbols = std::move(Syms);
     InvertedIndex = std::move(TempInvertedIndex);
     SymbolQuality = std::move(TempSymbolQuality);
   }
 }
 
+std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
+  auto Idx = llvm::make_unique<MemIndex>();
+  Idx->build(getSymbolsFromSlab(std::move(Slab)));
+  return std::move(Idx);
+}
+
 /// Constructs iterators over tokens extracted from the query and exhausts it
 /// while applying Callback to each symbol in the order of decreasing quality
 /// of the matched symbols.
 bool DexIndex::fuzzyFind(
     const FuzzyFindRequest &Req,
     llvm::function_ref<void(const Symbol &)> Callback) const {
   assert(!StringRef(Req.Query).contains("::") &&
          "There must be no :: in query.");

clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

 //===--- ClangdMain.cpp - clangd server loop ------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "ClangdLSPServer.h"
 #include "JSONRPCDispatcher.h"
 #include "Path.h"
 #include "Trace.h"
 #include "index/SymbolYAML.h"
+#include "index/dex/DexIndex.h"
 #include "clang/Basic/Version.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdlib>
 #include <iostream>
 #include <memory>
 #include <string>
 #include <thread>
 
 using namespace clang;
 using namespace clang::clangd;
 
 namespace {
+
 enum class PCHStorageFlag { Disk, Memory };
 
 // Build an in-memory static index for global symbols from a YAML-format file.
 // The size of global symbols should be relatively small, so that all symbols
 // can be managed in memory.
 std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile) {
   auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
   if (!Buffer) {
     llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
     return nullptr;
   }
   auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
   SymbolSlab::Builder SymsBuilder;
   for (auto Sym : Slab)
     SymsBuilder.insert(Sym);
 
-  return MemIndex::build(std::move(SymsBuilder).build());
+  return UseDex ? DexIndex::build(std::move(SymsBuilder).build())
+                : MemIndex::build(std::move(SymsBuilder).build());
 }
+
 } // namespace
 
 static llvm::cl::opt<Path> CompileCommandsDir(
     "compile-commands-dir",
     llvm::cl::desc("Specify a path to look for compile_commands.json. If path "
                    "is invalid, clangd will look in the current directory and "
                    "parent paths of each source file."));
 
static llvm::cl::opt<CompileArgsFrom> CompileArgsFrom(
     llvm::cl::values(clEnumValN(LSPCompileArgs, "lsp",
                                 "All compile commands come from LSP and "
                                 "'compile_commands.json' files are ignored"),
                      clEnumValN(FilesystemCompileArgs, "filesystem",
                                 "All compile commands come from the "
                                 "'compile_commands.json' files")),
     llvm::cl::init(FilesystemCompileArgs), llvm::cl::Hidden);
 
+static llvm::cl::opt<bool>
+    UseDex("use-dex-index",
+           llvm::cl::desc("Use experimental Dex static index."),
+           llvm::cl::init(false), llvm::cl::Hidden);
+
 int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
   llvm::cl::SetVersionPrinter([](llvm::raw_ostream &OS) {
     OS << clang::getClangToolFullVersion("clangd") << "\n";
   });
   llvm::cl::ParseCommandLineOptions(
       argc, argv,
       "clangd is a language server that provides IDE-like features to editors. "