[clangd] Move buildStaticIndex() to SymbolYAML

[kbobyrev]

Summary

buildStaticIndex() is used by two other tools that I'm building, now it's useful outside of tool/ClangdMain.cpp.

[kbobyrev]

Move function documentation to the header file.

[sammccall]

clang-tools-extra/clangd/index/SymbolYAML.cpp

193

this is deep copying the symbols for no reason

just build(std::move(Slab)) below.
(I know you're just moving this code, but while you're here...)

clang-tools-extra/clangd/index/SymbolYAML.h

44

nit: can you just say "index file", and call the parameter "SymbolFile"?
See D51585 which adds another format (it can be detected by content sniffing, so no parameter needed)

47

I'd prefer loadIndex for this function, what do you think?

"load" over "build" because much of the indexing work has already been done in producing the file, and in future potentially even more (serializing posting lists).
"static" is coupled to this index's role in ClangdMain, and the point of putting the function here is to allow other uses, so I'd prefer to drop that word.

[kbobyrev]

Rebase and resolve comments

clang-tools-extra/clangd/index/SymbolYAML.h

 // Convert a single symbol to YAML-format string.
 // The YAML result is safe to concatenate.
 std::string SymbolToYAML(Symbol Sym);
 
 // Convert symbols to a YAML-format string.
 // The YAML result is safe to concatenate if you have multiple symbol slabs.
 void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS);
 
+// Build an in-memory static index for global symbols from a YAML-format file.
+// The size of global symbols should be relatively small, so that all symbols
+// can be managed in memory.
+std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile,
+                                              bool UseDex = true);
+
 } // namespace clangd
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H

clang-tools-extra/clangd/index/SymbolYAML.cpp

 //===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "SymbolYAML.h"
 #include "Index.h"
+#include "dex/DexIndex.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 
 LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(clang::clangd::Symbol)
 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
 
 namespace llvm {
 namespace yaml {
 
 using clang::clangd::Symbol;
 using clang::clangd::SymbolID;
 using clang::clangd::SymbolLocation;
 using clang::index::SymbolInfo;
-using clang::index::SymbolLanguage;
 using clang::index::SymbolKind;
+using clang::index::SymbolLanguage;
 
 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
 struct NormalizedSymbolID {
   NormalizedSymbolID(IO &) {}
-  NormalizedSymbolID(IO &, const SymbolID& ID) {
+  NormalizedSymbolID(IO &, const SymbolID &ID) {
     llvm::raw_string_ostream OS(HexString);
     OS << ID;
   }
 
-  SymbolID denormalize(IO&) {
+  SymbolID denormalize(IO &) {
     SymbolID ID;
     HexString >> ID;
     return ID;
   }
 
   std::string HexString;
 };
 

 }
 
 Symbol SymbolFromYAML(llvm::yaml::Input &Input) {
   Symbol S;
   Input >> S;
   return S;
 }
 
-void SymbolsToYAML(const SymbolSlab& Symbols, llvm::raw_ostream &OS) {
+void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS) {
   llvm::yaml::Output Yout(OS);
   for (Symbol S : Symbols) // copy: Yout<< requires mutability.
     Yout << S;
 }
 
 std::string SymbolToYAML(Symbol Sym) {
   std::string Str;
   llvm::raw_string_ostream OS(Str);
   llvm::yaml::Output Yout(OS);
   Yout << Sym;
   return OS.str();
 }
 
+std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile,
+                                              bool UseDex) {
+  auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
+  if (!Buffer) {
+    llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
+    return nullptr;
+  }
+  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
+  SymbolSlab::Builder SymsBuilder;
+  for (auto Sym : Slab)
+    SymsBuilder.insert(Sym);
+
+  return UseDex ? dex::DexIndex::build(std::move(SymsBuilder).build())
+                : MemIndex::build(std::move(SymsBuilder).build(),
+                                  SymbolOccurrenceSlab::createEmpty());
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/tool/ClangdMain.cpp

static llvm::cl::opt<bool>
     UseDex("use-dex-index",
            llvm::cl::desc("Use experimental Dex static index."),
            llvm::cl::init(true), llvm::cl::Hidden);
 
 namespace {
 
 enum class PCHStorageFlag { Disk, Memory };
 
-// Build an in-memory static index for global symbols from a YAML-format file.
-// The size of global symbols should be relatively small, so that all symbols
-// can be managed in memory.
-std::unique_ptr<SymbolIndex> buildStaticIndex(llvm::StringRef YamlSymbolFile) {
-  auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
-  if (!Buffer) {
-    llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
-    return nullptr;
-  }
-  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
-  SymbolSlab::Builder SymsBuilder;
-  for (auto Sym : Slab)
-    SymsBuilder.insert(Sym);
-
-  return UseDex ? dex::DexIndex::build(std::move(SymsBuilder).build())
-                : MemIndex::build(std::move(SymsBuilder).build(),
-                                  SymbolOccurrenceSlab::createEmpty());
-}
-
 } // namespace
 
 static llvm::cl::opt<Path> CompileCommandsDir(
     "compile-commands-dir",
     llvm::cl::desc("Specify a path to look for compile_commands.json. If path "
                    "is invalid, clangd will look in the current directory and "
                    "parent paths of each source file."));
 
case PCHStorageFlag::Disk:
     Opts.StorePreamblesInMemory = false;
     break;
   }
   if (!ResourceDir.empty())
     Opts.ResourceDir = ResourceDir;
   Opts.BuildDynamicSymbolIndex = EnableIndex;
   std::unique_ptr<SymbolIndex> StaticIdx;
   if (EnableIndex && !YamlSymbolFile.empty()) {
-    StaticIdx = buildStaticIndex(YamlSymbolFile);
+    StaticIdx = buildStaticIndex(YamlSymbolFile, UseDex);
     Opts.StaticIndex = StaticIdx.get();
   }
   Opts.AsyncThreadsCount = WorkerThreadsCount;
 
   clangd::CodeCompleteOptions CCOpts;
   CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
   CCOpts.Limit = LimitResults;
   CCOpts.BundleOverloads = CompletionStyle != Detailed;