[clangd] Load static index asynchronously, add tracing.

[sammccall]

Summary

Like D51475 but simplified based on recent patches.
While here, clarify that loadIndex() takes a filename, not file content.

[ioeric]

lg. Thanks!

[ilya-biryukov]

clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

287

Wouldn't the future returned by runAsync wait in destructor?

288

What happens if clangd tries to exit before the index is loaded?
Could lead to crashes.

[sammccall]

I think you're right - this isn't actually asynchronous (I didn't manage to test that!), and if it were it'd interfere with clean shutdown.

I think both issues can be addressed by assigning the future to a variable scoped to main. Will send a patch.

clang-tools-extra/trunk/clangd/index/SymbolYAML.h

 
 // Convert symbols to a YAML-format string.
 // The YAML result is safe to concatenate if you have multiple symbol slabs.
 void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS);
 
 // Build an in-memory static index for global symbols from a symbol file.
 // The size of global symbols should be relatively small, so that all symbols
 // can be managed in memory.
-std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFile,
+std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
                                        bool UseDex = true);
 
 } // namespace clangd
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H

clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp

 //===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "SymbolYAML.h"
+#include "../Trace.h"
 #include "Index.h"
 #include "Serialization.h"
 #include "dex/DexIndex.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"

 std::string SymbolToYAML(Symbol Sym) {
   std::string Str;
   llvm::raw_string_ostream OS(Str);
   llvm::yaml::Output Yout(OS);
   Yout << Sym;
   return OS.str();
 }
 
-std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFile,
+std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
                                        bool UseDex) {
-  auto Buffer = llvm::MemoryBuffer::getFile(SymbolFile);
+  trace::Span OverallTracer("LoadIndex");
+  auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
   if (!Buffer) {
-    llvm::errs() << "Can't open " << SymbolFile << "\n";
+    llvm::errs() << "Can't open " << SymbolFilename << "\n";
     return nullptr;
   }
   StringRef Data = Buffer->get()->getBuffer();
 
   llvm::Optional<SymbolSlab> Slab;
   if (Data.startswith("RIFF")) { // Magic for binary index file.
+    trace::Span Tracer("ParseRIFF");
     if (auto RIFF = readIndexFile(Data))
       Slab = std::move(RIFF->Symbols);
     else
       llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n";
   } else {
+    trace::Span Tracer("ParseYAML");
     Slab = symbolsFromYAML(Data);
   }
 
+  if (!Slab)
+    return nullptr;
+  trace::Span Tracer("BuildIndex");
   return UseDex ? dex::DexIndex::build(std::move(*Slab))
                 : MemIndex::build(std::move(*Slab), RefSlab());
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

case PCHStorageFlag::Disk:
     Opts.StorePreamblesInMemory = false;
     break;
   }
   if (!ResourceDir.empty())
     Opts.ResourceDir = ResourceDir;
   Opts.BuildDynamicSymbolIndex = EnableIndex;
   std::unique_ptr<SymbolIndex> StaticIdx;
   if (EnableIndex && !YamlSymbolFile.empty()) {
-    StaticIdx = loadIndex(YamlSymbolFile, UseDex);
-    Opts.StaticIndex = StaticIdx.get();
+    // Load the index asynchronously. Meanwhile SwapIndex returns no results.
+    SwapIndex *Placeholder;
+    StaticIdx.reset(Placeholder = new SwapIndex(llvm::make_unique<MemIndex>()));
+    runAsync<void>([Placeholder] {
+      if (auto Idx = loadIndex(YamlSymbolFile))
+        Placeholder->reset(std::move(Idx));
+    });
   }
+  Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
 
   clangd::CodeCompleteOptions CCOpts;
   CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
   CCOpts.Limit = LimitResults;
   CCOpts.BundleOverloads = CompletionStyle != Detailed;
   CCOpts.ShowOrigins = ShowOrigins;
   if (!HeaderInsertionDecorators) {