Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -22,6 +22,7 @@ namespace clangd { class JSONOutput; +class SymbolIndex; /// This class provides implementation of an LSP server, glueing the JSON /// dispatch and ClangdServer together. @@ -35,7 +36,8 @@ const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir, - bool BuildDynamicSymbolIndex); + bool BuildDynamicSymbolIndex, + std::unique_ptr StaticIdx); /// Run LSP server loop, receiving input for it from \p In. \p In must be /// opened in binary mode. Output will be written using Out variable passed to Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -283,10 +283,12 @@ const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir, - bool BuildDynamicSymbolIndex) + bool BuildDynamicSymbolIndex, + std::unique_ptr StaticIdx) : Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, - StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {} + StorePreamblesInMemory, BuildDynamicSymbolIndex, + std::move(StaticIdx), ResourceDir) {} bool ClangdLSPServer::run(std::istream &In) { assert(!IsDone && "Run was called before"); Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -205,6 +205,7 @@ FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, bool BuildDynamicSymbolIndex = false, + std::unique_ptr StaticIdx = nullptr, llvm::Optional ResourceDir = llvm::None); /// Set the root path of the workspace. @@ -338,6 +339,8 @@ DraftStore DraftMgr; /// If set, this manages index for symbols in opened files. std::unique_ptr FileIdx; + /// If set, this provides static index for project-wide global symbols. + std::unique_ptr StaticIdx; CppFileCollection Units; std::string ResourceDir; // If set, this represents the workspace path. Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -135,9 +135,11 @@ unsigned AsyncThreadsCount, bool StorePreamblesInMemory, bool BuildDynamicSymbolIndex, + std::unique_ptr StaticIdx, llvm::Optional ResourceDir) : CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr), + StaticIdx(std::move(StaticIdx)), // Pass a callback into `Units` to extract symbols from a newly parsed // file and rebuild the file index synchronously each time an AST is // parsed. @@ -251,6 +253,8 @@ auto CodeCompleteOpts = Opts; if (FileIdx) CodeCompleteOpts.Index = FileIdx.get(); + if (StaticIdx) + CodeCompleteOpts.StaticIndex = StaticIdx.get(); // A task that will be run asynchronously. auto Task = // 'mutable' to reassign Preamble variable. Index: clangd/CodeComplete.h =================================================================== --- clangd/CodeComplete.h +++ clangd/CodeComplete.h @@ -67,6 +67,9 @@ /// FIXME(ioeric): we might want a better way to pass the index around inside /// clangd. const SymbolIndex *Index = nullptr; + + /// Static index for project-wide global symbols. + const SymbolIndex *StaticIndex = nullptr; }; /// Get code completions at a specified \p Pos in \p FileName. Index: clangd/CodeComplete.cpp =================================================================== --- clangd/CodeComplete.cpp +++ clangd/CodeComplete.cpp @@ -550,7 +550,8 @@ } CompletionItem indexCompletionItem(const Symbol &Sym, llvm::StringRef Filter, - const SpecifiedScope &SSInfo) { + const SpecifiedScope &SSInfo, + llvm::StringRef IndexSource) { CompletionItem Item; Item.kind = toCompletionItemKind(Sym.SymInfo.Kind); Item.label = Sym.Name; @@ -566,10 +567,13 @@ // FIXME(ioeric): use more symbol information (e.g. documentation, label) to // populate the completion item. + // FIXME: find out a better way to show the index source. + Item.detail = llvm::Twine("[" + IndexSource + "]").str(); return Item; } -void completeWithIndex(const Context &Ctx, const SymbolIndex &Index, +void completeWithIndex(const Context &Ctx, + const CodeCompleteOptions &CompleteOptions, llvm::StringRef Code, const SpecifiedScope &SSInfo, llvm::StringRef Filter, CompletionList *Items) { FuzzyFindRequest Req; @@ -579,9 +583,20 @@ StringRef Scope = SSInfo.Resolved.empty() ? SSInfo.Written : SSInfo.Resolved; Req.Scopes = {Scope.trim(':').str()}; - Items->isIncomplete = !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) { - Items->items.push_back(indexCompletionItem(Sym, Filter, SSInfo)); - }); + // FIXME: figure out a good algorithm to merge symbols from dynamic index and + // static index. + if (CompleteOptions.Index) + Items->isIncomplete = + !CompleteOptions.Index->fuzzyFind(Ctx, Req, [&](const Symbol &Sym) { + Items->items.push_back( + indexCompletionItem(Sym, Filter, SSInfo, "DynamicIndex")); + }); + if (CompleteOptions.StaticIndex) + Items->isIncomplete = !CompleteOptions.StaticIndex->fuzzyFind( + Ctx, Req, [&](const Symbol &Sym) { + Items->items.push_back( + indexCompletionItem(Sym, Filter, SSInfo, "StaticIndex")); + }); } SpecifiedScope extraCompletionScope(Sema &S, const CXXScopeSpec &SS) { @@ -633,12 +648,13 @@ invokeCodeComplete(Ctx, std::move(Consumer), Opts.getClangCompleteOpts(), FileName, Command, Preamble, Contents, Pos, std::move(VFS), std::move(PCHs)); - if (Opts.Index && CompletedName.SSInfo) { + if (CompletedName.SSInfo && (Opts.Index || Opts.StaticIndex)) { if (!Results.items.empty()) log(Ctx, "WARNING: Got completion results from sema for completion on " "qualified ID while symbol index is provided."); Results.items.clear(); - completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo, + + completeWithIndex(Ctx, Opts, Contents, *CompletedName.SSInfo, CompletedName.Filter, &Results); } return Results; Index: clangd/index/MemIndex.h =================================================================== --- clangd/index/MemIndex.h +++ clangd/index/MemIndex.h @@ -24,6 +24,9 @@ /// accessible as long as `Symbols` is kept alive. void build(std::shared_ptr> Symbols); + /// \brief Build index from a symbol slab. + static std::unique_ptr build(SymbolSlab Slab); + bool fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req, llvm::function_ref Callback) const override; Index: clangd/index/MemIndex.cpp =================================================================== --- clangd/index/MemIndex.cpp +++ clangd/index/MemIndex.cpp @@ -53,5 +53,21 @@ return true; } +std::unique_ptr MemIndex::build(SymbolSlab Slab) { + struct Snapshot { + SymbolSlab Slab; + std::vector Pointers; + }; + auto Snap = std::make_shared(); + Snap->Slab = std::move(Slab); + for (auto &Sym : Snap->Slab) + Snap->Pointers.push_back(&Sym); + auto S = std::shared_ptr>(std::move(Snap), + &Snap->Pointers); + auto MemIdx = llvm::make_unique(); + MemIdx->build(std::move(S)); + return std::move(MemIdx); +} + } // namespace clangd } // namespace clang Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -11,6 +11,7 @@ #include "JSONRPCDispatcher.h" #include "Path.h" #include "Trace.h" +#include "index/SymbolYAML.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -26,7 +27,24 @@ namespace { enum class PCHStorageFlag { Disk, Memory }; + +// Build an in-memory static index for global symbols from a YAML-format file. +// The size of global symbols should be relatively small, so that all symbols +// can be managed in memory. +std::unique_ptr BuildStaticIndex(llvm::StringRef YamlSymbolFile) { + auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile); + if (!Buffer) { + llvm::errs() << "Can't open " << YamlSymbolFile << "\n"; + return nullptr; + } + auto Slab = SymbolFromYAML(Buffer.get()->getBuffer()); + SymbolSlab::Builder SymsBuilder; + for (auto Sym : Slab) + SymsBuilder.insert(Sym); + + return MemIndex::build(std::move(SymsBuilder).build()); } +} // namespace static llvm::cl::opt CompileCommandsDir( "compile-commands-dir", @@ -97,6 +115,15 @@ "use index built from symbols in opened files"), llvm::cl::init(false), llvm::cl::Hidden); +static llvm::cl::opt YamlSymbolFile( + "yaml-symbol-file", + llvm::cl::desc( + "YAML-format global symbol file to build the static index. It is only " + "available when 'enable-index-based-completion' is enabled.\n" + "WARNING: This option is experimental only, and will be removed " + "eventually. Don't rely on it."), + llvm::cl::init(""), llvm::cl::Hidden); + int main(int argc, char *argv[]) { llvm::cl::ParseCommandLineOptions(argc, argv, "clangd"); @@ -182,13 +209,16 @@ // Change stdin to binary to not lose \r\n on windows. llvm::sys::ChangeStdinToBinary(); + std::unique_ptr StaticIdx; + if (EnableIndexBasedCompletion && !YamlSymbolFile.empty()) + StaticIdx = BuildStaticIndex(YamlSymbolFile); clangd::CodeCompleteOptions CCOpts; CCOpts.EnableSnippets = EnableSnippets; CCOpts.IncludeIneligibleResults = IncludeIneligibleResults; // Initialize and run ClangdLSPServer. ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, CCOpts, ResourceDirRef, CompileCommandsDirPath, - EnableIndexBasedCompletion); + EnableIndexBasedCompletion, std::move(StaticIdx)); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode; Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -68,6 +68,7 @@ MATCHER_P(Labeled, Label, "") { return arg.label == Label; } MATCHER_P(Kind, K, "") { return arg.kind == K; } MATCHER_P(Filter, F, "") { return arg.filterText == F; } +MATCHER_P(Detail, Text, "") { return arg.detail == Text; } MATCHER_P(PlainText, Text, "") { return arg.insertTextFormat == clangd::InsertTextFormat::PlainText && arg.insertText == Text; @@ -453,12 +454,6 @@ std::unique_ptr simpleIndexFromSymbols( std::vector> Symbols) { - auto I = llvm::make_unique(); - struct Snapshot { - SymbolSlab Slab; - std::vector Pointers; - }; - auto Snap = std::make_shared(); SymbolSlab::Builder Slab; for (const auto &Pair : Symbols) { Symbol Sym; @@ -475,13 +470,7 @@ Sym.SymInfo.Kind = Pair.second; Slab.insert(Sym); } - Snap->Slab = std::move(Slab).build(); - for (auto &Iter : Snap->Slab) - Snap->Pointers.push_back(&Iter); - auto S = std::shared_ptr>(std::move(Snap), - &Snap->Pointers); - I->build(std::move(S)); - return std::move(I); + return MemIndex::build(std::move(Slab).build()); } TEST(CompletionTest, NoIndex) { @@ -496,6 +485,38 @@ EXPECT_THAT(Results.items, Has("No")); } +TEST(CompletionTest, StaticIndex) { + clangd::CodeCompleteOptions Opts; + auto I = simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class}}); + Opts.StaticIndex = I.get(); + + auto Results = completions(R"cpp( + void f() { ::ns::^ } + )cpp", + Opts); + EXPECT_THAT(Results.items, + Contains(AllOf(Named("XYZ"), Detail("[StaticIndex]")))); +} + +TEST(CompletionTest, StaticAndDynamicIndex) { + clangd::CodeCompleteOptions Opts; + auto StaticIdx = + simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class}}); + Opts.StaticIndex = StaticIdx.get(); + auto DynamicIdx = + simpleIndexFromSymbols({{"ns::foo", index::SymbolKind::Function}}); + Opts.Index = DynamicIdx.get(); + + auto Results = completions(R"cpp( + void f() { ::ns::^ } + )cpp", + Opts); + EXPECT_THAT(Results.items, + Contains(AllOf(Named("XYZ"), Detail("[StaticIndex]")))); + EXPECT_THAT(Results.items, + Contains(AllOf(Named("foo"), Detail("[DynamicIndex]")))); +} + TEST(CompletionTest, SimpleIndexBased) { clangd::CodeCompleteOptions Opts; auto I = simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class},