Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -22,6 +22,7 @@ namespace clangd { class JSONOutput; +class SymbolIndex; /// This class provides implementation of an LSP server, glueing the JSON /// dispatch and ClangdServer together. @@ -35,7 +36,8 @@ const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir, - bool BuildDynamicSymbolIndex); + bool BuildDynamicSymbolIndex, + SymbolIndex *StaticIdx = nullptr); /// Run LSP server loop, receiving input for it from \p In. \p In must be /// opened in binary mode. Output will be written using Out variable passed to Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -283,10 +283,12 @@ const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir, - bool BuildDynamicSymbolIndex) + bool BuildDynamicSymbolIndex, + SymbolIndex *StaticIdx) : Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, - StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {} + StorePreamblesInMemory, BuildDynamicSymbolIndex, StaticIdx, + ResourceDir) {} bool ClangdLSPServer::run(std::istream &In) { assert(!IsDone && "Run was called before"); Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -200,11 +200,15 @@ /// If \p BuildDynamicSymbolIndex is true, ClangdServer builds a dynamic /// in-memory index for symbols in all opened files and uses the index to /// augment code completion results. + /// + /// If \p StaticIdx is set, ClangdServer uses the index for global code + /// completion. ClangdServer(GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, bool BuildDynamicSymbolIndex = false, + SymbolIndex *StaticIdx = nullptr, llvm::Optional ResourceDir = llvm::None); /// Set the root path of the workspace. @@ -338,6 +342,11 @@ DraftStore DraftMgr; /// If set, this manages index for symbols in opened files. std::unique_ptr FileIdx; + /// If set, this provides static index for project-wide global symbols. + /// clangd global code completion result will come from the static index and + /// the `FileIdx` above. + /// No owned, the life time is managed by clangd embedders. + SymbolIndex *StaticIdx; CppFileCollection Units; std::string ResourceDir; // If set, this represents the workspace path. Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -134,10 +134,11 @@ FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, - bool BuildDynamicSymbolIndex, + bool BuildDynamicSymbolIndex, SymbolIndex *StaticIdx, llvm::Optional ResourceDir) : CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr), + StaticIdx(StaticIdx), // Pass a callback into `Units` to extract symbols from a newly parsed // file and rebuild the file index synchronously each time an AST is // parsed. @@ -251,6 +252,8 @@ auto CodeCompleteOpts = Opts; if (FileIdx) CodeCompleteOpts.Index = FileIdx.get(); + if (StaticIdx) + CodeCompleteOpts.StaticIndex = StaticIdx; // A task that will be run asynchronously. auto Task = // 'mutable' to reassign Preamble variable. Index: clangd/CodeComplete.h =================================================================== --- clangd/CodeComplete.h +++ clangd/CodeComplete.h @@ -67,6 +67,10 @@ /// FIXME(ioeric): we might want a better way to pass the index around inside /// clangd. const SymbolIndex *Index = nullptr; + + // Populated internally by clangd, do not set. + /// Static index for project-wide global symbols. + const SymbolIndex *StaticIndex = nullptr; }; /// Get code completions at a specified \p Pos in \p FileName. Index: clangd/CodeComplete.cpp =================================================================== --- clangd/CodeComplete.cpp +++ clangd/CodeComplete.cpp @@ -23,6 +23,7 @@ #include "clang/Frontend/FrontendActions.h" #include "clang/Sema/CodeCompleteConsumer.h" #include "clang/Sema/Sema.h" +#include "llvm/Support/Format.h" #include namespace clang { @@ -550,10 +551,27 @@ } CompletionItem indexCompletionItem(const Symbol &Sym, llvm::StringRef Filter, - const SpecifiedScope &SSInfo) { + const SpecifiedScope &SSInfo, + llvm::StringRef DebuggingLabel = "") { CompletionItem Item; Item.kind = toCompletionItemKind(Sym.SymInfo.Kind); - Item.label = Sym.Name; + // Add DebuggingLabel to the completion results if DebuggingLabel is not + // empty. + // + // For symbols from static index, there are prefix "[G]" in the + // results (which is used for debugging purpose). + // So completion list will be like: + // clang::symbol_from_dynamic_index + // [G]clang::symbol_from_static_index + // + // FIXME: Find out a better way to show the index source. + if (!DebuggingLabel.empty()) { + llvm::raw_string_ostream Label(Item.label); + Label << llvm::format("[%s]%s", DebuggingLabel.str().c_str(), + Sym.Name.str().c_str()); + } else { + Item.label = Sym.Name; + } // FIXME(ioeric): support inserting/replacing scope qualifiers. Item.insertText = Sym.Name; // FIXME(ioeric): support snippets. @@ -571,7 +589,8 @@ void completeWithIndex(const Context &Ctx, const SymbolIndex &Index, llvm::StringRef Code, const SpecifiedScope &SSInfo, - llvm::StringRef Filter, CompletionList *Items) { + llvm::StringRef Filter, CompletionList *Items, + llvm::StringRef DebuggingLabel = "") { FuzzyFindRequest Req; Req.Query = Filter; // FIXME(ioeric): add more possible scopes based on using namespaces and @@ -579,8 +598,9 @@ StringRef Scope = SSInfo.Resolved.empty() ? SSInfo.Written : SSInfo.Resolved; Req.Scopes = {Scope.trim(':').str()}; - Items->isIncomplete = !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) { - Items->items.push_back(indexCompletionItem(Sym, Filter, SSInfo)); + Items->isIncomplete |= !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) { + Items->items.push_back( + indexCompletionItem(Sym, Filter, SSInfo, DebuggingLabel)); }); } @@ -633,13 +653,19 @@ invokeCodeComplete(Ctx, std::move(Consumer), Opts.getClangCompleteOpts(), FileName, Command, Preamble, Contents, Pos, std::move(VFS), std::move(PCHs)); - if (Opts.Index && CompletedName.SSInfo) { - if (!Results.items.empty()) - log(Ctx, "WARNING: Got completion results from sema for completion on " - "qualified ID while symbol index is provided."); - Results.items.clear(); - completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo, - CompletedName.Filter, &Results); + + // Got scope specifier (ns::f^) for code completion from sema, try to query + // global symbols from indexes. + if (CompletedName.SSInfo) { + // FIXME: figure out a good algorithm to merge symbols from different + // sources (dynamic index, static index, AST symbols from clang's completion + // engine). + if (Opts.Index) + completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo, + CompletedName.Filter, &Results); + if (Opts.StaticIndex) + completeWithIndex(Ctx, *Opts.StaticIndex, Contents, *CompletedName.SSInfo, + CompletedName.Filter, &Results, /*DebuggingLabel=*/"G"); } return Results; } Index: clangd/index/MemIndex.h =================================================================== --- clangd/index/MemIndex.h +++ clangd/index/MemIndex.h @@ -24,6 +24,9 @@ /// accessible as long as `Symbols` is kept alive. void build(std::shared_ptr> Symbols); + /// \brief Build index from a symbol slab. + static std::unique_ptr build(SymbolSlab Slab); + bool fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req, llvm::function_ref Callback) const override; Index: clangd/index/MemIndex.cpp =================================================================== --- clangd/index/MemIndex.cpp +++ clangd/index/MemIndex.cpp @@ -53,5 +53,21 @@ return true; } +std::unique_ptr MemIndex::build(SymbolSlab Slab) { + struct Snapshot { + SymbolSlab Slab; + std::vector Pointers; + }; + auto Snap = std::make_shared(); + Snap->Slab = std::move(Slab); + for (auto &Sym : Snap->Slab) + Snap->Pointers.push_back(&Sym); + auto S = std::shared_ptr>(std::move(Snap), + &Snap->Pointers); + auto MemIdx = llvm::make_unique(); + MemIdx->build(std::move(S)); + return std::move(MemIdx); +} + } // namespace clangd } // namespace clang Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -11,6 +11,7 @@ #include "JSONRPCDispatcher.h" #include "Path.h" #include "Trace.h" +#include "index/SymbolYAML.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -26,7 +27,24 @@ namespace { enum class PCHStorageFlag { Disk, Memory }; + +// Build an in-memory static index for global symbols from a YAML-format file. +// The size of global symbols should be relatively small, so that all symbols +// can be managed in memory. +std::unique_ptr BuildStaticIndex(llvm::StringRef YamlSymbolFile) { + auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile); + if (!Buffer) { + llvm::errs() << "Can't open " << YamlSymbolFile << "\n"; + return nullptr; + } + auto Slab = SymbolFromYAML(Buffer.get()->getBuffer()); + SymbolSlab::Builder SymsBuilder; + for (auto Sym : Slab) + SymsBuilder.insert(Sym); + + return MemIndex::build(std::move(SymsBuilder).build()); } +} // namespace static llvm::cl::opt CompileCommandsDir( "compile-commands-dir", @@ -97,6 +115,15 @@ "use index built from symbols in opened files"), llvm::cl::init(false), llvm::cl::Hidden); +static llvm::cl::opt YamlSymbolFile( + "yaml-symbol-file", + llvm::cl::desc( + "YAML-format global symbol file to build the static index. Clangd will " + "use the static index for global code completion.\n" + "WARNING: This option is experimental only, and will be removed " + "eventually. Don't rely on it."), + llvm::cl::init(""), llvm::cl::Hidden); + int main(int argc, char *argv[]) { llvm::cl::ParseCommandLineOptions(argc, argv, "clangd"); @@ -182,13 +209,16 @@ // Change stdin to binary to not lose \r\n on windows. llvm::sys::ChangeStdinToBinary(); + std::unique_ptr StaticIdx; + if (EnableIndexBasedCompletion && !YamlSymbolFile.empty()) + StaticIdx = BuildStaticIndex(YamlSymbolFile); clangd::CodeCompleteOptions CCOpts; CCOpts.EnableSnippets = EnableSnippets; CCOpts.IncludeIneligibleResults = IncludeIneligibleResults; // Initialize and run ClangdLSPServer. ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, CCOpts, ResourceDirRef, CompileCommandsDirPath, - EnableIndexBasedCompletion); + EnableIndexBasedCompletion, StaticIdx.get()); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode; Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -453,12 +453,6 @@ std::unique_ptr simpleIndexFromSymbols( std::vector> Symbols) { - auto I = llvm::make_unique(); - struct Snapshot { - SymbolSlab Slab; - std::vector Pointers; - }; - auto Snap = std::make_shared(); SymbolSlab::Builder Slab; for (const auto &Pair : Symbols) { Symbol Sym; @@ -475,13 +469,7 @@ Sym.SymInfo.Kind = Pair.second; Slab.insert(Sym); } - Snap->Slab = std::move(Slab).build(); - for (auto &Iter : Snap->Slab) - Snap->Pointers.push_back(&Iter); - auto S = std::shared_ptr>(std::move(Snap), - &Snap->Pointers); - I->build(std::move(S)); - return std::move(I); + return MemIndex::build(std::move(Slab).build()); } TEST(CompletionTest, NoIndex) { @@ -496,6 +484,23 @@ EXPECT_THAT(Results.items, Has("No")); } +TEST(CompletionTest, StaticAndDynamicIndex) { + clangd::CodeCompleteOptions Opts; + auto StaticIdx = + simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class}}); + Opts.StaticIndex = StaticIdx.get(); + auto DynamicIdx = + simpleIndexFromSymbols({{"ns::foo", index::SymbolKind::Function}}); + Opts.Index = DynamicIdx.get(); + + auto Results = completions(R"cpp( + void f() { ::ns::^ } + )cpp", + Opts); + EXPECT_THAT(Results.items, Contains(Labeled("[G]XYZ"))); + EXPECT_THAT(Results.items, Contains(Labeled("foo"))); +} + TEST(CompletionTest, SimpleIndexBased) { clangd::CodeCompleteOptions Opts; auto I = simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class},