Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -34,7 +34,8 @@ bool StorePreamblesInMemory, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, - llvm::Optional CompileCommandsDir); + llvm::Optional CompileCommandsDir, + bool BuildDynamicSymbolIndex); /// Run LSP server loop, receiving input for it from \p In. \p In must be /// opened in binary mode. Output will be written using Out variable passed to Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -282,10 +282,11 @@ bool StorePreamblesInMemory, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, - llvm::Optional CompileCommandsDir) + llvm::Optional CompileCommandsDir, + bool BuildDynamicSymbolIndex) : Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, - StorePreamblesInMemory, ResourceDir) {} + StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {} bool ClangdLSPServer::run(std::istream &In) { assert(!IsDone && "Run was called before"); Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -17,6 +17,7 @@ #include "Function.h" #include "GlobalCompilationDatabase.h" #include "Protocol.h" +#include "index/FileIndex.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" @@ -195,10 +196,15 @@ /// /// \p StorePreamblesInMemory defines whether the Preambles generated by /// clangd are stored in-memory or on disk. + /// + /// If \p BuildDynamicSymbolIndex is true, ClangdServer builds a dynamic + /// in-memory index for symbols in all opened files and uses the index to + /// augment code completion results. ClangdServer(GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, + bool BuildDynamicSymbolIndex = false, llvm::Optional ResourceDir = llvm::None); /// Set the root path of the workspace. @@ -330,6 +336,8 @@ DiagnosticsConsumer &DiagConsumer; FileSystemProvider &FSProvider; DraftStore DraftMgr; + /// If set, this manages index for symbols in opened files. + std::unique_ptr FileIdx; CppFileCollection Units; std::string ResourceDir; // If set, this represents the workspace path. Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -134,8 +134,19 @@ FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, + bool BuildDynamicSymbolIndex, llvm::Optional ResourceDir) : CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), + FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr), + // Pass a callback into `Units` to extract symbols from a newly parsed + // file and rebuild the file index synchronously each time an AST is + // parsed. + // FIXME(ioeric): this can be slow and we may be able to index on less + // critical paths. + Units(FileIdx + ? [this](const Context &Ctx, PathRef Path, + ParsedAST *AST) { FileIdx->update(Ctx, Path, AST); } + : ASTParsedCallback()), ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()), PCHs(std::make_shared()), StorePreamblesInMemory(StorePreamblesInMemory), @@ -238,6 +249,8 @@ Resources->getPossiblyStalePreamble(); // Copy completion options for passing them to async task handler. auto CodeCompleteOpts = Opts; + if (FileIdx) + CodeCompleteOpts.Index = FileIdx.get(); // A task that will be run asynchronously. auto Task = // 'mutable' to reassign Preamble variable. Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -136,6 +136,9 @@ mutable llvm::Optional AST; }; +using ASTParsedCallback = + std::function; + /// Manages resources, required by clangd. Allows to rebuild file with new /// contents, and provides AST and Preamble for it. class CppFile : public std::enable_shared_from_this { @@ -145,12 +148,14 @@ static std::shared_ptr Create(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs); + std::shared_ptr PCHs, + ASTParsedCallback ASTCallback); private: CppFile(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs); + std::shared_ptr PCHs, + ASTParsedCallback ASTCallback); public: CppFile(CppFile const &) = delete; @@ -252,6 +257,9 @@ std::shared_ptr LatestAvailablePreamble; /// Utility class, required by clang. std::shared_ptr PCHs; + /// This is called after the file is parsed. This can be nullptr if there is + /// no callback. + ASTParsedCallback ASTCallback; }; /// Get the beginning SourceLocation at a specified \p Pos. Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -357,17 +357,21 @@ std::shared_ptr CppFile::Create(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs) { - return std::shared_ptr(new CppFile( - FileName, std::move(Command), StorePreamblesInMemory, std::move(PCHs))); + std::shared_ptr PCHs, + ASTParsedCallback ASTCallback) { + return std::shared_ptr( + new CppFile(FileName, std::move(Command), StorePreamblesInMemory, + std::move(PCHs), std::move(ASTCallback))); } CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs) + std::shared_ptr PCHs, + ASTParsedCallback ASTCallback) : FileName(FileName), Command(std::move(Command)), StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0), - RebuildInProgress(false), PCHs(std::move(PCHs)) { + RebuildInProgress(false), PCHs(std::move(PCHs)), + ASTCallback(std::move(ASTCallback)) { // FIXME(ibiryukov): we should pass a proper Context here. log(Context::empty(), "Opened file " + FileName + " with command [" + this->Command.Directory + "] " + @@ -570,6 +574,8 @@ if (NewAST) { Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(), NewAST->getDiagnostics().end()); + if (That->ASTCallback) + That->ASTCallback(Ctx, That->FileName, NewAST.getPointer()); } else { // Don't report even Preamble diagnostics if we coulnd't build AST. Diagnostics.clear(); Index: clangd/ClangdUnitStore.h =================================================================== --- clangd/ClangdUnitStore.h +++ clangd/ClangdUnitStore.h @@ -25,6 +25,11 @@ /// Thread-safe mapping from FileNames to CppFile. class CppFileCollection { public: + /// \p ASTCallback is called when a file is parsed synchronously. This should + /// not be expensive since it blocks diagnostics. + explicit CppFileCollection(ASTParsedCallback ASTCallback) + : ASTCallback(std::move(ASTCallback)) {} + std::shared_ptr getOrCreateFile(PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory, @@ -38,7 +43,7 @@ It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(Command), StorePreamblesInMemory, - std::move(PCHs))) + std::move(PCHs), ASTCallback)) .first; } return It->second; @@ -85,6 +90,7 @@ std::mutex Mutex; llvm::StringMap> OpenedFiles; + ASTParsedCallback ASTCallback; }; } // namespace clangd } // namespace clang Index: clangd/ClangdUnitStore.cpp =================================================================== --- clangd/ClangdUnitStore.cpp +++ clangd/ClangdUnitStore.cpp @@ -41,13 +41,14 @@ It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory, - std::move(PCHs))) + std::move(PCHs), ASTCallback)) .first; } else if (!compileCommandsAreEqual(It->second->getCompileCommand(), NewCommand)) { Result.RemovedFile = std::move(It->second); - It->second = CppFile::Create(File, std::move(NewCommand), - StorePreamblesInMemory, std::move(PCHs)); + It->second = + CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory, + std::move(PCHs), ASTCallback); } Result.FileInCollection = It->second; return Result; Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -90,6 +90,13 @@ "Trace internal events and timestamps in chrome://tracing JSON format"), llvm::cl::init(""), llvm::cl::Hidden); +static llvm::cl::opt EnableIndexBasedCompletion( + "enable-index-based-completion", + llvm::cl::desc( + "Enable index-based global code completion (experimental). Clangd will " + "use index built from symbols in opened files"), + llvm::cl::init(false), llvm::cl::Hidden); + int main(int argc, char *argv[]) { llvm::cl::ParseCommandLineOptions(argc, argv, "clangd"); @@ -180,7 +187,8 @@ CCOpts.IncludeIneligibleResults = IncludeIneligibleResults; // Initialize and run ClangdLSPServer. ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, - CCOpts, ResourceDirRef, CompileCommandsDirPath); + CCOpts, ResourceDirRef, CompileCommandsDirPath, + EnableIndexBasedCompletion); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode; Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -558,6 +558,38 @@ EXPECT_THAT(Results.items, Has("XYZ", CompletionItemKind::Class)); } +TEST(CompletionTest, ASTIndexMultiFile) { + MockFSProvider FS; + MockCompilationDatabase CDB; + IgnoreDiagnostics DiagConsumer; + ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, + /*BuildDynamicSymbolIndex=*/true); + + Server + .addDocument(Context::empty(), getVirtualTestFilePath("foo.cpp"), R"cpp( + namespace ns { class XYZ {}; void foo() {} } + )cpp") + .wait(); + + auto File = getVirtualTestFilePath("bar.cpp"); + auto Test = parseTextMarker(R"cpp( + namespace ns { class XXX {}; void fooooo() {} } + void f() { ns::^ } + )cpp"); + Server.addDocument(Context::empty(), File, Test.Text).wait(); + + auto Results = Server.codeComplete(Context::empty(), File, Test.MarkerPos, {}) + .get() + .second.Value; + // "XYZ" and "foo" are not included in the file being completed but are still + // visible through the index. + EXPECT_THAT(Results.items, Has("XYZ", CompletionItemKind::Class)); + EXPECT_THAT(Results.items, Has("foo", CompletionItemKind::Function)); + EXPECT_THAT(Results.items, Has("XXX", CompletionItemKind::Class)); + EXPECT_THAT(Results.items, Has("fooooo", CompletionItemKind::Function)); +} + } // namespace } // namespace clangd } // namespace clang