Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.h +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h @@ -57,6 +57,7 @@ // Calls have signature void(const Params&, Callback). void onInitialize(const InitializeParams &, Callback); void onShutdown(const ShutdownParams &, Callback); + void onSync(const NoParams &, Callback); void onDocumentDidOpen(const DidOpenTextDocumentParams &); void onDocumentDidChange(const DidChangeTextDocumentParams &); void onDocumentDidClose(const DidCloseTextDocumentParams &); Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp @@ -337,6 +337,17 @@ Reply(nullptr); } +// sync is a clangd extension: it blocks until all background work completes. +// It blocks the calling thread, so no messages are processed until it returns! +void ClangdLSPServer::onSync(const NoParams &Params, + Callback Reply) { + if (Server->blockUntilIdleForTest(/*TimeoutSeconds=*/60)) + Reply(nullptr); + else + Reply(createStringError(llvm::inconvertibleErrorCode(), + "Not idle after a minute")); +} + void ClangdLSPServer::onDocumentDidOpen( const DidOpenTextDocumentParams &Params) { PathRef File = Params.textDocument.uri.file(); @@ -701,6 +712,7 @@ // clang-format off MsgHandler->bind("initialize", &ClangdLSPServer::onInitialize); MsgHandler->bind("shutdown", &ClangdLSPServer::onShutdown); + MsgHandler->bind("sync", &ClangdLSPServer::onSync); MsgHandler->bind("textDocument/rangeFormatting", &ClangdLSPServer::onDocumentRangeFormatting); MsgHandler->bind("textDocument/onTypeFormatting", &ClangdLSPServer::onDocumentOnTypeFormatting); MsgHandler->bind("textDocument/formatting", &ClangdLSPServer::onDocumentFormatting); Index: clang-tools-extra/trunk/clangd/ClangdServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.h +++ clang-tools-extra/trunk/clangd/ClangdServer.h @@ -18,6 +18,7 @@ #include "GlobalCompilationDatabase.h" #include "Protocol.h" #include "TUScheduler.h" +#include "index/Background.h" #include "index/FileIndex.h" #include "index/Index.h" #include "clang/Tooling/CompilationDatabase.h" @@ -78,6 +79,9 @@ /// Use a heavier and faster in-memory index implementation. /// FIXME: we should make this true if it isn't too slow to build!. bool HeavyweightDynamicSymbolIndex = false; + /// If true, ClangdServer automatically indexes files in the current project + /// on background threads. The index is stored in the project root. + bool BackgroundIndex = false; /// If set, use this index to augment code completion results. SymbolIndex *StaticIndex = nullptr; @@ -234,11 +238,13 @@ // - the dynamic index owned by ClangdServer (DynamicIdx) // - the static index passed to the constructor // - a merged view of a static and dynamic index (MergedIndex) - const SymbolIndex *Index; + const SymbolIndex *Index = nullptr; // If present, an index of symbols in open files. Read via *Index. std::unique_ptr DynamicIdx; - // If present, storage for the merged static/dynamic index. Read via *Index. - std::unique_ptr MergedIdx; + // If present, the new "auto-index" maintained in background threads. + std::unique_ptr BackgroundIdx; + // Storage for merged views of the various indexes. + std::vector> MergedIdx; // GUARDED_BY(CachedCompletionFuzzyFindRequestMutex) llvm::StringMap> Index: clang-tools-extra/trunk/clangd/ClangdServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdServer.cpp @@ -121,16 +121,25 @@ llvm::make_unique(DynamicIdx.get(), DiagConsumer), Opts.UpdateDebounce, Opts.RetentionPolicy) { - if (DynamicIdx && Opts.StaticIndex) { - MergedIdx = - llvm::make_unique(DynamicIdx.get(), Opts.StaticIndex); - Index = MergedIdx.get(); - } else if (DynamicIdx) - Index = DynamicIdx.get(); - else if (Opts.StaticIndex) - Index = Opts.StaticIndex; - else - Index = nullptr; + // Adds an index to the stack, at higher priority than existing indexes. + auto AddIndex = [&](SymbolIndex *Idx) { + if (this->Index != nullptr) { + MergedIdx.push_back(llvm::make_unique(Idx, this->Index)); + this->Index = MergedIdx.back().get(); + } else { + this->Index = Idx; + } + }; + if (Opts.StaticIndex) + AddIndex(Opts.StaticIndex); + if (Opts.BackgroundIndex) { + BackgroundIdx = llvm::make_unique( + Context::current().clone(), ResourceDir, FSProvider, CDB, + BackgroundIndexStorage::createDiskBackedStorageFactory()); + AddIndex(BackgroundIdx.get()); + } + if (DynamicIdx) + AddIndex(DynamicIdx.get()); } void ClangdServer::addDocument(PathRef File, StringRef Contents, @@ -501,7 +510,9 @@ LLVM_NODISCARD bool ClangdServer::blockUntilIdleForTest(Optional TimeoutSeconds) { - return WorkScheduler.blockUntilIdle(timeoutSeconds(TimeoutSeconds)); + return WorkScheduler.blockUntilIdle(timeoutSeconds(TimeoutSeconds)) && + (!BackgroundIdx || + BackgroundIdx->blockUntilIdleForTest(TimeoutSeconds)); } } // namespace clangd Index: clang-tools-extra/trunk/clangd/GlobalCompilationDatabase.cpp =================================================================== --- clang-tools-extra/trunk/clangd/GlobalCompilationDatabase.cpp +++ clang-tools-extra/trunk/clangd/GlobalCompilationDatabase.cpp @@ -87,6 +87,8 @@ Project->SourceRoot = Path; } } + // FIXME: getAllFiles() may return relative paths, we need absolute paths. + // Hopefully the fix is to change JSONCompilationDatabase and the interface. if (CDB && !Cached) OnCommandChanged.broadcast(CDB->getAllFiles()); return CDB; @@ -112,7 +114,7 @@ return It->second; } } - return Base ? Base->getCompileCommand(File) : None; + return Base ? Base->getCompileCommand(File, Project) : None; } tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const { Index: clang-tools-extra/trunk/clangd/index/Background.h =================================================================== --- clang-tools-extra/trunk/clangd/index/Background.h +++ clang-tools-extra/trunk/clangd/index/Background.h @@ -81,7 +81,8 @@ void stop(); // Wait until the queue is empty, to allow deterministic testing. - void blockUntilIdleForTest(); + LLVM_NODISCARD bool + blockUntilIdleForTest(llvm::Optional TimeoutSeconds = 10); using FileDigest = decltype(llvm::SHA1::hash({})); Index: clang-tools-extra/trunk/clangd/index/Background.cpp =================================================================== --- clang-tools-extra/trunk/clangd/index/Background.cpp +++ clang-tools-extra/trunk/clangd/index/Background.cpp @@ -99,9 +99,11 @@ } } -void BackgroundIndex::blockUntilIdleForTest() { +bool BackgroundIndex::blockUntilIdleForTest( + llvm::Optional TimeoutSeconds) { std::unique_lock Lock(QueueMu); - QueueCV.wait(Lock, [&] { return Queue.empty() && NumActiveTasks == 0; }); + return wait(Lock, QueueCV, timeoutSeconds(TimeoutSeconds), + [&] { return Queue.empty() && NumActiveTasks == 0; }); } void BackgroundIndex::enqueue(const std::vector &ChangedFiles) { Index: clang-tools-extra/trunk/clangd/index/BackgroundIndexStorage.cpp =================================================================== --- clang-tools-extra/trunk/clangd/index/BackgroundIndexStorage.cpp +++ clang-tools-extra/trunk/clangd/index/BackgroundIndexStorage.cpp @@ -30,7 +30,7 @@ llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) + "." + llvm::toHex(digest(FilePath)) + ".idx"); - return ShardRoot.str(); + return ShardRootSS.str(); } // Uses disk as a storage for index shards. Creates a directory called @@ -101,6 +101,9 @@ // Creates and owns IndexStorages for multiple CDBs. class DiskBackedIndexStorageManager { public: + DiskBackedIndexStorageManager() + : IndexStorageMapMu(llvm::make_unique()) {} + // Creates or fetches to storage from cache for the specified CDB. BackgroundIndexStorage *operator()(llvm::StringRef CDBDirectory) { std::lock_guard Lock(*IndexStorageMapMu); Index: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp =================================================================== --- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp +++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp @@ -164,6 +164,12 @@ "eventually. Don't rely on it."), cl::init(""), cl::Hidden); +static cl::opt EnableBackgroundIndex( + "background-index", + cl::desc("Index project code in the background and persist index on disk. " + "Experimental"), + cl::init(false), cl::Hidden); + enum CompileArgsFrom { LSPCompileArgs, FilesystemCompileArgs }; static cl::opt CompileArgsFrom( "compile_args_from", cl::desc("The source of compile commands"), @@ -344,6 +350,7 @@ Opts.ResourceDir = ResourceDir; Opts.BuildDynamicSymbolIndex = EnableIndex; Opts.HeavyweightDynamicSymbolIndex = UseDex; + Opts.BackgroundIndex = EnableBackgroundIndex; std::unique_ptr StaticIdx; std::future AsyncIndexLoad; // Block exit while loading the index. if (EnableIndex && !IndexFile.empty()) { Index: clang-tools-extra/trunk/test/clangd/Inputs/background-index/compile_commands.json =================================================================== --- clang-tools-extra/trunk/test/clangd/Inputs/background-index/compile_commands.json +++ clang-tools-extra/trunk/test/clangd/Inputs/background-index/compile_commands.json @@ -0,0 +1,5 @@ +[{ + "directory": "DIRECTORY", + "command": "clang foo.cpp", + "file": "DIRECTORY/foo.cpp" +}] Index: clang-tools-extra/trunk/test/clangd/Inputs/background-index/definition.jsonrpc =================================================================== --- clang-tools-extra/trunk/test/clangd/Inputs/background-index/definition.jsonrpc +++ clang-tools-extra/trunk/test/clangd/Inputs/background-index/definition.jsonrpc @@ -0,0 +1,51 @@ +{ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "processId": 123, + "rootPath": "clangd", + "capabilities": {}, + "trace": "off" + } +} +--- +{ + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://DIRECTORY/bar.cpp", + "languageId": "cpp", + "version": 1, + "text": "#include \"foo.h\"\nint main(){\nreturn foo();\n}" + } + } +} +--- +{ + "jsonrpc": "2.0", + "id": 1, + "method": "sync", + "params": null +} +--- +{ + "jsonrpc": "2.0", + "id": 2, + "method": "textDocument/definition", + "params": { + "textDocument": { + "uri": "file://DIRECTORY/bar.cpp" + }, + "position": { + "line": 2, + "character": 8 + } + } +} +# CHECK: "uri": "file://DIRECTORY/foo.cpp" +--- +{"jsonrpc":"2.0","id":3,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} Index: clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.h =================================================================== --- clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.h +++ clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.h @@ -0,0 +1,4 @@ +#ifndef FOO_H +#define FOO_H +int foo(); +#endif Index: clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.cpp =================================================================== --- clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.cpp +++ clang-tools-extra/trunk/test/clangd/Inputs/background-index/foo.cpp @@ -0,0 +1,2 @@ +#include "foo.h" +int foo() { return 42; } Index: clang-tools-extra/trunk/test/clangd/background-index.test =================================================================== --- clang-tools-extra/trunk/test/clangd/background-index.test +++ clang-tools-extra/trunk/test/clangd/background-index.test @@ -0,0 +1,21 @@ +# We need to splice paths into file:// URIs for this test. +# UNSUPPORTED: win32 + +# Use a copy of inputs, as we'll mutate it (as will the background index). +# RUN: rm -rf %t +# RUN: cp -r %S/Inputs/background-index %t +# Need to embed the correct temp path in the actual JSON-RPC requests. +# RUN: sed -i -e "s|DIRECTORY|%t|" %t/* + +# We're editing bar.cpp, which includes foo.h. +# foo() is declared in foo.h and defined in foo.cpp. +# The background index should allow us to go-to-definition on foo(). +# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | FileCheck %t/definition.jsonrpc + +# Test that the index is writing files in the expected location. +# RUN: ls %t/.clangd-index/foo.cpp.*.idx + +# Test the index is read from disk: delete code and restart clangd. +# FIXME: This test currently fails as we don't read the index yet. +# RUN: rm %t/foo.cpp +# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | not FileCheck %t/definition.jsonrpc Index: clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp =================================================================== --- clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp +++ clang-tools-extra/trunk/unittests/clangd/BackgroundIndexTests.cpp @@ -90,7 +90,7 @@ Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")}; CDB.setCompileCommand(testPath("root"), Cmd); - Idx.blockUntilIdleForTest(); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); EXPECT_THAT( runFuzzyFind(Idx, ""), UnorderedElementsAre(Named("common"), Named("A_CC"), @@ -100,7 +100,7 @@ Cmd.CommandLine = {"clang++", Cmd.Filename}; CDB.setCompileCommand(testPath("root"), Cmd); - Idx.blockUntilIdleForTest(); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); // B_CC is dropped as we don't collect symbols from A.h in this compilation. EXPECT_THAT(runFuzzyFind(Idx, ""), UnorderedElementsAre(Named("common"), Named("A_CC"), @@ -141,7 +141,7 @@ BackgroundIndex Idx(Context::empty(), "", FS, CDB, [&](llvm::StringRef) { return &MSS; }); CDB.setCompileCommand(testPath("root"), Cmd); - Idx.blockUntilIdleForTest(); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); } EXPECT_EQ(CacheHits, 0U); EXPECT_EQ(Storage.size(), 2U);