Index: clangd/index/Background.h =================================================================== --- clangd/index/Background.h +++ clangd/index/Background.h @@ -74,7 +74,6 @@ // The indexing happens in a background thread, so the symbols will be // available sometime later. void enqueue(const std::vector &ChangedFiles); - void enqueue(const std::string &File); // Cause background threads to stop after ther current task, any remaining // tasks will be discarded. @@ -106,13 +105,19 @@ std::mutex DigestsMu; BackgroundIndexStorage::Factory IndexStorageFactory; + // Loads the shards for all the sources and headers of the Cmd. Returns the + // list of dependencies for this Cmd and whether they need to be re-indexed. + std::vector< + std::pair> + loadShard(const tooling::CompileCommand &Cmd, + BackgroundIndexStorage *IndexStorage); + llvm::Error loadShards(std::vector ChangedFiles); + void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage); // queue management using Task = std::function; void run(); // Main loop executed by Thread. Runs tasks from Queue. void enqueueTask(Task T); - void enqueueLocked(tooling::CompileCommand Cmd, - BackgroundIndexStorage *IndexStorage); std::mutex QueueMu; unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks. std::condition_variable QueueCV; Index: clangd/index/Background.cpp =================================================================== --- clangd/index/Background.cpp +++ clangd/index/Background.cpp @@ -87,6 +87,17 @@ return IG; } + +llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) { + llvm::SmallString<128> AbsolutePath; + if (sys::path::is_absolute(Cmd.Filename)) { + AbsolutePath = Cmd.Filename; + } else { + AbsolutePath = Cmd.Directory; + sys::path::append(AbsolutePath, Cmd.Filename); + } + return AbsolutePath; +} } // namespace BackgroundIndex::BackgroundIndex( @@ -164,35 +175,23 @@ enqueueTask([this, ChangedFiles] { trace::Span Tracer("BackgroundIndexEnqueue"); // We're doing this asynchronously, because we'll read shards here too. - // FIXME: read shards here too. - log("Enqueueing {0} commands for indexing", ChangedFiles.size()); SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size())); - // We shuffle the files because processing them in a random order should - // quickly give us good coverage of headers in the project. - std::vector Permutation(ChangedFiles.size()); - std::iota(Permutation.begin(), Permutation.end(), 0); - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Permutation.begin(), Permutation.end(), Generator); - - for (const unsigned I : Permutation) - enqueue(ChangedFiles[I]); + if (auto Error = loadShards(std::move(ChangedFiles))) + log("Loading shards failed: {0}", std::move(Error)); }); } -void BackgroundIndex::enqueue(const std::string &File) { - ProjectInfo Project; - if (auto Cmd = CDB.getCompileCommand(File, &Project)) { - auto *Storage = IndexStorageFactory(Project.SourceRoot); - enqueueTask(Bind( - [this, File, Storage](tooling::CompileCommand Cmd) { - Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir); - if (auto Error = index(std::move(Cmd), Storage)) - log("Indexing {0} failed: {1}", File, std::move(Error)); - }, - std::move(*Cmd))); - } +void BackgroundIndex::enqueue(tooling::CompileCommand Cmd, + BackgroundIndexStorage *Storage) { + enqueueTask(Bind( + [this, Storage](tooling::CompileCommand Cmd) { + Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir); + if (auto Error = index(std::move(Cmd), Storage)) + log("Indexing {0} failed: {1}", Cmd.Filename, std::move(Error)); + }, + std::move(Cmd))); } void BackgroundIndex::enqueueTask(Task T) { @@ -336,11 +335,6 @@ llvm::StringMap DigestsSnapshot; { std::lock_guard Lock(DigestsMu); - if (IndexedFileDigests.lookup(AbsolutePath) == Hash) { - vlog("No need to index {0}, already up to date", AbsolutePath); - return Error::success(); - } - DigestsSnapshot = IndexedFileDigests; } @@ -406,5 +400,103 @@ return Error::success(); } +std::vector> +BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd, + BackgroundIndexStorage *IndexStorage) { + // Consumes Symbols and Refs in Shard. + auto LoadShard = [&](llvm::StringRef AbsolutePath, IndexFileIn *Shard, + const IncludeGraphNode &IGN) { + if (!Shard) + return; + std::unique_ptr SS; + std::unique_ptr RS; + if (Shard->Symbols) + SS = llvm::make_unique(std::move(*Shard->Symbols)); + if (Shard->Refs) + RS = llvm::make_unique(std::move(*Shard->Refs)); + // Load shard information into background-index. + { + std::lock_guard Lock(DigestsMu); + // This can override a newer version that is added in another thread, + // if this thread sees the older version but finishes later. This + // should be rare in practice. + IndexedFileDigests[AbsolutePath] = IGN.Digest; + IndexedSymbols.update(AbsolutePath, std::move(SS), std::move(RS)); + } + }; + + auto FS = FSProvider.getFileSystem(); + std::vector> Dependencies; + Dependencies.push_back({getAbsolutePath(Cmd).str(), true}); + for (size_t CurrentDependency = 0; CurrentDependency < Dependencies.size(); + CurrentDependency++) { + auto Dependency = Dependencies[CurrentDependency]; + auto Shard = IndexStorage->loadShard(Dependency.first); + + if (!Shard || !Shard->Sources) { + elog("Failed to load shard for: {0}", Dependency.first); + continue; + } + for (const auto &I : *Shard->Sources) { + if (auto U = URI::parse(I.getKey())) { + if (auto AbsolutePath = URI::resolve(*U, Dependency.first)) { + if (*AbsolutePath != Dependency.first) { + Dependencies.push_back({AbsolutePath.get(), true}); + continue; + } + + // If we found source file info for current file. + LoadShard(AbsolutePath.get(), Shard.get(), I.getValue()); + // Check if the source needs re-indexing. + // Get the digest, skip it if file doesn't exist. + auto Buf = FS->getBufferForFile(AbsolutePath.get()); + if (!Buf) + continue; + // If digests match then dependency doesn't need re-indexing. + Dependency.second = + digest(Buf->get()->getBuffer()) != I.getValue().Digest; + } + } + } + Dependencies[CurrentDependency] = std::move(Dependency); + } + + // FIXME: this should rebuild once-in-a-while, not after every file. + // At that point we should use Dex, too. + vlog("Rebuilding automatic index"); + reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge)); + + return Dependencies; +} + +llvm::Error BackgroundIndex::loadShards(std::vector ChangedFiles) { + llvm::StringSet<> BeingReindexed; + for (const auto &File : ChangedFiles) { + ProjectInfo PI; + auto Cmd = CDB.getCompileCommand(File, &PI); + if (!Cmd) + continue; + BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot); + auto Dependencies = loadShard(*Cmd, IndexStorage); + for (const auto &Dependency : Dependencies) { + // FIXME: Currently, we simply schedule indexing on a TU whenever any of + // its dependencies needs re-indexing. We might do it more smartly by + // figuring out a minimal set of TUs that will cover all the stale + // dependencies. + if (Dependency.second && !BeingReindexed.count(Dependency.first)) { + enqueue(std::move(*Cmd), IndexStorage); + // Mark all of this TUs dependencies as to-be-indexed so that we won't + // try to re-index those. + std::for_each(Dependencies.begin(), Dependencies.end(), + [&BeingReindexed](decltype(Dependency) Dependency) { + BeingReindexed.insert(Dependency.first); + }); + break; + } + } + } + return Error::success(); +} + } // namespace clangd } // namespace clang Index: test/clangd/background-index.test =================================================================== --- test/clangd/background-index.test +++ test/clangd/background-index.test @@ -16,6 +16,5 @@ # RUN: ls %t/.clangd-index/foo.cpp.*.idx # Test the index is read from disk: delete code and restart clangd. -# FIXME: This test currently fails as we don't read the index yet. # RUN: rm %t/foo.cpp -# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | not FileCheck %t/definition.jsonrpc +# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | FileCheck %t/definition.jsonrpc Index: unittests/clangd/BackgroundIndexTests.cpp =================================================================== --- unittests/clangd/BackgroundIndexTests.cpp +++ unittests/clangd/BackgroundIndexTests.cpp @@ -7,6 +7,7 @@ using testing::_; using testing::AllOf; +using testing::Contains; using testing::Not; using testing::UnorderedElementsAre; @@ -120,7 +121,7 @@ FileURI("unittest:///root/B.cc")})); } -TEST_F(BackgroundIndexTest, ShardStorageWriteTest) { +TEST_F(BackgroundIndexTest, ShardStorageTest) { MockFSProvider FS; FS.Files[testPath("root/A.h")] = R"cpp( void common(); @@ -149,6 +150,16 @@ EXPECT_EQ(CacheHits, 0U); EXPECT_EQ(Storage.size(), 2U); + { + OverlayCDB CDB(/*Base=*/nullptr); + BackgroundIndex Idx(Context::empty(), "", FS, CDB, + [&](llvm::StringRef) { return &MSS; }); + CDB.setCompileCommand(testPath("root"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + } + EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache. + EXPECT_EQ(Storage.size(), 2U); + auto ShardHeader = MSS.loadShard(testPath("root/A.h")); EXPECT_NE(ShardHeader, nullptr); EXPECT_THAT( @@ -208,5 +219,73 @@ UnorderedElementsAre("unittest:///root/B.h")); } +TEST_F(BackgroundIndexTest, ShardStorageLoad) { + MockFSProvider FS; + FS.Files[testPath("root/A.h")] = R"cpp( + void common(); + void f_b(); + class A_CC {}; + )cpp"; + FS.Files[testPath("root/A.cc")] = + "#include \"A.h\"\nvoid g() { (void)common; }"; + + llvm::StringMap Storage; + size_t CacheHits = 0; + MemoryShardStorage MSS(Storage, CacheHits); + + tooling::CompileCommand Cmd; + Cmd.Filename = testPath("root/A.cc"); + Cmd.Directory = testPath("root"); + Cmd.CommandLine = {"clang++", testPath("root/A.cc")}; + // Check nothing is loaded from Storage, but A.cc and A.h has been stored. + { + OverlayCDB CDB(/*Base=*/nullptr); + BackgroundIndex Idx(Context::empty(), "", FS, CDB, + [&](llvm::StringRef) { return &MSS; }); + CDB.setCompileCommand(testPath("root/A.cc"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + } + + // Change header. + FS.Files[testPath("root/A.h")] = R"cpp( + void common(); + void f_b(); + class A_CC {}; + class A_CCnew {}; + )cpp"; + { + OverlayCDB CDB(/*Base=*/nullptr); + BackgroundIndex Idx(Context::empty(), "", FS, CDB, + [&](llvm::StringRef) { return &MSS; }); + CDB.setCompileCommand(testPath("root"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + } + EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache. + + // Check if the new symbol has arrived. + auto ShardHeader = MSS.loadShard(testPath("root/A.h")); + EXPECT_NE(ShardHeader, nullptr); + EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew"))); + + // Change source. + FS.Files[testPath("root/A.cc")] = + "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}"; + { + CacheHits = 0; + OverlayCDB CDB(/*Base=*/nullptr); + BackgroundIndex Idx(Context::empty(), "", FS, CDB, + [&](llvm::StringRef) { return &MSS; }); + CDB.setCompileCommand(testPath("root"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + } + EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache. + + // Check if the new symbol has arrived. + auto ShardSource = MSS.loadShard(testPath("root/A.cc")); + EXPECT_NE(ShardHeader, nullptr); + EXPECT_THAT(*ShardSource->Symbols, + Contains(AllOf(Named("f_b"), Declared(), Defined()))); +} + } // namespace clangd } // namespace clang