diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -170,16 +170,20 @@ if (Opts.StaticIndex) AddIndex(Opts.StaticIndex); if (Opts.BackgroundIndex) { + BackgroundIndex::Options BGOpts; + BGOpts.ThreadPoolSize = std::max(Opts.AsyncThreadsCount, 1u); + BGOpts.OnProgress = [Callbacks](BackgroundQueue::Stats S) { + if (Callbacks) + Callbacks->onBackgroundIndexProgress(S); + }; + BGOpts.ContextProvider = [this](PathRef P) { + return createProcessingContext(P); + }; BackgroundIdx = std::make_unique( - Context::current().clone(), TFS, CDB, + TFS, CDB, BackgroundIndexStorage::createDiskBackedStorageFactory( [&CDB](llvm::StringRef File) { return CDB.getProjectInfo(File); }), - std::max(Opts.AsyncThreadsCount, 1u), - [Callbacks](BackgroundQueue::Stats S) { - if (Callbacks) - Callbacks->onBackgroundIndexProgress(S); - }, - [this](PathRef P) { return createProcessingContext(P); }); + std::move(BGOpts)); AddIndex(BackgroundIdx.get()); } if (DynamicIdx) diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -124,22 +124,26 @@ // Builds an in-memory index by by running the static indexer action over // all commands in a compilation database. Indexing happens in the background. -// FIXME: it should also persist its state on disk for fast start. // FIXME: it should watch for changes to files on disk. class BackgroundIndex : public SwapIndex { public: - /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be - /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is - /// rebuilt for each indexed file. - BackgroundIndex( - Context BackgroundContext, const ThreadsafeFS &, - const GlobalCompilationDatabase &CDB, - BackgroundIndexStorage::Factory IndexStorageFactory, - // Arbitrary value to ensure some concurrency in tests. - // In production an explicit value is passed. - size_t ThreadPoolSize = 4, - std::function OnProgress = nullptr, - std::function ContextProvider = nullptr); + struct Options { + // Arbitrary value to ensure some concurrency in tests. + // In production an explicit value is specified. + size_t ThreadPoolSize = 4; + // Callback that provides notifications as indexing makes progress. + std::function OnProgress = nullptr; + // Function called to obtain the Context to use while indexing the specified + // file. Called with the empty string for other tasks. + // (When called, the context from BackgroundIndex construction is active). + std::function ContextProvider = nullptr; + }; + + /// Creates a new background index and starts its threads. + /// The current Context will be propagated to each worker thread. + BackgroundIndex(const ThreadsafeFS &, const GlobalCompilationDatabase &CDB, + BackgroundIndexStorage::Factory IndexStorageFactory, + Options Opts); ~BackgroundIndex(); // Blocks while the current task finishes. // Enqueue translation units for indexing. @@ -183,7 +187,6 @@ // configuration const ThreadsafeFS &TFS; const GlobalCompilationDatabase &CDB; - Context BackgroundContext; std::function ContextProvider; llvm::Error index(tooling::CompileCommand); diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -90,28 +90,25 @@ } // namespace BackgroundIndex::BackgroundIndex( - Context BackgroundContext, const ThreadsafeFS &TFS, - const GlobalCompilationDatabase &CDB, - BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize, - std::function OnProgress, - std::function ContextProvider) + const ThreadsafeFS &TFS, const GlobalCompilationDatabase &CDB, + BackgroundIndexStorage::Factory IndexStorageFactory, Options Opts) : SwapIndex(std::make_unique()), TFS(TFS), CDB(CDB), - BackgroundContext(std::move(BackgroundContext)), - ContextProvider(std::move(ContextProvider)), - Rebuilder(this, &IndexedSymbols, ThreadPoolSize), + ContextProvider(std::move(Opts.ContextProvider)), + Rebuilder(this, &IndexedSymbols, Opts.ThreadPoolSize), IndexStorageFactory(std::move(IndexStorageFactory)), - Queue(std::move(OnProgress)), + Queue(std::move(Opts.OnProgress)), CommandsChanged( CDB.watch([&](const std::vector &ChangedFiles) { enqueue(ChangedFiles); })) { - assert(ThreadPoolSize > 0 && "Thread pool size can't be zero."); + assert(Opts.ThreadPoolSize > 0 && "Thread pool size can't be zero."); assert(this->IndexStorageFactory && "Storage factory can not be null!"); - for (unsigned I = 0; I < ThreadPoolSize; ++I) { - ThreadPool.runAsync("background-worker-" + llvm::Twine(I + 1), [this] { - WithContext Ctx(this->BackgroundContext.clone()); - Queue.work([&] { Rebuilder.idle(); }); - }); + for (unsigned I = 0; I < Opts.ThreadPoolSize; ++I) { + ThreadPool.runAsync("background-worker-" + llvm::Twine(I + 1), + [this, Ctx(Context::current().clone())]() mutable { + WithContext BGContext(std::move(Ctx)); + Queue.work([&] { Rebuilder.idle(); }); + }); } } diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp --- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp @@ -96,8 +96,8 @@ size_t CacheHits = 0; MemoryShardStorage MSS(Storage, CacheHits); OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); tooling::CompileCommand Cmd; Cmd.Filename = testPath("root/A.cc"); @@ -126,7 +126,8 @@ } // Context provider that installs a configuration mutating foo's command. // This causes it to define foo::two instead of foo::one. - auto ContextProvider = [](PathRef P) { + BackgroundIndex::Options Opts; + Opts.ContextProvider = [](PathRef P) { Config C; if (P.endswith("foo.cpp")) C.CompileFlags.Edits.push_back( @@ -140,9 +141,9 @@ // We need the CommandMangler, because that applies the config we're testing. OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{}, tooling::ArgumentsAdjuster(CommandMangler::forTests())); + BackgroundIndex Idx( - Context::empty(), FS, CDB, [&](llvm::StringRef) { return &MSS; }, - /*ThreadPoolSize=*/4, /*OnProgress=*/nullptr, std::move(ContextProvider)); + FS, CDB, [&](llvm::StringRef) { return &MSS; }, std::move(Opts)); // Index the two files. for (auto &Cmd : Cmds) CDB.setCompileCommand(testPath(Cmd.Filename), std::move(Cmd)); @@ -181,8 +182,8 @@ size_t CacheHits = 0; MemoryShardStorage MSS(Storage, CacheHits); OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); tooling::CompileCommand Cmd; Cmd.Filename = testPath("root/A.cc"); @@ -248,8 +249,8 @@ // Check nothing is loaded from Storage, but A.cc and A.h has been stored. { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -258,8 +259,8 @@ { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -316,8 +317,8 @@ Cmd.CommandLine = {"clang++", testPath("root/A.cc")}; { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -366,8 +367,8 @@ // Check nothing is loaded from Storage, but A.cc and A.h has been stored. { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -381,8 +382,8 @@ )cpp"; { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -399,8 +400,8 @@ { CacheHits = 0; OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -440,8 +441,8 @@ // Check that A.cc, A.h and B.h has been stored. { OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -456,8 +457,8 @@ { CacheHits = 0; OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -472,8 +473,8 @@ { CacheHits = 0; OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); CDB.setCompileCommand(testPath("root/A.cc"), Cmd); ASSERT_TRUE(Idx.blockUntilIdleForTest()); } @@ -490,8 +491,8 @@ size_t CacheHits = 0; MemoryShardStorage MSS(Storage, CacheHits); OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); ASSERT_TRUE(Idx.blockUntilIdleForTest()); tooling::CompileCommand Cmd; @@ -521,8 +522,8 @@ size_t CacheHits = 0; MemoryShardStorage MSS(Storage, CacheHits); OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); tooling::CompileCommand Cmd; FS.Files[testPath("A.h")] = "void foo();"; @@ -584,8 +585,8 @@ size_t CacheHits = 0; MemoryShardStorage MSS(Storage, CacheHits); OverlayCDB CDB(/*Base=*/nullptr); - BackgroundIndex Idx(Context::empty(), FS, CDB, - [&](llvm::StringRef) { return &MSS; }); + BackgroundIndex Idx(FS, CDB, [&](llvm::StringRef) { return &MSS; }, + /*Opts=*/{}); tooling::CompileCommand Cmd; FS.Files[testPath("A.cc")] = "#include \"A.h\"";