Index: clang-tools-extra/trunk/clangd/ClangdServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.h +++ clang-tools-extra/trunk/clangd/ClangdServer.h @@ -223,6 +223,8 @@ SymbolIndex *Index; // If present, an up-to-date of symbols in open files. Read via Index. std::unique_ptr FileIdx; + /// Callbacks responsible for updating FileIdx. + std::unique_ptr FileIdxUpdater; // If present, a merged view of FileIdx and an external index. Read via Index. std::unique_ptr MergedIndex; // If set, this represents the workspace path. Index: clang-tools-extra/trunk/clangd/ClangdServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdServer.cpp @@ -66,6 +66,24 @@ Optional> Result; }; +class UpdateFileIndex : public ParsingCallbacks { +public: + UpdateFileIndex(FileIndex *FileIdx) : FileIdx(FileIdx) {} + + void onPreambleAST(PathRef Path, ASTContext &Ctx, + std::shared_ptr PP) override { + if (FileIdx) + FileIdx->update(Path, &Ctx, std::move(PP)); + } + + void onMainAST(PathRef Path, ParsedAST &AST) override { + // FIXME: merge results from the main file into the index too. + } + +private: + FileIndex *FileIdx; +}; + } // namespace ClangdServer::Options ClangdServer::optsForTest() { @@ -85,20 +103,16 @@ : getStandardResourceDir()), FileIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex(Opts.URISchemes) : nullptr), + FileIdxUpdater(llvm::make_unique(FileIdx.get())), PCHs(std::make_shared()), // Pass a callback into `WorkScheduler` to extract symbols from a newly // parsed file and rebuild the file index synchronously each time an AST // is parsed. // FIXME(ioeric): this can be slow and we may be able to index on less // critical paths. - WorkScheduler( - Opts.AsyncThreadsCount, Opts.StorePreamblesInMemory, - FileIdx - ? [this](PathRef Path, ASTContext &AST, - std::shared_ptr - PP) { FileIdx->update(Path, &AST, std::move(PP)); } - : PreambleParsedCallback(), - Opts.UpdateDebounce, Opts.RetentionPolicy) { + WorkScheduler(Opts.AsyncThreadsCount, Opts.StorePreamblesInMemory, + *FileIdxUpdater, Opts.UpdateDebounce, + Opts.RetentionPolicy) { if (FileIdx && Opts.StaticIndex) { MergedIndex = mergeIndex(FileIdx.get(), Opts.StaticIndex); Index = MergedIndex.get(); Index: clang-tools-extra/trunk/clangd/TUScheduler.h =================================================================== --- clang-tools-extra/trunk/clangd/TUScheduler.h +++ clang-tools-extra/trunk/clangd/TUScheduler.h @@ -51,6 +51,30 @@ unsigned MaxRetainedASTs = 3; }; +class ParsingCallbacks { +public: + virtual ~ParsingCallbacks() = default; + + /// Called on the AST that was built for emitting the preamble. The built AST + /// contains only AST nodes from the #include directives at the start of the + /// file. AST node in the current file should be observed on onMainAST call. + virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, + std::shared_ptr PP) {} + /// Called on the AST built for the file itself. Note that preamble AST nodes + /// are not deserialized and should be processed in the onPreambleAST call + /// instead. + /// The \p AST always contains all AST nodes for the main file itself, and + /// only a portion of the AST nodes deserialized from the preamble. Note that + /// some nodes from the preamble may have been deserialized and may also be + /// accessed from the main file AST, e.g. redecls of functions from preamble, + /// etc. Clients are expected to process only the AST nodes from the main file + /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain + /// optimal performance. + virtual void onMainAST(PathRef Path, ParsedAST &AST) {} +}; + +ParsingCallbacks &noopParsingCallbacks(); + /// Handles running tasks for ClangdServer and managing the resources (e.g., /// preambles and ASTs) for opened files. /// TUScheduler is not thread-safe, only one thread should be providing updates @@ -61,7 +85,7 @@ class TUScheduler { public: TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback, + ParsingCallbacks &ASTCallbacks, std::chrono::steady_clock::duration UpdateDebounce, ASTRetentionPolicy RetentionPolicy); ~TUScheduler(); @@ -132,7 +156,7 @@ private: const bool StorePreamblesInMemory; const std::shared_ptr PCHOps; - const PreambleParsedCallback PreambleCallback; + ParsingCallbacks &Callbacks; Semaphore Barrier; llvm::StringMap> Files; std::unique_ptr IdleASTs; Index: clang-tools-extra/trunk/clangd/TUScheduler.cpp =================================================================== --- clang-tools-extra/trunk/clangd/TUScheduler.cpp +++ clang-tools-extra/trunk/clangd/TUScheduler.cpp @@ -158,8 +158,7 @@ Semaphore &Barrier, bool RunSync, steady_clock::duration UpdateDebounce, std::shared_ptr PCHs, - bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback); + bool StorePreamblesInMemory, ParsingCallbacks &Callbacks); public: /// Create a new ASTWorker and return a handle to it. @@ -173,7 +172,7 @@ steady_clock::duration UpdateDebounce, std::shared_ptr PCHs, bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback); + ParsingCallbacks &Callbacks); ~ASTWorker(); void update(ParseInputs Inputs, WantDiagnostics, @@ -228,8 +227,8 @@ const Path FileName; /// Whether to keep the built preambles in memory or on disk. const bool StorePreambleInMemory; - /// Callback, passed to the preamble builder. - const PreambleParsedCallback PreambleCallback; + /// Callback, invoked when preamble or main file AST is built. + ParsingCallbacks &Callbacks; /// Helper class required to build the ASTs. const std::shared_ptr PCHs; @@ -299,10 +298,10 @@ steady_clock::duration UpdateDebounce, std::shared_ptr PCHs, bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback) { + ParsingCallbacks &Callbacks) { std::shared_ptr Worker(new ASTWorker( FileName, IdleASTs, Barrier, /*RunSync=*/!Tasks, UpdateDebounce, - std::move(PCHs), StorePreamblesInMemory, std::move(PreambleCallback))); + std::move(PCHs), StorePreamblesInMemory, Callbacks)); if (Tasks) Tasks->runAsync("worker:" + llvm::sys::path::filename(FileName), [Worker]() { Worker->run(); }); @@ -314,12 +313,11 @@ Semaphore &Barrier, bool RunSync, steady_clock::duration UpdateDebounce, std::shared_ptr PCHs, - bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback) + bool StorePreamblesInMemory, ParsingCallbacks &Callbacks) : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce), FileName(FileName), StorePreambleInMemory(StorePreamblesInMemory), - PreambleCallback(std::move(PreambleCallback)), PCHs(std::move(PCHs)), - Barrier(Barrier), Done(false) {} + Callbacks(Callbacks), PCHs(std::move(PCHs)), Barrier(Barrier), + Done(false) {} ASTWorker::~ASTWorker() { // Make sure we remove the cached AST, if any. @@ -365,7 +363,11 @@ getPossiblyStalePreamble(); std::shared_ptr NewPreamble = buildPreamble(FileName, *Invocation, OldPreamble, OldCommand, Inputs, - PCHs, StorePreambleInMemory, PreambleCallback); + PCHs, StorePreambleInMemory, + [this](PathRef Path, ASTContext &Ctx, + std::shared_ptr PP) { + Callbacks.onPreambleAST(FileName, Ctx, std::move(PP)); + }); bool CanReuseAST = InputsAreTheSame && (OldPreamble == NewPreamble); { @@ -415,6 +417,7 @@ // Note *AST can be still be null if buildAST fails. if (*AST) { OnUpdated((*AST)->getDiagnostics()); + Callbacks.onMainAST(FileName, **AST); DiagsWereReported = true; } // Stash the AST in the cache for further use. @@ -618,6 +621,11 @@ return HardwareConcurrency; } +ParsingCallbacks &noopParsingCallbacks() { + static ParsingCallbacks *Instance = new ParsingCallbacks; + return *Instance; +} + struct TUScheduler::FileData { /// Latest inputs, passed to TUScheduler::update(). std::string Contents; @@ -627,12 +635,12 @@ TUScheduler::TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory, - PreambleParsedCallback PreambleCallback, + ParsingCallbacks &Callbacks, std::chrono::steady_clock::duration UpdateDebounce, ASTRetentionPolicy RetentionPolicy) : StorePreamblesInMemory(StorePreamblesInMemory), - PCHOps(std::make_shared()), - PreambleCallback(std::move(PreambleCallback)), Barrier(AsyncThreadsCount), + PCHOps(std::make_shared()), Callbacks(Callbacks), + Barrier(AsyncThreadsCount), IdleASTs(llvm::make_unique(RetentionPolicy.MaxRetainedASTs)), UpdateDebounce(UpdateDebounce) { if (0 < AsyncThreadsCount) { @@ -670,8 +678,7 @@ // Create a new worker to process the AST-related tasks. ASTWorkerHandle Worker = ASTWorker::create( File, *IdleASTs, WorkerThreads ? WorkerThreads.getPointer() : nullptr, - Barrier, UpdateDebounce, PCHOps, StorePreamblesInMemory, - PreambleCallback); + Barrier, UpdateDebounce, PCHOps, StorePreamblesInMemory, Callbacks); FD = std::unique_ptr(new FileData{ Inputs.Contents, Inputs.CompileCommand, std::move(Worker)}); } else { Index: clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp =================================================================== --- clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp +++ clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp @@ -17,10 +17,11 @@ namespace clang { namespace clangd { +namespace { using ::testing::_; -using ::testing::Each; using ::testing::AnyOf; +using ::testing::Each; using ::testing::Pair; using ::testing::Pointee; using ::testing::UnorderedElementsAre; @@ -44,8 +45,7 @@ TEST_F(TUSchedulerTests, MissingFiles) { TUScheduler S(getDefaultAsyncThreadsCount(), - /*StorePreamblesInMemory=*/true, - /*PreambleParsedCallback=*/nullptr, + /*StorePreamblesInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); @@ -101,8 +101,7 @@ Notification Ready; TUScheduler S( getDefaultAsyncThreadsCount(), - /*StorePreamblesInMemory=*/true, - /*PreambleParsedCallback=*/nullptr, + /*StorePreamblesInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); auto Path = testPath("foo.cpp"); @@ -130,8 +129,7 @@ std::atomic CallbackCount(0); { TUScheduler S(getDefaultAsyncThreadsCount(), - /*StorePreamblesInMemory=*/true, - /*PreambleParsedCallback=*/nullptr, + /*StorePreamblesInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::seconds(1), ASTRetentionPolicy()); // FIXME: we could probably use timeouts lower than 1 second here. @@ -162,8 +160,7 @@ // Run TUScheduler and collect some stats. { TUScheduler S(getDefaultAsyncThreadsCount(), - /*StorePreamblesInMemory=*/true, - /*PreambleParsedCallback=*/nullptr, + /*StorePreamblesInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::milliseconds(50), ASTRetentionPolicy()); @@ -261,7 +258,7 @@ Policy.MaxRetainedASTs = 2; TUScheduler S( /*AsyncThreadsCount=*/1, /*StorePreambleInMemory=*/true, - PreambleParsedCallback(), + noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), Policy); llvm::StringLiteral SourceContents = R"cpp( @@ -311,7 +308,7 @@ TEST_F(TUSchedulerTests, EmptyPreamble) { TUScheduler S( /*AsyncThreadsCount=*/4, /*StorePreambleInMemory=*/true, - PreambleParsedCallback(), + noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); @@ -358,7 +355,7 @@ // the same time. All reads should get the same non-null preamble. TUScheduler S( /*AsyncThreadsCount=*/4, /*StorePreambleInMemory=*/true, - PreambleParsedCallback(), + noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); auto Foo = testPath("foo.cpp"); @@ -391,7 +388,7 @@ TEST_F(TUSchedulerTests, NoopOnEmptyChanges) { TUScheduler S( /*AsyncThreadsCount=*/getDefaultAsyncThreadsCount(), - /*StorePreambleInMemory=*/true, PreambleParsedCallback(), + /*StorePreambleInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); @@ -444,7 +441,7 @@ TEST_F(TUSchedulerTests, NoChangeDiags) { TUScheduler S( /*AsyncThreadsCount=*/getDefaultAsyncThreadsCount(), - /*StorePreambleInMemory=*/true, PreambleParsedCallback(), + /*StorePreambleInMemory=*/true, noopParsingCallbacks(), /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), ASTRetentionPolicy()); @@ -475,5 +472,6 @@ ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1))); } +} // namespace } // namespace clangd } // namespace clang