diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h --- a/clang-tools-extra/clangd/TUScheduler.h +++ b/clang-tools-extra/clangd/TUScheduler.h @@ -13,10 +13,12 @@ #include "Diagnostics.h" #include "GlobalCompilationDatabase.h" #include "index/CanonicalIncludes.h" +#include "index/SymbolID.h" #include "support/Function.h" #include "support/MemoryTree.h" #include "support/Path.h" #include "support/Threading.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" @@ -33,6 +35,14 @@ /// synchronously). unsigned getDefaultAsyncThreadsCount(); +/// Signals derived from a valid AST of a file. +struct ASTSignals { + /// Number of occurrences of each Symbol present in the file. + llvm::DenseMap Symbols; + /// Number of Symbols belonging to each namespace present in the file. + llvm::StringMap RelatedNamespaces; +}; + struct InputsAndAST { const ParseInputs &Inputs; ParsedAST * @@ -43,6 +53,8 @@ const tooling::CompileCommand &Command; // This can be nullptr if no preamble is available. const PreambleData *Preamble; + // This can be nullptr if no ASTSignals are available. + const ASTSignals *Signals; }; /// Determines whether diagnostics should be generated for a file snapshot. diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -47,8 +47,10 @@ // requests will receive latest build preamble, which might possibly be stale. #include "TUScheduler.h" +#include "AST.h" #include "Compiler.h" #include "Diagnostics.h" +#include "FindTarget.h" #include "GlobalCompilationDatabase.h" #include "ParsedAST.h" #include "Preamble.h" @@ -60,6 +62,7 @@ #include "support/Path.h" #include "support/Threading.h" #include "support/Trace.h" +#include "clang/AST/Decl.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Tooling/CompilationDatabase.h" #include "llvm/ADT/FunctionExtras.h" @@ -415,6 +418,10 @@ /// getPossiblyStalePreamble() can be null even after this function returns. void waitForFirstPreamble() const; + std::shared_ptr getASTSignals() const; + + void updateASTSignals(ParsedAST &AST); + TUScheduler::FileStats stats() const; bool isASTCached() const; @@ -499,6 +506,7 @@ /// Signalled whenever a new request has been scheduled or processing of a /// request has completed. mutable std::condition_variable RequestsCV; + std::shared_ptr LatestASTSignals; /* GUARDED_BY(Mutex) */ /// Latest build preamble for current TU. /// None means no builds yet, null means there was an error while building. /// Only written by ASTWorker's thread. @@ -830,6 +838,35 @@ RequestsCV.notify_all(); } +void ASTWorker::updateASTSignals(ParsedAST &AST) { + ASTSignals Signals; + const SourceManager &SM = AST.getSourceManager(); + findExplicitReferences( + AST.getASTContext(), [&Signals, &SM](ReferenceLoc Ref) { + for (const NamedDecl *ND : Ref.Targets) { + if (!isInsideMainFile(Ref.NameLoc, SM)) + continue; + if (auto ID = getSymbolID(ND)) + Signals.Symbols[ID] += 1; + // FIXME: Do not count the primary namespace as a related namespace. + // Eg. clang::clangd:: for this file. + if (const auto *NSD = dyn_cast(ND->getDeclContext())) + if (!ND->isInAnonymousNamespace()) { + std::string NS; + llvm::raw_string_ostream OS(NS); + NSD->printQualifiedName(OS); + if (!OS.str().empty()) { + Signals.RelatedNamespaces[OS.str() + "::"] += 1; + } + } + } + }); + // Existing readers of ASTSignals will have their copy preserved until the + // read is completed. The last reader deletes the old ASTSignals. + std::lock_guard Lock(Mutex); + LatestASTSignals = std::make_shared(std::move(Signals)); +} + void ASTWorker::generateDiagnostics( std::unique_ptr Invocation, ParseInputs Inputs, std::vector CIDiags) { @@ -908,6 +945,7 @@ if (*AST) { trace::Span Span("Running main AST callback"); Callbacks.onMainAST(FileName, **AST, RunPublish); + updateASTSignals(**AST); } else { // Failed to build the AST, at least report diagnostics from the // command line if there were any. @@ -931,6 +969,11 @@ return LatestPreamble ? *LatestPreamble : nullptr; } +std::shared_ptr ASTWorker::getASTSignals() const { + std::lock_guard Lock(Mutex); + return LatestASTSignals; +} + void ASTWorker::waitForFirstPreamble() const { std::unique_lock Lock(Mutex); PreambleCV.wait(Lock, [this] { return LatestPreamble.hasValue() || Done; }); @@ -1366,36 +1409,39 @@ SPAN_ATTACH(Tracer, "file", File); std::shared_ptr Preamble = It->second->Worker->getPossiblyStalePreamble(); + std::shared_ptr Signals = + It->second->Worker->getASTSignals(); WithContext WithProvidedContext(Opts.ContextProvider(File)); Action(InputsAndPreamble{It->second->Contents, It->second->Worker->getCurrentCompileCommand(), - Preamble.get()}); + Preamble.get(), Signals.get()}); return; } std::shared_ptr Worker = It->second->Worker.lock(); - auto Task = - [Worker, Consistency, Name = Name.str(), File = File.str(), - Contents = It->second->Contents, - Command = Worker->getCurrentCompileCommand(), - Ctx = Context::current().derive(kFileBeingProcessed, std::string(File)), - Action = std::move(Action), this]() mutable { - std::shared_ptr Preamble; - if (Consistency == PreambleConsistency::Stale) { - // Wait until the preamble is built for the first time, if preamble - // is required. This avoids extra work of processing the preamble - // headers in parallel multiple times. - Worker->waitForFirstPreamble(); - } - Preamble = Worker->getPossiblyStalePreamble(); - - std::lock_guard BarrierLock(Barrier); - WithContext Guard(std::move(Ctx)); - trace::Span Tracer(Name); - SPAN_ATTACH(Tracer, "file", File); - WithContext WithProvidedContext(Opts.ContextProvider(File)); - Action(InputsAndPreamble{Contents, Command, Preamble.get()}); - }; + auto Task = [Worker, Consistency, Name = Name.str(), File = File.str(), + Contents = It->second->Contents, + Command = Worker->getCurrentCompileCommand(), + Ctx = Context::current().derive(kFileBeingProcessed, + std::string(File)), + Action = std::move(Action), this]() mutable { + std::shared_ptr Preamble; + if (Consistency == PreambleConsistency::Stale) { + // Wait until the preamble is built for the first time, if preamble + // is required. This avoids extra work of processing the preamble + // headers in parallel multiple times. + Worker->waitForFirstPreamble(); + } + Preamble = Worker->getPossiblyStalePreamble(); + std::shared_ptr Signals = Worker->getASTSignals(); + + std::lock_guard BarrierLock(Barrier); + WithContext Guard(std::move(Ctx)); + trace::Span Tracer(Name); + SPAN_ATTACH(Tracer, "file", File); + WithContext WithProvidedContext(Opts.ContextProvider(File)); + Action(InputsAndPreamble{Contents, Command, Preamble.get(), Signals.get()}); + }; PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File), std::move(Task)); diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -14,6 +14,7 @@ #include "Preamble.h" #include "TUScheduler.h" #include "TestFS.h" +#include "TestIndex.h" #include "support/Cancellation.h" #include "support/Context.h" #include "support/Path.h" @@ -42,12 +43,14 @@ namespace clangd { namespace { +using ::testing::_; using ::testing::AnyOf; using ::testing::Each; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::Field; using ::testing::IsEmpty; +using ::testing::Pair; using ::testing::Pointee; using ::testing::SizeIs; using ::testing::UnorderedElementsAre; @@ -679,12 +682,12 @@ cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size, 0u); }); - // Wait for the preamble is being built. + // Wait while the preamble is being built. ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10))); // Update the file which results in an empty preamble. S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto); - // Wait for the preamble is being built. + // Wait while the preamble is being built. ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10))); S.runWithPreamble( "getEmptyPreamble", Foo, TUScheduler::Stale, @@ -696,6 +699,73 @@ }); } +TEST_F(TUSchedulerTests, ASTSignals) { + TUScheduler S(CDB, optsForTest()); + auto Foo = testPath("foo.cpp"); + auto Header = testPath("foo.h"); + + FS.Files[Header] = R"cpp( + #define ADD(x, y, z) (x + y + z) + namespace tar { // A related namespace. + int kConst = 5; + int foo(); + void bar(); // Unused symbols are not recorded. + class X { + public: int Y; + }; + } // namespace tar + namespace ns1::ns2 { int fooInNS2(); }} + )cpp"; + const char *Contents = R"cpp( + #include "foo.h" + namespace ns1 { + namespace ns2 { + namespace { + int func() { + tar::X a; + a.Y = 1; + return ADD(tar::kConst, a.Y, tar::foo()) + fooInNS2(); + } + } // namespace + } // namespace ns2 + } // namespace ns1 + )cpp"; + // Update the file which results in an empty preamble. + S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes); + // Wait while the preamble is being built. + ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10))); + Notification TaskRun; + S.runWithPreamble( + "ASTSignals", Foo, TUScheduler::Stale, + [&](Expected IP) { + ASSERT_FALSE(!IP); + std::vector> NS; + for (const auto &P : IP->Signals->RelatedNamespaces) + NS.emplace_back(P.getKey(), P.getValue()); + EXPECT_THAT(NS, UnorderedElementsAre(Pair("ns1::", 1), + Pair("ns1::ns2::", 1), + Pair("tar::", 3))); + + std::vector> Sym; + for (const auto &P : IP->Signals->Symbols) + Sym.emplace_back(P.getFirst(), P.getSecond()); + EXPECT_THAT( + Sym, + UnorderedElementsAre( + Pair(ns("tar").ID, 3), Pair(ns("ns1").ID, 1), + Pair(ns("ns1::ns2").ID, 1), Pair(_ /*int func();*/, 1), + Pair(cls("tar::X").ID, 1), Pair(var("tar::kConst").ID, 1), + Pair(func("tar::foo").ID, 1), + Pair(func("ns1::ns2::fooInNS2").ID, 1), + Pair(sym("Y", index::SymbolKind::Variable, "@N@tar@S@X@FI@\\0") + .ID, + 2), + Pair(_ /*a*/, 3))); + TaskRun.notify(); + }); + TaskRun.wait(); +} + TEST_F(TUSchedulerTests, RunWaitsForPreamble) { // Testing strategy: we update the file and schedule a few preamble reads at // the same time. All reads should get the same non-null preamble.