diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -121,6 +121,7 @@ endif() add_subdirectory(tool) add_subdirectory(indexer) +add_subdirectory(background-indexer) add_subdirectory(index/dex/dexp) if (LLVM_INCLUDE_BENCHMARKS) diff --git a/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp b/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp @@ -0,0 +1,64 @@ +//===--- BackgroundIndexer.cpp -----------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A standalone binary to run clangd's background indexer. +// +//===----------------------------------------------------------------------===// + +#include "/usr/local/google/home/kadircet/repos/llvm/clang-tools-extra/clangd/Context.h" +#include "/usr/local/google/home/kadircet/repos/llvm/clang-tools-extra/clangd/Logger.h" +#include "FSProvider.h" +#include "GlobalCompilationDatabase.h" +#include "index/Background.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" +#include + +static llvm::cl::opt CompileCommandsDir( + llvm::cl::Positional, + llvm::cl::desc("Directory containing compilation database"), + llvm::cl::init(".")); + +int main(int Argc, const char **Argv) { + llvm::sys::PrintStackTraceOnErrorSignal(Argv[0]); + llvm::cl::ParseCommandLineOptions( + Argc, Argv, "A standalone binary to run clangd's background indexer."); + clang::clangd::RealFileSystemProvider FSProvider; + + llvm::errs().SetBuffered(); + clang::clangd::StreamLogger Logger(llvm::errs(), clang::clangd::Logger::Info); + clang::clangd::LoggingSession LoggingSession(Logger); + + clang::clangd::DirectoryBasedGlobalCompilationDatabase DirectoryBasedCDB( + /*CompileCommandsDir=*/llvm::None); + clang::clangd::OverlayCDB CDB(&DirectoryBasedCDB); + // FIXME: Maybe add a way to not lower thread priority? + clang::clangd::BackgroundIndex BackgroundIdx( + clang::clangd::Context::empty(), FSProvider, CDB, + clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory(), + // We don't want to build dex/mem index until every file has been + // indexed/loaded. + // FIXME: Change BackgroundIndex to only built after all files have been + // read if build period is zero, which should only be used by + // non-interactive tools like this one. + 24 * 60 * 60 * 1000); + llvm::SmallString<128> DummyFile(CompileCommandsDir); + llvm::sys::fs::make_absolute(DummyFile); + llvm::sys::path::remove_dots(DummyFile, true); + llvm::sys::path::append(DummyFile, "dummy.cpp"); + CDB.getCompileCommand(DummyFile); + BackgroundIdx.blockUntilIdleForTest(llvm::None); + return 0; +} diff --git a/clang-tools-extra/clangd/background-indexer/CMakeLists.txt b/clang-tools-extra/clangd/background-indexer/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/background-indexer/CMakeLists.txt @@ -0,0 +1,20 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../) + +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_executable(clangd-background-indexer + BackgroundIndexer.cpp +) + +target_link_libraries(clangd-background-indexer + PRIVATE + clangAST + clangIndex + clangDaemon + clangBasic + clangFrontend + clangLex + clangTooling +) diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -144,6 +144,11 @@ std::deque> Queue; std::vector ThreadPool; // FIXME: Abstract this away. GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged; + + // For logging + size_t EnqueuedTUs = 0; + size_t IndexedTUs = 0; + std::mutex TUCounterMutex; }; } // namespace clangd diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -215,6 +216,10 @@ // We're doing this asynchronously, because we'll read shards here too. log("Enqueueing {0} commands for indexing", ChangedFiles.size()); SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size())); + { + std::lock_guard Lock(TUCounterMutex); + EnqueuedTUs += ChangedFiles.size(); + } auto NeedsReIndexing = loadShards(std::move(ChangedFiles)); // Run indexing for files that need to be updated. @@ -447,9 +452,13 @@ assert(Index.Symbols && Index.Refs && Index.Sources && "Symbols, Refs and Sources must be set."); - log("Indexed {0} ({1} symbols, {2} refs, {3} files)", - Inputs.CompileCommand.Filename, Index.Symbols->size(), - Index.Refs->numRefs(), Index.Sources->size()); + { + std::lock_guard Lock(TUCounterMutex); + ++IndexedTUs; + log("[{0}/{1}] Indexed {2} ({3} symbols, {4} refs, {5} files)", IndexedTUs, + EnqueuedTUs, Inputs.CompileCommand.Filename, Index.Symbols->size(), + Index.Refs->numRefs(), Index.Sources->size()); + } SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size())); SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs())); SPAN_ATTACH(Tracer, "sources", int(Index.Sources->size())); @@ -581,20 +590,23 @@ // Keeps track of the loaded shards to make sure we don't perform redundant // disk IO. Keys are absolute paths. llvm::StringSet<> LoadedShards; + size_t UpToDateTUs = 0; for (const auto &File : ChangedFiles) { ProjectInfo PI; auto Cmd = CDB.getCompileCommand(File, &PI); if (!Cmd) continue; BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot); + ++UpToDateTUs; auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards); for (const auto &Dependency : Dependencies) { if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path)) continue; + --UpToDateTUs; // FIXME: Currently, we simply schedule indexing on a TU whenever any of // its dependencies needs re-indexing. We might do it smarter by figuring // out a minimal set of TUs that will cover all the stale dependencies. - vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.", + dlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.", Cmd->Filename, Dependency.Path); NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage}); // Mark all of this TU's dependencies as to-be-indexed so that we won't @@ -604,7 +616,11 @@ break; } } - vlog("Loaded all shards"); + { + std::lock_guard Lock(TUCounterMutex); + IndexedTUs += UpToDateTUs; + log("[{0}/{1}] Loaded shards from storage", IndexedTUs, EnqueuedTUs); + } reset(IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge)); vlog("BackgroundIndex: built symbol index with estimated memory {0} " "bytes.",