diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -121,6 +121,7 @@ endif() add_subdirectory(tool) add_subdirectory(indexer) +add_subdirectory(background-indexer) add_subdirectory(index/dex/dexp) if (LLVM_INCLUDE_BENCHMARKS) diff --git a/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp b/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/background-indexer/BackgroundIndexer.cpp @@ -0,0 +1,70 @@ +//===--- BackgroundIndexer.cpp -----------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A standalone binary to run clangd's background indexer. +// +//===----------------------------------------------------------------------===// + +#include "Context.h" +#include "FSProvider.h" +#include "GlobalCompilationDatabase.h" +#include "Logger.h" +#include "index/Background.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" +#include + +static llvm::cl::opt CompileCommandsDir( + llvm::cl::Positional, + llvm::cl::desc("Directory containing compilation database"), + llvm::cl::init(".")); + +int main(int Argc, const char **Argv) { + llvm::sys::PrintStackTraceOnErrorSignal(Argv[0]); + llvm::cl::ParseCommandLineOptions( + Argc, Argv, "A standalone binary to run clangd's background indexer."); + clang::clangd::RealFileSystemProvider FSProvider; + + llvm::errs().SetBuffered(); + clang::clangd::StreamLogger Logger(llvm::errs(), clang::clangd::Logger::Info); + clang::clangd::LoggingSession LoggingSession(Logger); + + clang::clangd::DirectoryBasedGlobalCompilationDatabase DirectoryBasedCDB( + /*CompileCommandsDir=*/llvm::None); + clang::clangd::OverlayCDB CDB(&DirectoryBasedCDB); + // FIXME: Maybe add a way to not lower thread priority? + clang::clangd::BackgroundIndex BackgroundIdx( + clang::clangd::Context::empty(), FSProvider, CDB, + clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory(), + // We don't want to build dex/mem index until every file has been + // indexed/loaded. + // FIXME: Change BackgroundIndex to only built after all files have been + // read if build period is zero, which should only be used by + // non-interactive tools like this one. + 24 * 60 * 60 * 1000); + + // To trigger background indexing we need to discover a compilation database, + // which can be done by simply querying compile commands for a cpp file under + // the directory containing compilation_database.jon. + llvm::SmallString<128> DummyFile(CompileCommandsDir); + llvm::sys::fs::make_absolute(DummyFile); + llvm::sys::path::remove_dots(DummyFile, true); + llvm::sys::path::append(DummyFile, "dummy.cpp"); + CDB.getCompileCommand(DummyFile); + + // Wait untill indexing finishes. + BackgroundIdx.blockUntilIdleForTest(llvm::None); + return 0; +} diff --git a/clang-tools-extra/clangd/background-indexer/CMakeLists.txt b/clang-tools-extra/clangd/background-indexer/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/background-indexer/CMakeLists.txt @@ -0,0 +1,20 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../) + +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_executable(clangd-background-indexer + BackgroundIndexer.cpp +) + +target_link_libraries(clangd-background-indexer + PRIVATE + clangAST + clangIndex + clangDaemon + clangBasic + clangFrontend + clangLex + clangTooling +) diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -144,6 +144,10 @@ std::deque> Queue; std::vector ThreadPool; // FIXME: Abstract this away. GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged; + + // For logging + std::atomic EnqueuedTUs{0}; + std::atomic IndexedTUs{0}; }; } // namespace clangd diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -215,6 +216,7 @@ // We're doing this asynchronously, because we'll read shards here too. log("Enqueueing {0} commands for indexing", ChangedFiles.size()); SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size())); + EnqueuedTUs += ChangedFiles.size(); auto NeedsReIndexing = loadShards(std::move(ChangedFiles)); // Run indexing for files that need to be updated. @@ -447,8 +449,9 @@ assert(Index.Symbols && Index.Refs && Index.Sources && "Symbols, Refs and Sources must be set."); - log("Indexed {0} ({1} symbols, {2} refs, {3} files)", - Inputs.CompileCommand.Filename, Index.Symbols->size(), + ++IndexedTUs; + log("[{0}/{1}] Indexed {2} ({3} symbols, {4} refs, {5} files)", IndexedTUs, + EnqueuedTUs, Inputs.CompileCommand.Filename, Index.Symbols->size(), Index.Refs->numRefs(), Index.Sources->size()); SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size())); SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs())); @@ -581,12 +584,14 @@ // Keeps track of the loaded shards to make sure we don't perform redundant // disk IO. Keys are absolute paths. llvm::StringSet<> LoadedShards; + size_t UpToDateTUs = 0; for (const auto &File : ChangedFiles) { ProjectInfo PI; auto Cmd = CDB.getCompileCommand(File, &PI); if (!Cmd) continue; BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot); + ++UpToDateTUs; auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards); for (const auto &Dependency : Dependencies) { if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path)) @@ -594,17 +599,19 @@ // FIXME: Currently, we simply schedule indexing on a TU whenever any of // its dependencies needs re-indexing. We might do it smarter by figuring // out a minimal set of TUs that will cover all the stale dependencies. - vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.", + dlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.", Cmd->Filename, Dependency.Path); NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage}); // Mark all of this TU's dependencies as to-be-indexed so that we won't // try to re-index those. for (const auto &Dependency : Dependencies) FilesToIndex.insert(Dependency.Path); + --UpToDateTUs; break; } } - vlog("Loaded all shards"); + IndexedTUs += UpToDateTUs; + log("[{0}/{1}] Loaded shards from storage", IndexedTUs, EnqueuedTUs); reset(IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge)); vlog("BackgroundIndex: built symbol index with estimated memory {0} " "bytes.",