Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -322,6 +322,15 @@ /// Called when an event occurs for a watched file in the workspace. void onFileEvent(const DidChangeWatchedFilesParams &Params); + /// Returns estimated memory usage for each of the currently open files. + /// The order of results is unspecified. + /// Overall memory usage of clangd may be significantly more than reported + /// here, as this metric does not account (at least) for: + /// - memory occupied by static and dynamic index, + /// - memory required for in-flight requests, + /// FIXME: those metrics might be useful too, we should add them. + std::vector> getUsedBytesPerFile() const; + private: /// FIXME: This stats several files to find a .clang-format file. I/O can be /// slow. Think of a way to cache this. Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -635,3 +635,8 @@ // FIXME: Do nothing for now. This will be used for indexing and potentially // invalidating other caches. } + +std::vector> +ClangdServer::getUsedBytesPerFile() const { + return Units.getUsedBytesPerFile(); +} Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -96,6 +96,10 @@ const std::vector &getDiagnostics() const; + /// Returns the esitmated size of the AST and the accessory structures, in + /// bytes. Does not include the size of the preamble. + std::size_t getUsedBytes() const; + private: ParsedAST(std::shared_ptr Preamble, std::unique_ptr Clang, @@ -216,6 +220,10 @@ /// always be non-null. std::shared_future> getAST() const; + /// Returns an estimated size, in bytes, currently occupied by the AST and the + /// Preamble. + std::size_t getUsedBytes() const; + private: /// A helper guard that manages the state of CppFile during rebuild. class RebuildGuard { @@ -244,6 +252,11 @@ /// Condition variable to indicate changes to RebuildInProgress. std::condition_variable RebuildCond; + /// Size of the last built AST, in bytes. + std::size_t ASTMemUsage; + /// Size of the last build Preamble, in bytes. + std::size_t PreambleMemUsage; + /// Promise and future for the latests AST. Fulfilled during rebuild. /// We use std::shared_ptr here because MVSC fails to compile non-copyable /// classes as template arguments of promise/future. Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -35,6 +35,10 @@ namespace { +template std::size_t getUsedBytes(const std::vector &Vec) { + return Vec.capacity() * sizeof(T); +} + class DeclTrackingASTConsumer : public ASTConsumer { public: DeclTrackingASTConsumer(std::vector &TopLevelDecls) @@ -332,6 +336,14 @@ return Diags; } +std::size_t ParsedAST::getUsedBytes() const { + auto &AST = getASTContext(); + // FIXME(ibiryukov): we do not account for the dynamically allocated part of + // SmallVector inside each Diag. + return AST.getASTAllocatedMemory() + AST.getSideTableAllocatedMemory() + + ::getUsedBytes(TopLevelDecls) + ::getUsedBytes(Diags); +} + PreambleData::PreambleData(PrecompiledPreamble Preamble, std::vector TopLevelDeclIDs, std::vector Diags) @@ -370,7 +382,8 @@ std::shared_ptr PCHs, ASTParsedCallback ASTCallback) : FileName(FileName), StorePreamblesInMemory(StorePreamblesInMemory), - RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)), + RebuildCounter(0), RebuildInProgress(false), ASTMemUsage(0), + PreambleMemUsage(0), PCHs(std::move(PCHs)), ASTCallback(std::move(ASTCallback)) { // FIXME(ibiryukov): we should pass a proper Context here. log(Context::empty(), "Created CppFile for " + FileName); @@ -419,7 +432,9 @@ return; // Set empty results for Promises. + That->PreambleMemUsage = 0; That->PreamblePromise.set_value(nullptr); + That->ASTMemUsage = 0; That->ASTPromise.set_value(std::make_shared(llvm::None)); }; } @@ -573,6 +588,8 @@ That->LatestAvailablePreamble = NewPreamble; if (RequestRebuildCounter != That->RebuildCounter) return llvm::None; // Our rebuild request was cancelled, do nothing. + That->PreambleMemUsage = + NewPreamble ? NewPreamble->Preamble.getSize() : 0; That->PreamblePromise.set_value(NewPreamble); } // unlock Mutex @@ -609,6 +626,7 @@ return Diagnostics; // Our rebuild request was cancelled, don't set // ASTPromise. + That->ASTMemUsage = NewAST ? NewAST->getUsedBytes() : 0; That->ASTPromise.set_value( std::make_shared(std::move(NewAST))); } // unlock Mutex @@ -635,6 +653,14 @@ return ASTFuture; } +std::size_t CppFile::getUsedBytes() const { + std::lock_guard Lock(Mutex); + // FIXME: We should not store extra size fields. When we store AST and + // Preamble directly, not inside futures, we could compute the sizes from the + // stored AST and the preamble in this function directly. + return ASTMemUsage + PreambleMemUsage; +} + CppFile::RebuildGuard::RebuildGuard(CppFile &File, unsigned RequestRebuildCounter) : File(File), RequestRebuildCounter(RequestRebuildCounter) { Index: clangd/ClangdUnitStore.h =================================================================== --- clangd/ClangdUnitStore.h +++ clangd/ClangdUnitStore.h @@ -44,7 +44,7 @@ return It->second; } - std::shared_ptr getFile(PathRef File) { + std::shared_ptr getFile(PathRef File) const { std::lock_guard Lock(Mutex); auto It = OpenedFiles.find(File); @@ -57,8 +57,11 @@ /// returns it. std::shared_ptr removeIfPresent(PathRef File); + /// Gets used memory for each of the stored files. + std::vector> getUsedBytesPerFile() const; + private: - std::mutex Mutex; + mutable std::mutex Mutex; llvm::StringMap> OpenedFiles; ASTParsedCallback ASTCallback; }; Index: clangd/ClangdUnitStore.cpp =================================================================== --- clangd/ClangdUnitStore.cpp +++ clangd/ClangdUnitStore.cpp @@ -25,3 +25,13 @@ OpenedFiles.erase(It); return Result; } +std::vector> +CppFileCollection::getUsedBytesPerFile() const { + std::lock_guard Lock(Mutex); + std::vector> Result; + Result.reserve(OpenedFiles.size()); + for (auto &&PathAndFile : OpenedFiles) + Result.push_back( + {PathAndFile.first().str(), PathAndFile.second->getUsedBytes()}); + return Result; +} Index: unittests/clangd/ClangdTests.cpp =================================================================== --- unittests/clangd/ClangdTests.cpp +++ unittests/clangd/ClangdTests.cpp @@ -17,6 +17,7 @@ #include "llvm/Support/Errc.h" #include "llvm/Support/Path.h" #include "llvm/Support/Regex.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include #include @@ -28,6 +29,14 @@ namespace clang { namespace clangd { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Gt; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + namespace { // Don't wait for async ops in clangd test more than that to avoid blocking @@ -416,6 +425,42 @@ EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags()); } +TEST_F(ClangdVFSTest, MemoryUsage) { + MockFSProvider FS; + ErrorCheckingDiagConsumer DiagConsumer; + MockCompilationDatabase CDB; + ClangdServer Server(CDB, DiagConsumer, FS, + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true); + + // No need to sync reparses, because reparses are performed on the calling + // thread. + Path FooCpp = getVirtualTestFilePath("foo.cpp").str(); + const auto SourceContents = R"cpp( +struct Something { + int method(); +}; +)cpp"; + Path BarCpp = getVirtualTestFilePath("bar.cpp").str(); + + FS.Files[FooCpp] = ""; + FS.Files[BarCpp] = ""; + + EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty()); + + Server.addDocument(Context::empty(), FooCpp, SourceContents); + Server.addDocument(Context::empty(), BarCpp, SourceContents); + + EXPECT_THAT(Server.getUsedBytesPerFile(), + UnorderedElementsAre(Pair(FooCpp, Gt(0u)), Pair(BarCpp, Gt(0u)))); + + Server.removeDocument(Context::empty(), FooCpp); + EXPECT_THAT(Server.getUsedBytesPerFile(), ElementsAre(Pair(BarCpp, Gt(0u)))); + + Server.removeDocument(Context::empty(), BarCpp); + EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty()); +} + class ClangdThreadingTest : public ClangdVFSTest {}; TEST_F(ClangdThreadingTest, StressTest) {