Skip to content

Commit

Permalink
[clangd] Provide a helper to report estimated memory usage per-file
Browse files Browse the repository at this point in the history
Reviewers: sammccall, ioeric, hokein

Reviewed By: ioeric

Subscribers: klimek, cfe-commits, jkorous-apple

Differential Revision: https://reviews.llvm.org/D42480

llvm-svn: 323425
  • Loading branch information
ilya-biryukov committed Jan 25, 2018
1 parent 3ef08a9 commit df84234
Showing 7 changed files with 114 additions and 3 deletions.
5 changes: 5 additions & 0 deletions clang-tools-extra/clangd/ClangdServer.cpp
Original file line number Diff line number Diff line change
@@ -635,3 +635,8 @@ void ClangdServer::onFileEvent(const DidChangeWatchedFilesParams &Params) {
// FIXME: Do nothing for now. This will be used for indexing and potentially
// invalidating other caches.
}

std::vector<std::pair<Path, std::size_t>>
ClangdServer::getUsedBytesPerFile() const {
return Units.getUsedBytesPerFile();
}
9 changes: 9 additions & 0 deletions clang-tools-extra/clangd/ClangdServer.h
Original file line number Diff line number Diff line change
@@ -322,6 +322,15 @@ class ClangdServer {
/// Called when an event occurs for a watched file in the workspace.
void onFileEvent(const DidChangeWatchedFilesParams &Params);

/// Returns estimated memory usage for each of the currently open files.
/// The order of results is unspecified.
/// Overall memory usage of clangd may be significantly more than reported
/// here, as this metric does not account (at least) for:
/// - memory occupied by static and dynamic index,
/// - memory required for in-flight requests,
/// FIXME: those metrics might be useful too, we should add them.
std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;

private:
/// FIXME: This stats several files to find a .clang-format file. I/O can be
/// slow. Think of a way to cache this.
28 changes: 27 additions & 1 deletion clang-tools-extra/clangd/ClangdUnit.cpp
Original file line number Diff line number Diff line change
@@ -35,6 +35,10 @@ using namespace clang;

namespace {

template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
return Vec.capacity() * sizeof(T);
}

class DeclTrackingASTConsumer : public ASTConsumer {
public:
DeclTrackingASTConsumer(std::vector<const Decl *> &TopLevelDecls)
@@ -332,6 +336,14 @@ const std::vector<DiagWithFixIts> &ParsedAST::getDiagnostics() const {
return Diags;
}

std::size_t ParsedAST::getUsedBytes() const {
auto &AST = getASTContext();
// FIXME(ibiryukov): we do not account for the dynamically allocated part of
// SmallVector<FixIt> inside each Diag.
return AST.getASTAllocatedMemory() + AST.getSideTableAllocatedMemory() +
::getUsedBytes(TopLevelDecls) + ::getUsedBytes(Diags);
}

PreambleData::PreambleData(PrecompiledPreamble Preamble,
std::vector<serialization::DeclID> TopLevelDeclIDs,
std::vector<DiagWithFixIts> Diags)
@@ -370,7 +382,8 @@ CppFile::CppFile(PathRef FileName, bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
ASTParsedCallback ASTCallback)
: FileName(FileName), StorePreamblesInMemory(StorePreamblesInMemory),
RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)),
RebuildCounter(0), RebuildInProgress(false), ASTMemUsage(0),
PreambleMemUsage(0), PCHs(std::move(PCHs)),
ASTCallback(std::move(ASTCallback)) {
// FIXME(ibiryukov): we should pass a proper Context here.
log(Context::empty(), "Created CppFile for " + FileName);
@@ -419,7 +432,9 @@ UniqueFunction<void()> CppFile::deferCancelRebuild() {
return;

// Set empty results for Promises.
That->PreambleMemUsage = 0;
That->PreamblePromise.set_value(nullptr);
That->ASTMemUsage = 0;
That->ASTPromise.set_value(std::make_shared<ParsedASTWrapper>(llvm::None));
};
}
@@ -573,6 +588,8 @@ CppFile::deferRebuild(ParseInputs &&Inputs) {
That->LatestAvailablePreamble = NewPreamble;
if (RequestRebuildCounter != That->RebuildCounter)
return llvm::None; // Our rebuild request was cancelled, do nothing.
That->PreambleMemUsage =
NewPreamble ? NewPreamble->Preamble.getSize() : 0;
That->PreamblePromise.set_value(NewPreamble);
} // unlock Mutex

@@ -609,6 +626,7 @@ CppFile::deferRebuild(ParseInputs &&Inputs) {
return Diagnostics; // Our rebuild request was cancelled, don't set
// ASTPromise.

That->ASTMemUsage = NewAST ? NewAST->getUsedBytes() : 0;
That->ASTPromise.set_value(
std::make_shared<ParsedASTWrapper>(std::move(NewAST)));
} // unlock Mutex
@@ -635,6 +653,14 @@ std::shared_future<std::shared_ptr<ParsedASTWrapper>> CppFile::getAST() const {
return ASTFuture;
}

std::size_t CppFile::getUsedBytes() const {
std::lock_guard<std::mutex> Lock(Mutex);
// FIXME: We should not store extra size fields. When we store AST and
// Preamble directly, not inside futures, we could compute the sizes from the
// stored AST and the preamble in this function directly.
return ASTMemUsage + PreambleMemUsage;
}

CppFile::RebuildGuard::RebuildGuard(CppFile &File,
unsigned RequestRebuildCounter)
: File(File), RequestRebuildCounter(RequestRebuildCounter) {
13 changes: 13 additions & 0 deletions clang-tools-extra/clangd/ClangdUnit.h
Original file line number Diff line number Diff line change
@@ -96,6 +96,10 @@ class ParsedAST {

const std::vector<DiagWithFixIts> &getDiagnostics() const;

/// Returns the esitmated size of the AST and the accessory structures, in
/// bytes. Does not include the size of the preamble.
std::size_t getUsedBytes() const;

private:
ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
@@ -216,6 +220,10 @@ class CppFile : public std::enable_shared_from_this<CppFile> {
/// always be non-null.
std::shared_future<std::shared_ptr<ParsedASTWrapper>> getAST() const;

/// Returns an estimated size, in bytes, currently occupied by the AST and the
/// Preamble.
std::size_t getUsedBytes() const;

private:
/// A helper guard that manages the state of CppFile during rebuild.
class RebuildGuard {
@@ -244,6 +252,11 @@ class CppFile : public std::enable_shared_from_this<CppFile> {
/// Condition variable to indicate changes to RebuildInProgress.
std::condition_variable RebuildCond;

/// Size of the last built AST, in bytes.
std::size_t ASTMemUsage;
/// Size of the last build Preamble, in bytes.
std::size_t PreambleMemUsage;

/// Promise and future for the latests AST. Fulfilled during rebuild.
/// We use std::shared_ptr here because MVSC fails to compile non-copyable
/// classes as template arguments of promise/future.
10 changes: 10 additions & 0 deletions clang-tools-extra/clangd/ClangdUnitStore.cpp
Original file line number Diff line number Diff line change
@@ -25,3 +25,13 @@ std::shared_ptr<CppFile> CppFileCollection::removeIfPresent(PathRef File) {
OpenedFiles.erase(It);
return Result;
}
std::vector<std::pair<Path, std::size_t>>
CppFileCollection::getUsedBytesPerFile() const {
std::lock_guard<std::mutex> Lock(Mutex);
std::vector<std::pair<Path, std::size_t>> Result;
Result.reserve(OpenedFiles.size());
for (auto &&PathAndFile : OpenedFiles)
Result.push_back(
{PathAndFile.first().str(), PathAndFile.second->getUsedBytes()});
return Result;
}
7 changes: 5 additions & 2 deletions clang-tools-extra/clangd/ClangdUnitStore.h
Original file line number Diff line number Diff line change
@@ -44,7 +44,7 @@ class CppFileCollection {
return It->second;
}

std::shared_ptr<CppFile> getFile(PathRef File) {
std::shared_ptr<CppFile> getFile(PathRef File) const {
std::lock_guard<std::mutex> Lock(Mutex);

auto It = OpenedFiles.find(File);
@@ -57,8 +57,11 @@ class CppFileCollection {
/// returns it.
std::shared_ptr<CppFile> removeIfPresent(PathRef File);

/// Gets used memory for each of the stored files.
std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;

private:
std::mutex Mutex;
mutable std::mutex Mutex;
llvm::StringMap<std::shared_ptr<CppFile>> OpenedFiles;
ASTParsedCallback ASTCallback;
};
45 changes: 45 additions & 0 deletions clang-tools-extra/unittests/clangd/ClangdTests.cpp
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
#include "llvm/Support/Errc.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Regex.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <chrono>
@@ -28,6 +29,14 @@

namespace clang {
namespace clangd {

using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::Gt;
using ::testing::IsEmpty;
using ::testing::Pair;
using ::testing::UnorderedElementsAre;

namespace {

// Don't wait for async ops in clangd test more than that to avoid blocking
@@ -416,6 +425,42 @@ struct bar { T x; };
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
}

TEST_F(ClangdVFSTest, MemoryUsage) {
MockFSProvider FS;
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB;
ClangdServer Server(CDB, DiagConsumer, FS,
/*AsyncThreadsCount=*/0,
/*StorePreamblesInMemory=*/true);

// No need to sync reparses, because reparses are performed on the calling
// thread.
Path FooCpp = getVirtualTestFilePath("foo.cpp").str();
const auto SourceContents = R"cpp(
struct Something {
int method();
};
)cpp";
Path BarCpp = getVirtualTestFilePath("bar.cpp").str();

FS.Files[FooCpp] = "";
FS.Files[BarCpp] = "";

EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty());

Server.addDocument(Context::empty(), FooCpp, SourceContents);
Server.addDocument(Context::empty(), BarCpp, SourceContents);

EXPECT_THAT(Server.getUsedBytesPerFile(),
UnorderedElementsAre(Pair(FooCpp, Gt(0u)), Pair(BarCpp, Gt(0u))));

Server.removeDocument(Context::empty(), FooCpp);
EXPECT_THAT(Server.getUsedBytesPerFile(), ElementsAre(Pair(BarCpp, Gt(0u))));

Server.removeDocument(Context::empty(), BarCpp);
EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty());
}

class ClangdThreadingTest : public ClangdVFSTest {};

TEST_F(ClangdThreadingTest, StressTest) {

0 comments on commit df84234

Please sign in to comment.