Index: clang-tools-extra/clangd/index/FileIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/FileIndex.cpp +++ clang-tools-extra/clangd/index/FileIndex.cpp @@ -127,11 +127,18 @@ } } + size_t StorageSize = RefsStorage.size() * sizeof(Ref); + for (const auto &Slab : SymbolSlabs) + StorageSize += Slab->bytes(); + for (const auto &RefSlab : RefSlabs) + StorageSize += RefSlab->bytes(); + // Index must keep the slabs and contiguous ranges alive. return llvm::make_unique( llvm::make_pointee_range(AllSymbols), std::move(AllRefs), std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), - std::move(RefsStorage))); + std::move(RefsStorage)), + StorageSize); } void FileIndex::update(PathRef Path, ASTContext *AST, Index: clang-tools-extra/clangd/index/Index.h =================================================================== --- clang-tools-extra/clangd/index/Index.h +++ clang-tools-extra/clangd/index/Index.h @@ -509,10 +509,13 @@ // until the call returns (even if reset() is called). bool fuzzyFind(const FuzzyFindRequest &, llvm::function_ref) const override; + void lookup(const LookupRequest &, llvm::function_ref) const override; + void refs(const RefsRequest &, llvm::function_ref) const override; + size_t estimateMemoryUsage() const override; private: Index: clang-tools-extra/clangd/index/Index.cpp =================================================================== --- clang-tools-extra/clangd/index/Index.cpp +++ clang-tools-extra/clangd/index/Index.cpp @@ -179,14 +179,17 @@ llvm::function_ref CB) const { return snapshot()->fuzzyFind(R, CB); } + void SwapIndex::lookup(const LookupRequest &R, llvm::function_ref CB) const { return snapshot()->lookup(R, CB); } + void SwapIndex::refs(const RefsRequest &R, llvm::function_ref CB) const { return snapshot()->refs(R, CB); } + size_t SwapIndex::estimateMemoryUsage() const { return snapshot()->estimateMemoryUsage(); } Index: clang-tools-extra/clangd/index/MemIndex.h =================================================================== --- clang-tools-extra/clangd/index/MemIndex.h +++ clang-tools-extra/clangd/index/MemIndex.h @@ -30,11 +30,13 @@ } // Symbols are owned by BackingData, Index takes ownership. template - MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData) + MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData, + size_t BackingDataSize) : MemIndex(std::forward(Symbols), std::forward(Refs)) { KeepAlive = std::shared_ptr( std::make_shared(std::move(BackingData)), nullptr); + this->BackingDataSize = BackingDataSize; } /// Builds an index from slabs. The index takes ownership of the data. @@ -58,6 +60,8 @@ // A map from symbol ID to symbol refs, support query by IDs. llvm::DenseMap> Refs; std::shared_ptr KeepAlive; // poor man's move-only std::any + // Size of memory retained by KeepAlive. + size_t BackingDataSize = 0; }; } // namespace clangd Index: clang-tools-extra/clangd/index/MemIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/MemIndex.cpp +++ clang-tools-extra/clangd/index/MemIndex.cpp @@ -16,8 +16,11 @@ namespace clangd { std::unique_ptr MemIndex::build(SymbolSlab Slab, RefSlab Refs) { + // Store Slab size before it is moved. + const auto BackingDataSize = Slab.bytes(); auto Data = std::make_pair(std::move(Slab), std::move(Refs)); - return llvm::make_unique(Data.first, Data.second, std::move(Data)); + return llvm::make_unique(Data.first, Data.second, std::move(Data), + BackingDataSize); } bool MemIndex::fuzzyFind( @@ -70,7 +73,7 @@ } size_t MemIndex::estimateMemoryUsage() const { - return Index.getMemorySize(); + return Index.getMemorySize() + BackingDataSize; } } // namespace clangd Index: clang-tools-extra/clangd/index/dex/DexIndex.h =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.h +++ clang-tools-extra/clangd/index/dex/DexIndex.h @@ -55,17 +55,21 @@ } // Symbols are owned by BackingData, Index takes ownership. template - DexIndex(Range &&Symbols, Payload &&BackingData, + DexIndex(Range &&Symbols, Payload &&BackingData, size_t BackingDataSize, llvm::ArrayRef URISchemes) : DexIndex(std::forward(Symbols), URISchemes) { KeepAlive = std::shared_ptr( std::make_shared(std::move(BackingData)), nullptr); + this->BackingDataSize = BackingDataSize; } /// Builds an index from a slab. The index takes ownership of the slab. static std::unique_ptr build(SymbolSlab Slab, llvm::ArrayRef URISchemes) { - return llvm::make_unique(Slab, std::move(Slab), URISchemes); + // Store Slab size before it is moved. + const auto BackingDataSize = Slab.bytes(); + return llvm::make_unique(Slab, std::move(Slab), BackingDataSize, + URISchemes); } bool @@ -96,6 +100,8 @@ /// during the fuzzyFind process. llvm::DenseMap InvertedIndex; std::shared_ptr KeepAlive; // poor man's move-only std::any + // Size of memory retained by KeepAlive. + size_t BackingDataSize = 0; std::vector URISchemes; }; Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.cpp +++ clang-tools-extra/clangd/index/dex/DexIndex.cpp @@ -228,15 +228,13 @@ } size_t DexIndex::estimateMemoryUsage() const { - size_t Bytes = - LookupTable.size() * sizeof(std::pair); - Bytes += SymbolQuality.size() * sizeof(std::pair); - Bytes += InvertedIndex.size() * sizeof(Token); - - for (const auto &P : InvertedIndex) { + size_t Bytes = Symbols.size() * sizeof(const Symbol *); + Bytes += SymbolQuality.size() * sizeof(float); + Bytes += LookupTable.getMemorySize(); + Bytes += InvertedIndex.getMemorySize(); + for (const auto &P : InvertedIndex) Bytes += P.second.size() * sizeof(DocID); - } - return Bytes; + return Bytes + BackingDataSize; } std::vector generateProximityURIs(llvm::StringRef URIPath) { Index: clang-tools-extra/unittests/clangd/IndexTests.cpp =================================================================== --- clang-tools-extra/unittests/clangd/IndexTests.cpp +++ clang-tools-extra/unittests/clangd/IndexTests.cpp @@ -10,11 +10,11 @@ #include "Annotations.h" #include "TestIndex.h" #include "TestTU.h" -#include "gmock/gmock.h" #include "index/FileIndex.h" #include "index/Index.h" #include "index/MemIndex.h" #include "index/Merge.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" using testing::_; @@ -58,11 +58,11 @@ auto Token = std::make_shared(); std::weak_ptr WeakToken = Token; - SwapIndex S( - llvm::make_unique(SymbolSlab(), RefSlab(), std::move(Token))); - EXPECT_FALSE(WeakToken.expired()); // Current MemIndex keeps it alive. + SwapIndex S(llvm::make_unique( + SymbolSlab(), RefSlab(), std::move(Token), /*BackingDataSize=*/0)); + EXPECT_FALSE(WeakToken.expired()); // Current MemIndex keeps it alive. S.reset(llvm::make_unique()); // Now the MemIndex is destroyed. - EXPECT_TRUE(WeakToken.expired()); // So the token is too. + EXPECT_TRUE(WeakToken.expired()); // So the token is too. } TEST(MemIndexTest, MemIndexDeduplicate) { @@ -281,7 +281,7 @@ FileURI("unittest:///test2.cc")))))); } -MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") { +MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") { return (arg.IncludeHeader == IncludeHeader) && (arg.References == References); }