Index: clang-tools-extra/clangd/index/FileIndex.h =================================================================== --- clang-tools-extra/clangd/index/FileIndex.h +++ clang-tools-extra/clangd/index/FileIndex.h @@ -47,6 +47,8 @@ // The index keeps the symbols alive. std::unique_ptr buildMemIndex(); + size_t estimateMemoryUsage() const; + private: mutable std::mutex Mutex; @@ -73,6 +75,8 @@ update(PathRef Path, ASTContext *AST, std::shared_ptr PP, llvm::Optional> TopLevelDecls = llvm::None); + size_t estimateMemoryUsage() const; + private: // Only update() should swap the index. using SwapIndex::reset; Index: clang-tools-extra/clangd/index/FileIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/FileIndex.cpp +++ clang-tools-extra/clangd/index/FileIndex.cpp @@ -113,7 +113,15 @@ // Index must keep the slabs alive. return llvm::make_unique( llvm::make_pointee_range(AllSymbols), std::move(AllOccurrences), - std::make_pair(std::move(Slabs), std::move(OccurrenceSlabs))); + std::make_pair(std::move(Slabs), std::move(OccurrenceSlabs)), + /*BackingMemory=*/0); +} + +size_t FileSymbols::estimateMemoryUsage() const { + size_t Result = 0; + for (const auto &FileAndSlab : FileToSlabs) + Result += FileAndSlab.second->bytes(); + return Result; } void FileIndex::update(PathRef Path, ASTContext *AST, @@ -132,5 +140,9 @@ reset(FSymbols.buildMemIndex()); } +size_t FileIndex::estimateMemoryUsage() const { + return FSymbols.estimateMemoryUsage(); +} + } // namespace clangd } // namespace clang Index: clang-tools-extra/clangd/index/MemIndex.h =================================================================== --- clang-tools-extra/clangd/index/MemIndex.h +++ clang-tools-extra/clangd/index/MemIndex.h @@ -28,15 +28,18 @@ // All symbols and occurrences must outlive this index. // TODO: find a better type for Occurrences here. template - MemIndex(SymbolRange &&Symbols, OccurrenceMap Occurrences) - : Occurrences(std::move(Occurrences)) { + MemIndex(SymbolRange &&Symbols, OccurrenceMap Occurrences, + size_t BackingMemory = 0) + : Occurrences(std::move(Occurrences)), BackingMemory(BackingMemory) { for (const Symbol &S : Symbols) Index[S.ID] = &S; } // Symbols are owned by BackingData, Index takes ownership. template - MemIndex(Range &&Symbols, OccurrenceMap Occurrences, Payload &&BackingData) - : MemIndex(std::forward(Symbols), std::move(Occurrences)) { + MemIndex(Range &&Symbols, OccurrenceMap Occurrences, Payload &&BackingData, + size_t BackingMemory = 0) + : MemIndex(std::forward(Symbols), std::move(Occurrences), + BackingMemory) { KeepAlive = std::shared_ptr( std::make_shared(std::move(BackingData)), nullptr); } @@ -64,6 +67,10 @@ // A map from symbol ID to symbol occurrences, support query by IDs. OccurrenceMap Occurrences; std::shared_ptr KeepAlive; // poor man's move-only std::any + /// When paired with the SymbolSlab, the index owns underlying SymbolSlab and + /// keeps it alive. BackingMemory is the only way to know the size of that + /// Slab. + size_t BackingMemory; }; } // namespace clangd Index: clang-tools-extra/clangd/index/MemIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/MemIndex.cpp +++ clang-tools-extra/clangd/index/MemIndex.cpp @@ -23,8 +23,10 @@ for (const auto &Occurrence : SymbolAndOccurrences.second) Entry.push_back(&Occurrence); } + size_t BackingData = Slab.bytes(); auto Data = std::make_pair(std::move(Slab), std::move(Occurrences)); - return llvm::make_unique(Data.first, std::move(M), std::move(Data)); + return llvm::make_unique(Data.first, std::move(M), std::move(Data), + BackingData); } bool MemIndex::fuzzyFind( @@ -82,7 +84,7 @@ } size_t MemIndex::estimateMemoryUsage() const { - return Index.getMemorySize(); + return Index.getMemorySize() + BackingMemory; } } // namespace clangd Index: clang-tools-extra/clangd/index/dex/DexIndex.h =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.h +++ clang-tools-extra/clangd/index/dex/DexIndex.h @@ -40,8 +40,9 @@ public: // All symbols must outlive this index. template - DexIndex(Range &&Symbols, llvm::ArrayRef URISchemes) - : URISchemes(URISchemes) { + DexIndex(Range &&Symbols, llvm::ArrayRef URISchemes, + size_t BackingMemory = 0) + : BackingMemory(BackingMemory), URISchemes(URISchemes) { for (auto &&Sym : Symbols) this->Symbols.push_back(&Sym); buildIndex(); @@ -49,8 +50,8 @@ // Symbols are owned by BackingData, Index takes ownership. template DexIndex(Range &&Symbols, Payload &&BackingData, - llvm::ArrayRef URISchemes) - : DexIndex(std::forward(Symbols), URISchemes) { + llvm::ArrayRef URISchemes, size_t BackingMemory = 0) + : DexIndex(std::forward(Symbols), URISchemes, BackingMemory) { KeepAlive = std::shared_ptr( std::make_shared(std::move(BackingData)), nullptr); } @@ -58,7 +59,10 @@ /// Builds an index from a slab. The index takes ownership of the slab. static std::unique_ptr build(SymbolSlab Slab, llvm::ArrayRef URISchemes) { - return llvm::make_unique(Slab, std::move(Slab), URISchemes); + // Store SymbolSlab size before it's moved. + size_t BackingMemory = Slab.size(); + return llvm::make_unique(Slab, std::move(Slab), URISchemes, + BackingMemory); } bool @@ -90,6 +94,10 @@ /// during the fuzzyFind process. llvm::DenseMap InvertedIndex; std::shared_ptr KeepAlive; // poor man's move-only std::any + /// When paired with the SymbolSlab, the index owns underlying SymbolSlab and + /// keeps it alive. BackingMemory is the only way to know the size of that + /// Slab. + size_t BackingMemory; const std::vector URISchemes; }; Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.cpp +++ clang-tools-extra/clangd/index/dex/DexIndex.cpp @@ -213,15 +213,14 @@ } size_t DexIndex::estimateMemoryUsage() const { - size_t Bytes = - LookupTable.size() * sizeof(std::pair); - Bytes += SymbolQuality.size() * sizeof(std::pair); - Bytes += InvertedIndex.size() * sizeof(Token); + size_t Bytes = LookupTable.getMemorySize(); + Bytes += SymbolQuality.size() * sizeof(float); + Bytes += InvertedIndex.getMemorySize(); for (const auto &P : InvertedIndex) { Bytes += P.second.size() * sizeof(DocID); } - return Bytes; + return Bytes + BackingMemory; } } // namespace dex