Index: clang-tools-extra/clangd/index/FileIndex.h =================================================================== --- clang-tools-extra/clangd/index/FileIndex.h +++ clang-tools-extra/clangd/index/FileIndex.h @@ -46,6 +46,8 @@ // The shared_ptr keeps the symbols alive std::shared_ptr> allSymbols(); + size_t estimateMemoryUsage() const; + private: mutable std::mutex Mutex; Index: clang-tools-extra/clangd/index/FileIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/FileIndex.cpp +++ clang-tools-extra/clangd/index/FileIndex.cpp @@ -85,6 +85,13 @@ return {std::move(Snap), Pointers}; } +size_t FileSymbols::estimateMemoryUsage() const { + size_t Result = 0; + for (const auto &P : FileToSlabs) + Result += P.second->size(); + return Result; +} + void FileIndex::update(PathRef Path, ASTContext *AST, std::shared_ptr PP, llvm::Optional> TopLevelDecls) { @@ -119,7 +126,7 @@ } size_t FileIndex::estimateMemoryUsage() const { - return Index.estimateMemoryUsage(); + return Index.estimateMemoryUsage() + FSymbols.estimateMemoryUsage(); } } // namespace clangd Index: clang-tools-extra/clangd/index/MemIndex.h =================================================================== --- clang-tools-extra/clangd/index/MemIndex.h +++ clang-tools-extra/clangd/index/MemIndex.h @@ -22,7 +22,8 @@ public: /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain /// accessible as long as `Symbols` is kept alive. - void build(std::shared_ptr> Symbols); + void build(std::shared_ptr> Symbols, + size_t SlabSize=0); /// \brief Build index from a symbol slab. static std::unique_ptr build(SymbolSlab Slab); @@ -47,6 +48,13 @@ // Index is a set of symbols that are deduplicated by symbol IDs. // FIXME: build smarter index structure. llvm::DenseMap Index; + + // Stores size of the symbol slab which is referred by the Symbols stored in + // Mem. This is necessary for memory consumption estimation, because Mem only + // stores symbols and the actual link to the underlying symbol slab is lost as + // soon as it is constructed. + size_t PairedSlabSize; + mutable std::mutex Mutex; }; Index: clang-tools-extra/clangd/index/MemIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/MemIndex.cpp +++ clang-tools-extra/clangd/index/MemIndex.cpp @@ -15,7 +15,8 @@ namespace clang { namespace clangd { -void MemIndex::build(std::shared_ptr> Syms) { +void MemIndex::build(std::shared_ptr> Syms, + size_t SlabSize) { llvm::DenseMap TempIndex; for (const Symbol *Sym : *Syms) TempIndex[Sym->ID] = Sym; @@ -25,6 +26,7 @@ std::lock_guard Lock(Mutex); Index = std::move(TempIndex); Symbols = std::move(Syms); // Relase old symbols. + PairedSlabSize = SlabSize; } vlog("Built MemIndex with estimated memory usage {0} bytes.", @@ -33,7 +35,8 @@ std::unique_ptr MemIndex::build(SymbolSlab Slab) { auto Idx = llvm::make_unique(); - Idx->build(getSymbolsFromSlab(std::move(Slab))); + size_t SlabSize = Slab.bytes(); + Idx->build(getSymbolsFromSlab(std::move(Slab)), SlabSize); return std::move(Idx); } @@ -103,7 +106,7 @@ size_t MemIndex::estimateMemoryUsage() const { std::lock_guard Lock(Mutex); - return Index.getMemorySize(); + return Index.getMemorySize() + PairedSlabSize; } } // namespace clangd Index: clang-tools-extra/clangd/index/dex/DexIndex.h =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.h +++ clang-tools-extra/clangd/index/dex/DexIndex.h @@ -41,7 +41,8 @@ public: /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain /// accessible as long as `Symbols` is kept alive. - void build(std::shared_ptr> Syms); + void build(std::shared_ptr> Syms, + size_t SlabSize=0); /// \brief Build index from a symbol slab. static std::unique_ptr build(SymbolSlab Slab); @@ -73,6 +74,12 @@ // Inverted index is used to retrieve posting lists which are processed during // the fuzzyFind process. llvm::DenseMap InvertedIndex /*GUARDED_BY(Mutex)*/; + + // Stores size of the symbol slab which is referred by the Symbols stored in + // Dex. This is necessary for memory consumption estimation, because Dex only + // stores symbols and the actual link to the underlying symbol slab is lost as + // soon as it is constructed. + size_t PairedSlabSize; }; } // namespace dex Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/DexIndex.cpp +++ clang-tools-extra/clangd/index/dex/DexIndex.cpp @@ -36,7 +36,8 @@ } // namespace -void DexIndex::build(std::shared_ptr> Syms) { +void DexIndex::build(std::shared_ptr> Syms, + size_t SlabSize) { llvm::DenseMap TempLookupTable; llvm::DenseMap TempSymbolQuality; for (const Symbol *Sym : *Syms) { @@ -66,6 +67,7 @@ Symbols = std::move(Syms); InvertedIndex = std::move(TempInvertedIndex); SymbolQuality = std::move(TempSymbolQuality); + PairedSlabSize = SlabSize; } vlog("Built DexIndex with estimated memory usage {0} bytes.", @@ -74,7 +76,8 @@ std::unique_ptr DexIndex::build(SymbolSlab Slab) { auto Idx = llvm::make_unique(); - Idx->build(getSymbolsFromSlab(std::move(Slab))); + size_t SlabSize = Slab.bytes(); + Idx->build(getSymbolsFromSlab(std::move(Slab)), SlabSize); return std::move(Idx); } @@ -177,14 +180,15 @@ size_t DexIndex::estimateMemoryUsage() const { std::lock_guard Lock(Mutex); - size_t Bytes = - LookupTable.size() * sizeof(std::pair); + size_t Bytes = PairedSlabSize; + Bytes += LookupTable.size() * sizeof(std::pair); Bytes += SymbolQuality.size() * sizeof(std::pair); Bytes += InvertedIndex.size() * sizeof(Token); for (const auto &P : InvertedIndex) { Bytes += P.second.size() * sizeof(DocID); } + return Bytes; }