Index: clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp =================================================================== --- clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp +++ clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp @@ -33,6 +33,15 @@ return loadIndex(IndexFilename, {}, true); } +SymbolSlab loadSlab() { + auto Slab = symbolsFromFile(IndexFilename); + if (!Slab) { + llvm::errs() << llvm::toString(Slab.takeError()) << '\n'; + exit(1); + } + return std::move(*Slab); +} + // Reads JSON array of serialized FuzzyFindRequest's from user-provided file. std::vector extractQueriesFromLogs() { std::ifstream InputStream(RequestsFilename); @@ -66,6 +75,15 @@ return Requests; } +static void DexQueries(benchmark::State &State) { + const auto Dex = buildDex(); + const auto Requests = extractQueriesFromLogs(); + for (auto _ : State) + for (const auto &Request : Requests) + Dex->fuzzyFind(Request, [](const Symbol &S) {}); +} +BENCHMARK(DexQueries)->Unit(benchmark::kMillisecond); + static void MemQueries(benchmark::State &State) { const auto Mem = buildMem(); const auto Requests = extractQueriesFromLogs(); @@ -73,24 +91,70 @@ for (const auto &Request : Requests) Mem->fuzzyFind(Request, [](const Symbol &S) {}); } -BENCHMARK(MemQueries); +BENCHMARK(MemQueries)->Unit(benchmark::kMillisecond); -static void DexQueries(benchmark::State &State) { +// This is not a *real* benchmark: it shows size of built MemIndex (in bytes). +// Same for the next "benchmark". +// FIXME(kbobyrev): Track memory usage caused by different index parts: +// SymbolSlab and RefSlabs, InverseIndex, PostingLists of different types for +// Dex, etc. +static void MemSize(benchmark::State &State) { + const auto Mem = buildMem(); + for (auto _ : State) + // Divide size of Mem by 1000 so that it will be correctly displayed in the + // benchmark report (possible options for time units are ms, ns and us). + State.SetIterationTime(/*double Seconds=*/Mem->estimateMemoryUsage() / + 1000); + State.SetLabel("This tracks in-memory size of MemIndex (SymbolSlab + Index " + "overhead) in bytes"); +} +BENCHMARK(MemSize) + ->UseManualTime() + ->Unit(benchmark::kMillisecond) + ->Iterations(1); + +static void DexSize(benchmark::State &State) { const auto Dex = buildDex(); - const auto Requests = extractQueriesFromLogs(); for (auto _ : State) - for (const auto &Request : Requests) - Dex->fuzzyFind(Request, [](const Symbol &S) {}); + State.SetIterationTime(Dex->estimateMemoryUsage() / 1000); + State.SetLabel("This tracks in-memory size of Dex (SymbolSlab + Index " + "overhead) in bytes"); +} +BENCHMARK(DexSize) + ->UseManualTime() + ->Unit(benchmark::kMillisecond) + ->Iterations(1); + +static void DexOverhead(benchmark::State &State) { + const auto Slab = loadSlab(); + const auto Dex = buildDex(); + for (auto _ : State) + State.SetIterationTime((Dex->estimateMemoryUsage() - Slab.bytes()) / 1000); + State.SetLabel("This tracks in-memory size of Dex Index overhead (and " + "excludes underlying SymbolSlab size) in bytes"); +} +BENCHMARK(DexOverhead) + ->UseManualTime() + ->Unit(benchmark::kMillisecond) + ->Iterations(1); + +static void SymbolSlabLoading(benchmark::State &State) { + for (auto _ : State) + benchmark::DoNotOptimize(loadSlab()); +} +BENCHMARK(SymbolSlabLoading)->Unit(benchmark::kMillisecond); + +static void DexBuild(benchmark::State &State) { + auto Slab = loadSlab(); + for (auto _ : State) + benchmark::DoNotOptimize(dex::Dex::build(std::move(Slab), {})); } -BENCHMARK(DexQueries); +BENCHMARK(DexBuild)->Unit(benchmark::kMillisecond); } // namespace } // namespace clangd } // namespace clang -// FIXME(kbobyrev): Add index building time benchmarks. -// FIXME(kbobyrev): Add memory consumption "benchmarks" by manually measuring -// in-memory index size and reporting it as time. // FIXME(kbobyrev): Create a logger wrapper to suppress debugging info printer. int main(int argc, char *argv[]) { if (argc < 3) { Index: clang-tools-extra/clangd/index/Serialization.cpp =================================================================== --- clang-tools-extra/clangd/index/Serialization.cpp +++ clang-tools-extra/clangd/index/Serialization.cpp @@ -6,9 +6,11 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// + #include "Serialization.h" #include "Index.h" #include "RIFF.h" +#include "Trace.h" #include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -309,6 +311,7 @@ constexpr static uint32_t Version = 3; Expected readIndexFile(StringRef Data) { + trace::Span Tracer("ParseRIFF"); auto RIFF = riff::readFile(Data); if (!RIFF) return RIFF.takeError(); Index: clang-tools-extra/clangd/index/SymbolYAML.h =================================================================== --- clang-tools-extra/clangd/index/SymbolYAML.h +++ clang-tools-extra/clangd/index/SymbolYAML.h @@ -29,6 +29,9 @@ // Read symbols from a YAML-format string. SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent); +// Read symbols from YAML or RIFF file. +llvm::Expected symbolsFromFile(llvm::StringRef SymbolFilename); + // Read one symbol from a YAML-stream. // The returned symbol is backed by Input. Symbol SymbolFromYAML(llvm::yaml::Input &Input); Index: clang-tools-extra/clangd/index/SymbolYAML.cpp =================================================================== --- clang-tools-extra/clangd/index/SymbolYAML.cpp +++ clang-tools-extra/clangd/index/SymbolYAML.cpp @@ -9,6 +9,7 @@ #include "SymbolYAML.h" #include "Index.h" +#include "Logger.h" #include "Serialization.h" #include "Trace.h" #include "dex/Dex.h" @@ -172,6 +173,7 @@ namespace clangd { SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent) { + trace::Span Tracer("ParseYAML"); llvm::yaml::Input Yin(YAMLContent); std::vector S; Yin >> S; @@ -182,6 +184,24 @@ return std::move(Syms).build(); } +llvm::Expected symbolsFromFile(llvm::StringRef SymbolFilename) { + auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); + if (!Buffer) + return llvm::make_error( + ("Can't open " + SymbolFilename).str(), llvm::errc::invalid_argument); + + StringRef Data = Buffer->get()->getBuffer(); + + if (!Data.startswith("RIFF")) { // Magic for binary index file. + return symbolsFromYAML(Data); + } + auto RIFF = readIndexFile(Data); + return RIFF ? llvm::Expected(std::move(*RIFF->Symbols)) + : llvm::make_error( + ("Can't open " + SymbolFilename).str(), + llvm::errc::invalid_argument); +} + Symbol SymbolFromYAML(llvm::yaml::Input &Input) { Symbol S; Input >> S; @@ -206,30 +226,16 @@ llvm::ArrayRef URISchemes, bool UseDex) { trace::Span OverallTracer("LoadIndex"); - auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); - if (!Buffer) { - llvm::errs() << "Can't open " << SymbolFilename << "\n"; - return nullptr; - } - StringRef Data = Buffer->get()->getBuffer(); - - llvm::Optional Slab; - if (Data.startswith("RIFF")) { // Magic for binary index file. - trace::Span Tracer("ParseRIFF"); - if (auto RIFF = readIndexFile(Data)) - Slab = std::move(RIFF->Symbols); - else - llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n"; - } else { - trace::Span Tracer("ParseYAML"); - Slab = symbolsFromYAML(Data); - } - + auto Slab = symbolsFromFile(SymbolFilename); if (!Slab) return nullptr; trace::Span Tracer("BuildIndex"); - return UseDex ? dex::Dex::build(std::move(*Slab), URISchemes) - : MemIndex::build(std::move(*Slab), RefSlab()); + auto Index = UseDex ? dex::Dex::build(std::move(*Slab), URISchemes) + : MemIndex::build(std::move(*Slab), RefSlab()); + vlog("Loaded {0} from {1} with estimated memory usage {2}", + UseDex ? "Dex" : "MemIndex", SymbolFilename, + Index->estimateMemoryUsage()); + return Index; } } // namespace clangd Index: clang-tools-extra/clangd/index/dex/Dex.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/Dex.cpp +++ clang-tools-extra/clangd/index/dex/Dex.cpp @@ -124,9 +124,6 @@ for (const auto &TokenToPostingList : TempInvertedIndex) InvertedIndex.insert({TokenToPostingList.first, PostingList(move(TokenToPostingList.second))}); - - vlog("Built Dex with estimated memory usage {0} bytes.", - estimateMemoryUsage()); } /// Constructs iterators over tokens extracted from the query and exhausts it @@ -238,8 +235,9 @@ Bytes += SymbolQuality.size() * sizeof(float); Bytes += LookupTable.getMemorySize(); Bytes += InvertedIndex.getMemorySize(); - for (const auto &P : InvertedIndex) - Bytes += P.second.bytes(); + for (const auto &TokenAndPostingList : InvertedIndex) + Bytes += TokenAndPostingList.first.Data.size() + + TokenAndPostingList.second.bytes(); return Bytes + BackingDataSize; }