Index: clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp =================================================================== --- clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp +++ clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp @@ -33,6 +33,16 @@ return loadIndex(IndexFilename, {}, true); } +SymbolSlab loadSlab() { + auto Slab = symbolsFromFile(IndexFilename); + if (!Slab) { + llvm::errs() << "Error when loading SymbolSlab from file: " << IndexFilename + << '\n'; + exit(1); + } + return std::move(*Slab); +} + // Reads JSON array of serialized FuzzyFindRequest's from user-provided file. std::vector extractQueriesFromLogs() { std::ifstream InputStream(RequestsFilename); @@ -66,6 +76,42 @@ return Requests; } +// This is not a *real* benchmark: it shows size of built MemIndex (in bytes). +// Same for the next "benchmark". +// FIXME(kbobyrev): Should this be separated into the BackingMemorySize +// (underlying SymbolSlab size) and Symbol Index (MemIndex/Dex) overhead? +static void MemSize(benchmark::State &State) { + const auto Mem = buildMem(); + for (auto _ : State) + // Divide size of Mem by 1000 so that it will be correctly displayed in the + // benchmark report (possible options for time units are ms, ns and us). + State.SetIterationTime(/*double Seconds=*/Mem->estimateMemoryUsage() / + 1000); + State.counters["Memory Usage (bytes)"] = Mem->estimateMemoryUsage(); +} +BENCHMARK(MemSize)->UseManualTime()->Unit(benchmark::kMillisecond); + +static void DexSize(benchmark::State &State) { + const auto Dex = buildDex(); + for (auto _ : State) + State.SetIterationTime(Dex->estimateMemoryUsage() / 1000); + State.counters["Memory Usage (bytes)"] = Dex->estimateMemoryUsage(); +} +BENCHMARK(DexSize)->UseManualTime()->Unit(benchmark::kMillisecond); + +static void SymbolSlabLoading(benchmark::State &State) { + for (auto _ : State) + benchmark::DoNotOptimize(loadSlab()); +} +BENCHMARK(SymbolSlabLoading)->Unit(benchmark::kMillisecond); + +static void DexBuild(benchmark::State &State) { + auto Slab = loadSlab(); + for (auto _ : State) + benchmark::DoNotOptimize(dex::Dex::build(std::move(Slab), {})); +} +BENCHMARK(DexBuild)->Unit(benchmark::kMillisecond); + static void MemQueries(benchmark::State &State) { const auto Mem = buildMem(); const auto Requests = extractQueriesFromLogs(); @@ -73,7 +119,7 @@ for (const auto &Request : Requests) Mem->fuzzyFind(Request, [](const Symbol &S) {}); } -BENCHMARK(MemQueries); +BENCHMARK(MemQueries)->Unit(benchmark::kMillisecond); static void DexQueries(benchmark::State &State) { const auto Dex = buildDex(); @@ -82,15 +128,12 @@ for (const auto &Request : Requests) Dex->fuzzyFind(Request, [](const Symbol &S) {}); } -BENCHMARK(DexQueries); +BENCHMARK(DexQueries)->Unit(benchmark::kMillisecond); } // namespace } // namespace clangd } // namespace clang -// FIXME(kbobyrev): Add index building time benchmarks. -// FIXME(kbobyrev): Add memory consumption "benchmarks" by manually measuring -// in-memory index size and reporting it as time. // FIXME(kbobyrev): Create a logger wrapper to suppress debugging info printer. int main(int argc, char *argv[]) { if (argc < 3) { @@ -108,4 +151,5 @@ argc -= 2; ::benchmark::Initialize(&argc, argv); ::benchmark::RunSpecifiedBenchmarks(); + return 0; } Index: clang-tools-extra/clangd/index/SymbolYAML.h =================================================================== --- clang-tools-extra/clangd/index/SymbolYAML.h +++ clang-tools-extra/clangd/index/SymbolYAML.h @@ -29,6 +29,9 @@ // Read symbols from a YAML-format string. SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent); +// Read symbols from YAML or RIFF file. +llvm::Optional symbolsFromFile(llvm::StringRef SymbolFilename); + // Read one symbol from a YAML-stream. // The returned symbol is backed by Input. Symbol SymbolFromYAML(llvm::yaml::Input &Input); Index: clang-tools-extra/clangd/index/SymbolYAML.cpp =================================================================== --- clang-tools-extra/clangd/index/SymbolYAML.cpp +++ clang-tools-extra/clangd/index/SymbolYAML.cpp @@ -9,6 +9,7 @@ #include "SymbolYAML.h" #include "Index.h" +#include "Logger.h" #include "Serialization.h" #include "Trace.h" #include "dex/Dex.h" @@ -182,6 +183,28 @@ return std::move(Syms).build(); } +llvm::Optional symbolsFromFile(llvm::StringRef SymbolFilename) { + auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); + if (!Buffer) { + llvm::errs() << "Can't open " << SymbolFilename << "\n"; + return llvm::None; + } + StringRef Data = Buffer->get()->getBuffer(); + + llvm::Optional Slab; + if (Data.startswith("RIFF")) { // Magic for binary index file. + trace::Span Tracer("ParseRIFF"); + if (auto RIFF = readIndexFile(Data)) + Slab = std::move(RIFF->Symbols); + else + llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n"; + } else { + trace::Span Tracer("ParseYAML"); + Slab = symbolsFromYAML(Data); + } + return Slab; +} + Symbol SymbolFromYAML(llvm::yaml::Input &Input) { Symbol S; Input >> S; @@ -206,30 +229,16 @@ llvm::ArrayRef URISchemes, bool UseDex) { trace::Span OverallTracer("LoadIndex"); - auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); - if (!Buffer) { - llvm::errs() << "Can't open " << SymbolFilename << "\n"; - return nullptr; - } - StringRef Data = Buffer->get()->getBuffer(); - - llvm::Optional Slab; - if (Data.startswith("RIFF")) { // Magic for binary index file. - trace::Span Tracer("ParseRIFF"); - if (auto RIFF = readIndexFile(Data)) - Slab = std::move(RIFF->Symbols); - else - llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n"; - } else { - trace::Span Tracer("ParseYAML"); - Slab = symbolsFromYAML(Data); - } - + auto Slab = symbolsFromFile(SymbolFilename); if (!Slab) return nullptr; trace::Span Tracer("BuildIndex"); - return UseDex ? dex::Dex::build(std::move(*Slab), URISchemes) - : MemIndex::build(std::move(*Slab), RefSlab()); + auto Index = UseDex ? dex::Dex::build(std::move(*Slab), URISchemes) + : MemIndex::build(std::move(*Slab), RefSlab()); + vlog("Loaded {0} from {1} with estimated memory usage {2}", + UseDex ? "Dex" : "MemIndex", SymbolFilename, + Index->estimateMemoryUsage()); + return Index; } } // namespace clangd Index: clang-tools-extra/clangd/index/dex/Dex.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/Dex.cpp +++ clang-tools-extra/clangd/index/dex/Dex.cpp @@ -124,9 +124,6 @@ for (const auto &TokenToPostingList : TempInvertedIndex) InvertedIndex.insert({TokenToPostingList.first, PostingList(move(TokenToPostingList.second))}); - - vlog("Built Dex with estimated memory usage {0} bytes.", - estimateMemoryUsage()); } /// Constructs iterators over tokens extracted from the query and exhausts it @@ -239,7 +236,7 @@ Bytes += LookupTable.getMemorySize(); Bytes += InvertedIndex.getMemorySize(); for (const auto &P : InvertedIndex) - Bytes += P.second.bytes(); + Bytes += P.first.Data.size() + P.second.bytes(); return Bytes + BackingDataSize; }