Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -74,3 +74,4 @@ endif() add_subdirectory(tool) add_subdirectory(global-symbol-builder) +add_subdirectory(index/dex/dexplorer) Index: clang-tools-extra/clangd/index/dex/dexplorer/CMakeLists.txt =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/dex/dexplorer/CMakeLists.txt @@ -0,0 +1,15 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../) + +set(LLVM_LINK_COMPONENTS + LineEditor + Support + ) + +add_clang_executable(dexplorer + DExplorer.cpp + ) + +target_link_libraries(dexplorer + PRIVATE + clangDaemon +) Index: clang-tools-extra/clangd/index/dex/dexplorer/DExplorer.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/dex/dexplorer/DExplorer.cpp @@ -0,0 +1,186 @@ +//===--- DExplorer.cpp - Dex Exploration tool -------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements a simple interactive tool which can be used to manually +// evaluate symbol search quality of Clangd index. +// +//===----------------------------------------------------------------------===// + +#include "../../../index/SymbolYAML.h" +#include "../Dex.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/YAMLTraits.h" +#include + +using clang::clangd::FuzzyFindRequest; +using clang::clangd::loadIndex; +using clang::clangd::Symbol; +using clang::clangd::SymbolIndex; +using llvm::StringRef; + +namespace { + +llvm::cl::opt + SymbolCollection("symbol-collection-file", + llvm::cl::desc("Path to the file with symbol collection"), + llvm::cl::Positional, llvm::cl::Required); + +static const std::string Overview = R"( +This is an **experimental** interactive tool to process user-provided search +queries over given symbol collection obtained via global-symbol-builder. The +tool can be used to evaluate search quality of existing index implementations +and manually construct non-trivial test cases. + +Type use "help" request to get information about the details. +)"; + +template std::string unitToString() { return ""; } + +template <> std::string unitToString() { + return "ms"; +} + +template <> std::string unitToString() { return "s"; } + +template +void reportTime(StringRef Name, Function F) { + const auto TimerStart = std::chrono::high_resolution_clock::now(); + F(); + const auto TimerStop = std::chrono::high_resolution_clock::now(); + llvm::outs() + << Name << " took " + << std::chrono::duration_cast(TimerStop - TimerStart).count() + << ' ' << unitToString() << ".\n"; +} + +void fuzzyFindRequest(const std::unique_ptr &Index, + const FuzzyFindRequest &Request) { + std::vector Symbols; + Index->fuzzyFind(Request, [&](const Symbol &S) { Symbols.push_back(S); }); + // FIXME(kbobyrev): Print symbol final scores to see the distribution. + llvm::outs() << "\nRetrieved Symbols:\n"; + llvm::outs() << "Symbol Name\n"; + for (const auto &Symbol : Symbols) + llvm::outs() << Symbol.Name << '\n'; + llvm::outs() << '\n'; +} + +static const std::string HelpMessage = R"( +DExplorer commands: + +> fuzzy-find Name + +Constructs fuzzy find request given unqualified symbol name and returns top 10 +symbols retrieved from index. + +> lookup-id SymbolID + +Retrieves symbol names given USR. + +> help + +Shows this message. + +Press Ctrl-D to exit. +)"; + +void helpRequest() { llvm::outs() << HelpMessage; } + +void lookupRequest(const std::unique_ptr &Index, + clang::clangd::LookupRequest &Request) { + std::vector Symbols; + Index->lookup(Request, [&](const Symbol &S) { Symbols.push_back(S); }); + // FIXME(kbobyrev): Print symbol final scores to see the distribution. + llvm::outs() << "\nRetrieved Symbols:\n"; + llvm::outs() << "Rank. Symbol Name | Symbol ID\n"; + for (size_t Rank = 0; Rank < Symbols.size(); ++Rank) + llvm::outs() << Rank << ". " << Symbols[Rank].Name << " | " + << Symbols[Rank].ID.str() << '\n'; + llvm::outs() << '\n'; +} + +// FIXME(kbobyrev): Make this an actual REPL: probably use LLVM Command Line +// library for parsing flags and arguments. +// FIXME(kbobyrev): Ideas for commands: +// * symbol lookup: print out symbol in YAML format given SymbolID +// * find symbol references: print set of reference locations +// * load/swap/reload index: this would make it possible to get rid of llvm::cl +// usages in the tool driver and actually use llvm::cl library in the REPL. +// * show posting list density histogram (our dump data somewhere so that user +// could build one) +// * show number of tokens of each kind +// * print out tokens with the most dense posting lists +// * print out tokens with least dense posting lists +void dispatchRequest(const std::unique_ptr &Index, + StringRef Request) { + llvm::SmallVector Arguments; + Request.split(Arguments, ' '); + if (Arguments.empty()) { + llvm::outs() << "ERROR: Request can not be empty.\n"; + helpRequest(); + return; + } + + if (Arguments.front() == "fuzzy-find") { + if (Arguments.size() != 2) { + llvm::outs() << "ERROR: fuzzy-find request must specify unqualified " + "symbol name.\n"; + helpRequest(); + return; + } + FuzzyFindRequest Req; + Req.MaxCandidateCount = 10; + Req.Query = Arguments.back(); + reportTime("fuzzy-find request", [&]() { fuzzyFindRequest(Index, Req); }); + } else if (Arguments.front() == "lookup") { + if (Arguments.size() != 2) { + llvm::outs() << "ERROR: lookup-id request must specify symbol ID .\n"; + helpRequest(); + return; + } + llvm::DenseSet IDs{ + clang::clangd::SymbolID{Arguments.back()}}; + clang::clangd::LookupRequest Req{IDs}; + lookupRequest(Index, Req); + } else { + helpRequest(); + } +} + +} // namespace + +int main(int argc, const char *argv[]) { + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + // FIXME(kbobyrev): Wrap time measurements into something like + // measureTime(Function, Arguments...). + std::unique_ptr Index; + reportTime, std::chrono::seconds>("Dex build", [&]() { + Index = loadIndex(SymbolCollection, /*URISchemes=*/{}, + /*UseDex=*/true); + }); + + if (!Index) { + llvm::outs() << "ERROR: Please provide a valid YAML symbol collection.\n"; + return -1; + } + + llvm::LineEditor LE("dexplorer"); + + while (llvm::Optional Line = LE.readLine()) + dispatchRequest(Index, Line.getValue()); + + return 0; +}