Skip to content

Commit e1e19c7

Browse files
kirillbobyrevkirillbobyrev
committedSep 12, 2018
[clangd] Implement a Proof-of-Concept tool for symbol index exploration
Reviewed By: sammccall, ilya-biryukov Differential Revision: https://reviews.llvm.org/D51628 llvm-svn: 342025
1 parent 1ea7578 commit e1e19c7

File tree

3 files changed

+177
-0
lines changed

3 files changed

+177
-0
lines changed
 

‎clang-tools-extra/clangd/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,4 @@ if( LLVM_LIB_FUZZING_ENGINE OR LLVM_USE_SANITIZE_COVERAGE )
7474
endif()
7575
add_subdirectory(tool)
7676
add_subdirectory(global-symbol-builder)
77+
add_subdirectory(index/dex/dexp)

‎clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../)
2+
3+
set(LLVM_LINK_COMPONENTS
4+
LineEditor
5+
Support
6+
)
7+
8+
add_clang_executable(dexp
9+
Dexp.cpp
10+
)
11+
12+
target_link_libraries(dexp
13+
PRIVATE
14+
clangDaemon
15+
)

‎clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
//===--- Dexp.cpp - Dex EXPloration tool ------------------------*- C++ -*-===//
2+
//
3+
// The LLVM Compiler Infrastructure
4+
//
5+
// This file is distributed under the University of Illinois Open Source
6+
// License. See LICENSE.TXT for details.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
//
10+
// This file implements a simple interactive tool which can be used to manually
11+
// evaluate symbol search quality of Clangd index.
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
#include "../../../index/SymbolYAML.h"
16+
#include "../Dex.h"
17+
#include "llvm/ADT/SmallVector.h"
18+
#include "llvm/ADT/StringRef.h"
19+
#include "llvm/ADT/StringSwitch.h"
20+
#include "llvm/LineEditor/LineEditor.h"
21+
#include "llvm/Support/CommandLine.h"
22+
#include "llvm/Support/Signals.h"
23+
24+
using clang::clangd::FuzzyFindRequest;
25+
using clang::clangd::loadIndex;
26+
using clang::clangd::Symbol;
27+
using clang::clangd::SymbolIndex;
28+
using llvm::StringRef;
29+
30+
namespace {
31+
32+
llvm::cl::opt<std::string>
33+
SymbolCollection("symbol-collection-file",
34+
llvm::cl::desc("Path to the file with symbol collection"),
35+
llvm::cl::Positional, llvm::cl::Required);
36+
37+
static const std::string Overview = R"(
38+
This is an **experimental** interactive tool to process user-provided search
39+
queries over given symbol collection obtained via global-symbol-builder. The
40+
tool can be used to evaluate search quality of existing index implementations
41+
and manually construct non-trivial test cases.
42+
43+
Type use "help" request to get information about the details.
44+
)";
45+
46+
void reportTime(StringRef Name, llvm::function_ref<void()> F) {
47+
const auto TimerStart = std::chrono::high_resolution_clock::now();
48+
F();
49+
const auto TimerStop = std::chrono::high_resolution_clock::now();
50+
const auto Duration = std::chrono::duration_cast<std::chrono::milliseconds>(
51+
TimerStop - TimerStart);
52+
llvm::outs() << llvm::formatv("{0} took {1:ms+n}.\n", Name, Duration);
53+
}
54+
55+
void fuzzyFind(llvm::StringRef UnqualifiedName, const SymbolIndex &Index) {
56+
FuzzyFindRequest Request;
57+
Request.MaxCandidateCount = 10;
58+
Request.Query = UnqualifiedName;
59+
// FIXME(kbobyrev): Print symbol final scores to see the distribution.
60+
static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n";
61+
llvm::outs() << llvm::formatv(OutputFormat, "Rank", "Symbol ID",
62+
"Symbol Name");
63+
size_t Rank = 0;
64+
Index.fuzzyFind(Request, [&](const Symbol &Sym) {
65+
llvm::outs() << llvm::formatv(OutputFormat, Rank++, Sym.ID.str(), Sym.Name);
66+
});
67+
}
68+
69+
static const std::string HelpMessage = R"(dexp commands:
70+
71+
> find Name
72+
73+
Constructs fuzzy find request given unqualified symbol name and returns top 10
74+
symbols retrieved from index.
75+
76+
> lookup SymbolID
77+
78+
Retrieves symbol names given USR.
79+
)";
80+
81+
void help() { llvm::outs() << HelpMessage; }
82+
83+
void lookup(StringRef USR, const SymbolIndex &Index) {
84+
llvm::DenseSet<clang::clangd::SymbolID> IDs{clang::clangd::SymbolID{USR}};
85+
clang::clangd::LookupRequest Request{IDs};
86+
bool FoundSymbol = false;
87+
Index.lookup(Request, [&](const Symbol &Sym) {
88+
if (!FoundSymbol)
89+
FoundSymbol = true;
90+
llvm::outs() << SymbolToYAML(Sym);
91+
});
92+
if (!FoundSymbol)
93+
llvm::outs() << "not found\n";
94+
}
95+
96+
// FIXME(kbobyrev): Make this an actual REPL: probably use LLVM Command Line
97+
// library for parsing flags and arguments.
98+
// FIXME(kbobyrev): Ideas for commands:
99+
// * symbol lookup: print out symbol in YAML format given SymbolID
100+
// * find symbol references: print set of reference locations
101+
// * load/swap/reload index: this would make it possible to get rid of llvm::cl
102+
// usages in the tool driver and actually use llvm::cl library in the REPL.
103+
// * show posting list density histogram (our dump data somewhere so that user
104+
// could build one)
105+
// * show number of tokens of each kind
106+
// * print out tokens with the most dense posting lists
107+
// * print out tokens with least dense posting lists
108+
void dispatch(StringRef Request, const SymbolIndex &Index) {
109+
llvm::SmallVector<StringRef, 2> Arguments;
110+
Request.split(Arguments, ' ');
111+
if (Arguments.empty()) {
112+
llvm::outs() << "Request can not be empty.\n";
113+
help();
114+
return;
115+
}
116+
117+
if (Arguments.front() == "find") {
118+
if (Arguments.size() != 2) {
119+
llvm::outs() << "find request must specify unqualified symbol name.\n";
120+
return;
121+
}
122+
reportTime("fuzzy find request",
123+
[&]() { fuzzyFind(Arguments.back(), Index); });
124+
} else if (Arguments.front() == "lookup") {
125+
if (Arguments.size() != 2) {
126+
llvm::outs() << "lookup request must specify symbol ID .\n";
127+
return;
128+
}
129+
reportTime("lookup request", [&]() { lookup(Arguments.back(), Index); });
130+
} else if (Arguments.front() == "help") {
131+
help();
132+
} else {
133+
llvm::outs() << "Unknown command. Try 'help'\n";
134+
}
135+
}
136+
137+
} // namespace
138+
139+
int main(int argc, const char *argv[]) {
140+
llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
141+
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
142+
143+
std::unique_ptr<SymbolIndex> Index;
144+
reportTime("Dex build", [&]() {
145+
Index = loadIndex(SymbolCollection, /*URISchemes=*/{},
146+
/*UseDex=*/true);
147+
});
148+
149+
if (!Index) {
150+
llvm::outs()
151+
<< "ERROR: Please provide a valid path to symbol collection file.\n";
152+
return -1;
153+
}
154+
155+
llvm::LineEditor LE("dexp");
156+
157+
while (llvm::Optional<std::string> Request = LE.readLine())
158+
dispatch(Request.getValue(), *Index);
159+
160+
return 0;
161+
}

0 commit comments

Comments
 (0)