Skip to content

Commit ba28e9a

Browse files
hokeinhokein
committedJan 10, 2018
[clangd] Add static index for the global code completion.
Summary: Use the YAML-format symbols (generated by the global-symbol-builder tool) to do the global code completion. It is **experimental** only , but it allows us to experience global code completion on a relatively small project. Tested with LLVM project. Reviewers: sammccall, ioeric Reviewed By: sammccall, ioeric Subscribers: klimek, ilya-biryukov, cfe-commits Differential Revision: https://reviews.llvm.org/D41668 llvm-svn: 322191
1 parent 7da8475 commit ba28e9a

File tree

10 files changed

+130
-30
lines changed

10 files changed

+130
-30
lines changed
 

‎clang-tools-extra/clangd/ClangdLSPServer.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,10 +283,12 @@ ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount,
283283
const clangd::CodeCompleteOptions &CCOpts,
284284
llvm::Optional<StringRef> ResourceDir,
285285
llvm::Optional<Path> CompileCommandsDir,
286-
bool BuildDynamicSymbolIndex)
286+
bool BuildDynamicSymbolIndex,
287+
SymbolIndex *StaticIdx)
287288
: Out(Out), CDB(std::move(CompileCommandsDir)), CCOpts(CCOpts),
288289
Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount,
289-
StorePreamblesInMemory, BuildDynamicSymbolIndex, ResourceDir) {}
290+
StorePreamblesInMemory, BuildDynamicSymbolIndex, StaticIdx,
291+
ResourceDir) {}
290292

291293
bool ClangdLSPServer::run(std::istream &In) {
292294
assert(!IsDone && "Run was called before");

‎clang-tools-extra/clangd/ClangdLSPServer.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ namespace clang {
2222
namespace clangd {
2323

2424
class JSONOutput;
25+
class SymbolIndex;
2526

2627
/// This class provides implementation of an LSP server, glueing the JSON
2728
/// dispatch and ClangdServer together.
@@ -35,7 +36,8 @@ class ClangdLSPServer : private DiagnosticsConsumer, private ProtocolCallbacks {
3536
const clangd::CodeCompleteOptions &CCOpts,
3637
llvm::Optional<StringRef> ResourceDir,
3738
llvm::Optional<Path> CompileCommandsDir,
38-
bool BuildDynamicSymbolIndex);
39+
bool BuildDynamicSymbolIndex,
40+
SymbolIndex *StaticIdx = nullptr);
3941

4042
/// Run LSP server loop, receiving input for it from \p In. \p In must be
4143
/// opened in binary mode. Output will be written using Out variable passed to

‎clang-tools-extra/clangd/ClangdServer.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,10 +134,11 @@ ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB,
134134
FileSystemProvider &FSProvider,
135135
unsigned AsyncThreadsCount,
136136
bool StorePreamblesInMemory,
137-
bool BuildDynamicSymbolIndex,
137+
bool BuildDynamicSymbolIndex, SymbolIndex *StaticIdx,
138138
llvm::Optional<StringRef> ResourceDir)
139139
: CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider),
140140
FileIdx(BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
141+
StaticIdx(StaticIdx),
141142
// Pass a callback into `Units` to extract symbols from a newly parsed
142143
// file and rebuild the file index synchronously each time an AST is
143144
// parsed.
@@ -251,6 +252,8 @@ void ClangdServer::codeComplete(
251252
auto CodeCompleteOpts = Opts;
252253
if (FileIdx)
253254
CodeCompleteOpts.Index = FileIdx.get();
255+
if (StaticIdx)
256+
CodeCompleteOpts.StaticIndex = StaticIdx;
254257

255258
// Copy File, as it is a PathRef that will go out of scope before Task is
256259
// executed.

‎clang-tools-extra/clangd/ClangdServer.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,15 @@ class ClangdServer {
200200
/// If \p BuildDynamicSymbolIndex is true, ClangdServer builds a dynamic
201201
/// in-memory index for symbols in all opened files and uses the index to
202202
/// augment code completion results.
203+
///
204+
/// If \p StaticIdx is set, ClangdServer uses the index for global code
205+
/// completion.
203206
ClangdServer(GlobalCompilationDatabase &CDB,
204207
DiagnosticsConsumer &DiagConsumer,
205208
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
206209
bool StorePreamblesInMemory,
207210
bool BuildDynamicSymbolIndex = false,
211+
SymbolIndex *StaticIdx = nullptr,
208212
llvm::Optional<StringRef> ResourceDir = llvm::None);
209213

210214
/// Set the root path of the workspace.
@@ -338,6 +342,11 @@ class ClangdServer {
338342
DraftStore DraftMgr;
339343
/// If set, this manages index for symbols in opened files.
340344
std::unique_ptr<FileIndex> FileIdx;
345+
/// If set, this provides static index for project-wide global symbols.
346+
/// clangd global code completion result will come from the static index and
347+
/// the `FileIdx` above.
348+
/// No owned, the life time is managed by clangd embedders.
349+
SymbolIndex *StaticIdx;
341350
CppFileCollection Units;
342351
std::string ResourceDir;
343352
// If set, this represents the workspace path.

‎clang-tools-extra/clangd/CodeComplete.cpp

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "clang/Frontend/FrontendActions.h"
2424
#include "clang/Sema/CodeCompleteConsumer.h"
2525
#include "clang/Sema/Sema.h"
26+
#include "llvm/Support/Format.h"
2627
#include <queue>
2728

2829
namespace clang {
@@ -558,10 +559,27 @@ bool invokeCodeComplete(const Context &Ctx,
558559
}
559560

560561
CompletionItem indexCompletionItem(const Symbol &Sym, llvm::StringRef Filter,
561-
const SpecifiedScope &SSInfo) {
562+
const SpecifiedScope &SSInfo,
563+
llvm::StringRef DebuggingLabel = "") {
562564
CompletionItem Item;
563565
Item.kind = toCompletionItemKind(Sym.SymInfo.Kind);
564-
Item.label = Sym.Name;
566+
// Add DebuggingLabel to the completion results if DebuggingLabel is not
567+
// empty.
568+
//
569+
// For symbols from static index, there are prefix "[G]" in the
570+
// results (which is used for debugging purpose).
571+
// So completion list will be like:
572+
// clang::symbol_from_dynamic_index
573+
// [G]clang::symbol_from_static_index
574+
//
575+
// FIXME: Find out a better way to show the index source.
576+
if (!DebuggingLabel.empty()) {
577+
llvm::raw_string_ostream Label(Item.label);
578+
Label << llvm::format("[%s]%s", DebuggingLabel.str().c_str(),
579+
Sym.Name.str().c_str());
580+
} else {
581+
Item.label = Sym.Name;
582+
}
565583
// FIXME(ioeric): support inserting/replacing scope qualifiers.
566584

567585
// FIXME(ioeric): support snippets.
@@ -582,16 +600,18 @@ CompletionItem indexCompletionItem(const Symbol &Sym, llvm::StringRef Filter,
582600

583601
void completeWithIndex(const Context &Ctx, const SymbolIndex &Index,
584602
llvm::StringRef Code, const SpecifiedScope &SSInfo,
585-
llvm::StringRef Filter, CompletionList *Items) {
603+
llvm::StringRef Filter, CompletionList *Items,
604+
llvm::StringRef DebuggingLabel = "") {
586605
FuzzyFindRequest Req;
587606
Req.Query = Filter;
588607
// FIXME(ioeric): add more possible scopes based on using namespaces and
589608
// containing namespaces.
590609
StringRef Scope = SSInfo.Resolved.empty() ? SSInfo.Written : SSInfo.Resolved;
591610
Req.Scopes = {Scope.trim(':').str()};
592611

593-
Items->isIncomplete = !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) {
594-
Items->items.push_back(indexCompletionItem(Sym, Filter, SSInfo));
612+
Items->isIncomplete |= !Index.fuzzyFind(Ctx, Req, [&](const Symbol &Sym) {
613+
Items->items.push_back(
614+
indexCompletionItem(Sym, Filter, SSInfo, DebuggingLabel));
595615
});
596616
}
597617

@@ -644,13 +664,19 @@ CompletionList codeComplete(const Context &Ctx, PathRef FileName,
644664
invokeCodeComplete(Ctx, std::move(Consumer), Opts.getClangCompleteOpts(),
645665
FileName, Command, Preamble, Contents, Pos, std::move(VFS),
646666
std::move(PCHs));
647-
if (Opts.Index && CompletedName.SSInfo) {
648-
if (!Results.items.empty())
649-
log(Ctx, "WARNING: Got completion results from sema for completion on "
650-
"qualified ID while symbol index is provided.");
651-
Results.items.clear();
652-
completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo,
653-
CompletedName.Filter, &Results);
667+
668+
// Got scope specifier (ns::f^) for code completion from sema, try to query
669+
// global symbols from indexes.
670+
if (CompletedName.SSInfo) {
671+
// FIXME: figure out a good algorithm to merge symbols from different
672+
// sources (dynamic index, static index, AST symbols from clang's completion
673+
// engine).
674+
if (Opts.Index)
675+
completeWithIndex(Ctx, *Opts.Index, Contents, *CompletedName.SSInfo,
676+
CompletedName.Filter, &Results);
677+
if (Opts.StaticIndex)
678+
completeWithIndex(Ctx, *Opts.StaticIndex, Contents, *CompletedName.SSInfo,
679+
CompletedName.Filter, &Results, /*DebuggingLabel=*/"G");
654680
}
655681
return Results;
656682
}

‎clang-tools-extra/clangd/CodeComplete.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ struct CodeCompleteOptions {
6767
/// FIXME(ioeric): we might want a better way to pass the index around inside
6868
/// clangd.
6969
const SymbolIndex *Index = nullptr;
70+
71+
// Populated internally by clangd, do not set.
72+
/// Static index for project-wide global symbols.
73+
const SymbolIndex *StaticIndex = nullptr;
7074
};
7175

7276
/// Get code completions at a specified \p Pos in \p FileName.

‎clang-tools-extra/clangd/index/MemIndex.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,21 @@ bool MemIndex::fuzzyFind(
5353
return true;
5454
}
5555

56+
std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
57+
struct Snapshot {
58+
SymbolSlab Slab;
59+
std::vector<const Symbol *> Pointers;
60+
};
61+
auto Snap = std::make_shared<Snapshot>();
62+
Snap->Slab = std::move(Slab);
63+
for (auto &Sym : Snap->Slab)
64+
Snap->Pointers.push_back(&Sym);
65+
auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
66+
&Snap->Pointers);
67+
auto MemIdx = llvm::make_unique<MemIndex>();
68+
MemIdx->build(std::move(S));
69+
return std::move(MemIdx);
70+
}
71+
5672
} // namespace clangd
5773
} // namespace clang

‎clang-tools-extra/clangd/index/MemIndex.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ class MemIndex : public SymbolIndex {
2424
/// accessible as long as `Symbols` is kept alive.
2525
void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
2626

27+
/// \brief Build index from a symbol slab.
28+
static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab);
29+
2730
bool
2831
fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req,
2932
llvm::function_ref<void(const Symbol &)> Callback) const override;

‎clang-tools-extra/clangd/tool/ClangdMain.cpp

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "JSONRPCDispatcher.h"
1212
#include "Path.h"
1313
#include "Trace.h"
14+
#include "index/SymbolYAML.h"
1415
#include "llvm/Support/CommandLine.h"
1516
#include "llvm/Support/FileSystem.h"
1617
#include "llvm/Support/Path.h"
@@ -26,7 +27,24 @@ using namespace clang::clangd;
2627

2728
namespace {
2829
enum class PCHStorageFlag { Disk, Memory };
30+
31+
// Build an in-memory static index for global symbols from a YAML-format file.
32+
// The size of global symbols should be relatively small, so that all symbols
33+
// can be managed in memory.
34+
std::unique_ptr<SymbolIndex> BuildStaticIndex(llvm::StringRef YamlSymbolFile) {
35+
auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
36+
if (!Buffer) {
37+
llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
38+
return nullptr;
39+
}
40+
auto Slab = SymbolFromYAML(Buffer.get()->getBuffer());
41+
SymbolSlab::Builder SymsBuilder;
42+
for (auto Sym : Slab)
43+
SymsBuilder.insert(Sym);
44+
45+
return MemIndex::build(std::move(SymsBuilder).build());
2946
}
47+
} // namespace
3048

3149
static llvm::cl::opt<Path> CompileCommandsDir(
3250
"compile-commands-dir",
@@ -97,6 +115,15 @@ static llvm::cl::opt<bool> EnableIndexBasedCompletion(
97115
"use index built from symbols in opened files"),
98116
llvm::cl::init(false), llvm::cl::Hidden);
99117

118+
static llvm::cl::opt<Path> YamlSymbolFile(
119+
"yaml-symbol-file",
120+
llvm::cl::desc(
121+
"YAML-format global symbol file to build the static index. Clangd will "
122+
"use the static index for global code completion.\n"
123+
"WARNING: This option is experimental only, and will be removed "
124+
"eventually. Don't rely on it."),
125+
llvm::cl::init(""), llvm::cl::Hidden);
126+
100127
int main(int argc, char *argv[]) {
101128
llvm::cl::ParseCommandLineOptions(argc, argv, "clangd");
102129

@@ -182,13 +209,16 @@ int main(int argc, char *argv[]) {
182209
// Change stdin to binary to not lose \r\n on windows.
183210
llvm::sys::ChangeStdinToBinary();
184211

212+
std::unique_ptr<SymbolIndex> StaticIdx;
213+
if (EnableIndexBasedCompletion && !YamlSymbolFile.empty())
214+
StaticIdx = BuildStaticIndex(YamlSymbolFile);
185215
clangd::CodeCompleteOptions CCOpts;
186216
CCOpts.EnableSnippets = EnableSnippets;
187217
CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
188218
// Initialize and run ClangdLSPServer.
189219
ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory,
190220
CCOpts, ResourceDirRef, CompileCommandsDirPath,
191-
EnableIndexBasedCompletion);
221+
EnableIndexBasedCompletion, StaticIdx.get());
192222
constexpr int NoShutdownRequestErrorCode = 1;
193223
llvm::set_thread_name("clangd.main");
194224
return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;

‎clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -455,12 +455,6 @@ TEST(SignatureHelpTest, ActiveArg) {
455455

456456
std::unique_ptr<SymbolIndex> simpleIndexFromSymbols(
457457
std::vector<std::pair<std::string, index::SymbolKind>> Symbols) {
458-
auto I = llvm::make_unique<MemIndex>();
459-
struct Snapshot {
460-
SymbolSlab Slab;
461-
std::vector<const Symbol *> Pointers;
462-
};
463-
auto Snap = std::make_shared<Snapshot>();
464458
SymbolSlab::Builder Slab;
465459
for (const auto &Pair : Symbols) {
466460
Symbol Sym;
@@ -478,13 +472,7 @@ std::unique_ptr<SymbolIndex> simpleIndexFromSymbols(
478472
Sym.SymInfo.Kind = Pair.second;
479473
Slab.insert(Sym);
480474
}
481-
Snap->Slab = std::move(Slab).build();
482-
for (auto &Iter : Snap->Slab)
483-
Snap->Pointers.push_back(&Iter);
484-
auto S = std::shared_ptr<std::vector<const Symbol *>>(std::move(Snap),
485-
&Snap->Pointers);
486-
I->build(std::move(S));
487-
return std::move(I);
475+
return MemIndex::build(std::move(Slab).build());
488476
}
489477

490478
TEST(CompletionTest, NoIndex) {
@@ -499,6 +487,23 @@ TEST(CompletionTest, NoIndex) {
499487
EXPECT_THAT(Results.items, Has("No"));
500488
}
501489

490+
TEST(CompletionTest, StaticAndDynamicIndex) {
491+
clangd::CodeCompleteOptions Opts;
492+
auto StaticIdx =
493+
simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class}});
494+
Opts.StaticIndex = StaticIdx.get();
495+
auto DynamicIdx =
496+
simpleIndexFromSymbols({{"ns::foo", index::SymbolKind::Function}});
497+
Opts.Index = DynamicIdx.get();
498+
499+
auto Results = completions(R"cpp(
500+
void f() { ::ns::^ }
501+
)cpp",
502+
Opts);
503+
EXPECT_THAT(Results.items, Contains(Labeled("[G]XYZ")));
504+
EXPECT_THAT(Results.items, Contains(Labeled("foo")));
505+
}
506+
502507
TEST(CompletionTest, SimpleIndexBased) {
503508
clangd::CodeCompleteOptions Opts;
504509
auto I = simpleIndexFromSymbols({{"ns::XYZ", index::SymbolKind::Class},

0 commit comments

Comments
 (0)
Please sign in to comment.