Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -79,6 +79,7 @@ const ASTContext &getASTContext() const; Preprocessor &getPreprocessor(); + std::shared_ptr getPreprocessorPtr(); const Preprocessor &getPreprocessor() const; /// This function returns all top-level decls, including those that come Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -316,6 +316,10 @@ Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); } +std::shared_ptr ParsedAST::getPreprocessorPtr() { + return Clang->getPreprocessorPtr(); +} + const Preprocessor &ParsedAST::getPreprocessor() const { return Clang->getPreprocessor(); } Index: clangd/CodeComplete.cpp =================================================================== --- clangd/CodeComplete.cpp +++ clangd/CodeComplete.cpp @@ -556,16 +556,19 @@ Item.kind = toCompletionItemKind(Sym.SymInfo.Kind); Item.label = Sym.Name; // FIXME(ioeric): support inserting/replacing scope qualifiers. - Item.insertText = Sym.Name; + // FIXME(ioeric): support snippets. + Item.insertText = Sym.CompletionPlainInsertText; Item.insertTextFormat = InsertTextFormat::PlainText; Item.filterText = Sym.Name; // FIXME(ioeric): sort symbols appropriately. Item.sortText = ""; - // FIXME(ioeric): use more symbol information (e.g. documentation, label) to - // populate the completion item. + if (Sym.Detail) { + Item.documentation = Sym.Detail->Documentation; + Item.detail = Sym.Detail->CompletionDetail; + } return Item; } Index: clangd/index/FileIndex.cpp =================================================================== --- clangd/index/FileIndex.cpp +++ clangd/index/FileIndex.cpp @@ -17,8 +17,10 @@ /// Retrieves namespace and class level symbols in \p Decls. std::unique_ptr indexAST(ASTContext &Ctx, + std::shared_ptr PP, llvm::ArrayRef Decls) { auto Collector = std::make_shared(); + Collector->setPreprocessor(std::move(PP)); index::IndexingOptions IndexOpts; IndexOpts.SystemSymbolFilter = index::IndexingOptions::SystemSymbolFilterKind::All; @@ -67,7 +69,8 @@ if (!AST) { FSymbols.update(Path, nullptr); } else { - auto Slab = indexAST(AST->getASTContext(), AST->getTopLevelDecls()); + auto Slab = indexAST(AST->getASTContext(), AST->getPreprocessorPtr(), + AST->getTopLevelDecls()); FSymbols.update(Path, std::move(Slab)); } auto Symbols = FSymbols.allSymbols(); Index: clangd/index/Index.h =================================================================== --- clangd/index/Index.h +++ clangd/index/Index.h @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringExtras.h" #include #include @@ -126,10 +127,39 @@ // (not a definition), which is usually declared in ".h" file. SymbolLocation CanonicalDeclaration; + /// A brief description of the symbol that can be displayed in the completion + /// candidate list. For example, "Foo(X x, Y y) const" is a labal for a + /// function. + llvm::StringRef CompletionLabel; + /// The piece of text that the user is expected to type to match the + /// code-completion string, typically a keyword or the name of a declarator or + /// macro. + llvm::StringRef CompletionFilterText; + /// What to insert when completing this symbol (plain text version). + llvm::StringRef CompletionPlainInsertText; + /// What to insert when completing this symbol (snippet version). This is + /// empty if it is the same as the plain insert text above. + llvm::StringRef CompletionSnippetInsertText; + + /// Optional symbol details that are not required to be set. For example, an + /// index fuzzy match can return a large number of symbol candidates, and it + /// is preferable to send only core symbol information in the batched results + /// and have clients resolve full symbol information for a specific candidate + /// if needed. + struct Details { + // Documentation including comment for the symbol declaration. + llvm::StringRef Documentation; + // This is what goes into the LSP detail field in a completion item. For + // example, the result type of a function. + llvm::StringRef CompletionDetail; + }; + + // Optional details of the symbol. + Details *Detail = nullptr; + // FIXME: add definition location of the symbol. // FIXME: add all occurrences support. // FIXME: add extra fields for index scoring signals. - // FIXME: add code completion information. }; // An immutable symbol container that stores a set of symbols. Index: clangd/index/Index.cpp =================================================================== --- clangd/index/Index.cpp +++ clangd/index/Index.cpp @@ -55,6 +55,23 @@ Intern(S.Name); Intern(S.Scope); Intern(S.CanonicalDeclaration.FilePath); + + Intern(S.CompletionLabel); + Intern(S.CompletionFilterText); + Intern(S.CompletionPlainInsertText); + Intern(S.CompletionSnippetInsertText); + + if (S.Detail) { + // Copy values of StringRefs into arena. + auto *Detail = Arena.Allocate(); + Detail->Documentation = S.Detail->Documentation; + Detail->CompletionDetail = S.Detail->CompletionDetail; + S.Detail = Detail; + + // Intern the actual strings. + Intern(S.Detail->Documentation); + Intern(S.Detail->CompletionDetail); + } } void SymbolSlab::Builder::insert(const Symbol &S) { Index: clangd/index/SymbolCollector.h =================================================================== --- clangd/index/SymbolCollector.h +++ clangd/index/SymbolCollector.h @@ -8,8 +8,11 @@ //===----------------------------------------------------------------------===// #include "Index.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Decl.h" #include "clang/Index/IndexDataConsumer.h" #include "clang/Index/IndexSymbol.h" +#include "clang/Sema/CodeCompleteConsumer.h" namespace clang { namespace clangd { @@ -21,7 +24,11 @@ // changed. class SymbolCollector : public index::IndexDataConsumer { public: - SymbolCollector() = default; + void initialize(ASTContext &Ctx) override; + + void setPreprocessor(std::shared_ptr PP) override { + this->PP = std::move(PP); + } bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, @@ -34,6 +41,10 @@ private: // All Symbols collected from the AST. SymbolSlab::Builder Symbols; + ASTContext *ASTCtx; + std::shared_ptr PP; + std::shared_ptr CompletionAllocator; + std::unique_ptr CompletionTUInfo; }; } // namespace clangd Index: clangd/index/SymbolCollector.cpp =================================================================== --- clangd/index/SymbolCollector.cpp +++ clangd/index/SymbolCollector.cpp @@ -8,8 +8,7 @@ //===----------------------------------------------------------------------===// #include "SymbolCollector.h" -#include "clang/AST/ASTContext.h" -#include "clang/AST/Decl.h" +#include "../CodeCompletionStrings.h" #include "clang/AST/DeclCXX.h" #include "clang/Basic/SourceManager.h" #include "clang/Index/IndexSymbol.h" @@ -56,7 +55,6 @@ return AbsolutePath.str(); } -// Split a qualified symbol name into scope and unqualified name, e.g. given // "a::b::c", return {"a::b", "c"}. Scope is empty if it doesn't exist. std::pair splitQualifiedName(llvm::StringRef QName) { @@ -69,6 +67,13 @@ } // namespace +void SymbolCollector::initialize(ASTContext &Ctx) { + ASTCtx = &Ctx; + CompletionAllocator = std::make_shared(); + CompletionTUInfo = + llvm::make_unique(CompletionAllocator); +} + // Always return true to continue indexing. bool SymbolCollector::handleDeclOccurence( const Decl *D, index::SymbolRoleSet Roles, @@ -79,6 +84,8 @@ Roles & static_cast(index::SymbolRole::Definition))) return true; + assert(CompletionAllocator && CompletionTUInfo); + if (const NamedDecl *ND = llvm::dyn_cast(D)) { // FIXME: figure out a way to handle internal linkage symbols (e.g. static // variables, function) defined in the .cc files. Also we skip the symbols @@ -113,6 +120,35 @@ S.Name = ScopeAndName.second; S.SymInfo = index::getSymbolInfo(D); S.CanonicalDeclaration = Location; + + // Add completion info. + assert(ASTCtx && PP.get() && "ASTContext and Preprocessor must be set."); + CodeCompletionResult SymbolCompletion(ND, 0); + const auto *CCS = SymbolCompletion.CreateCodeCompletionString( + *ASTCtx, *PP, CodeCompletionContext::CCC_Name, *CompletionAllocator, + *CompletionTUInfo, + /*IncludeBriefComments*/ true); + std::string Label; + std::string SnippetInsertText; + std::string IgnoredLabel; + std::string PlainInsertText; + getLabelAndInsertText(*CCS, &Label, &SnippetInsertText, + /*EnableSnippets=*/true); + getLabelAndInsertText(*CCS, &IgnoredLabel, &PlainInsertText, + /*EnableSnippets=*/false); + std::string FilterText = getFilterText(*CCS); + std::string Documentation = getDocumentation(*CCS); + std::string CompletionDetail = getDetail(*CCS); + + S.CompletionFilterText = FilterText; + S.CompletionLabel = Label; + S.CompletionPlainInsertText = PlainInsertText; + S.CompletionSnippetInsertText = SnippetInsertText; + Symbol::Details Detail; + Detail.Documentation = Documentation; + Detail.CompletionDetail = CompletionDetail; + S.Detail = &Detail; + Symbols.insert(S); } Index: clangd/index/SymbolYAML.cpp =================================================================== --- clangd/index/SymbolYAML.cpp +++ clangd/index/SymbolYAML.cpp @@ -9,6 +9,7 @@ #include "SymbolYAML.h" #include "Index.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/Errc.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/YAMLTraits.h" @@ -59,15 +60,39 @@ } }; -template<> struct MappingTraits { +template <> +struct MappingTraits { + static void mapping(IO &io, Symbol::Details *&Detail) { + if (!io.outputting()) { + assert(io.getContext() && "Expecting an arena (as context) to allocate " + "data for new symbols."); + Detail = static_cast(io.getContext()) + ->Allocate(); + } else if (!Detail) { + // Detail is optional in outputting. + return; + } + assert(Detail); + io.mapOptional("Documentation", Detail->Documentation); + io.mapOptional("CompletionDetail", Detail->CompletionDetail); + } +}; + +template <> struct MappingTraits { static void mapping(IO &IO, Symbol &Sym) { - MappingNormalization NSymbolID( - IO, Sym.ID); + MappingNormalization NSymbolID(IO, Sym.ID); IO.mapRequired("ID", NSymbolID->HexString); IO.mapRequired("Name", Sym.Name); IO.mapRequired("Scope", Sym.Scope); IO.mapRequired("SymInfo", Sym.SymInfo); IO.mapRequired("CanonicalDeclaration", Sym.CanonicalDeclaration); + IO.mapRequired("CompletionLabel", Sym.CompletionLabel); + IO.mapRequired("CompletionFilterText", Sym.CompletionFilterText); + IO.mapRequired("CompletionPlainInsertText", Sym.CompletionPlainInsertText); + + IO.mapOptional("CompletionSnippetInsertText", + Sym.CompletionSnippetInsertText); + IO.mapOptional("Detail", Sym.Detail); } }; @@ -124,11 +149,14 @@ namespace clangd { SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent) { + // Store data of pointer fields (excl. `StringRef`) like `Detail`. + llvm::BumpPtrAllocator Arena; + llvm::yaml::Input Yin(YAMLContent, &Arena); std::vector S; - llvm::yaml::Input Yin(YAMLContent); Yin >> S; + SymbolSlab::Builder Syms; - for (auto& Sym : S) + for (auto &Sym : S) Syms.insert(Sym); return std::move(Syms).build(); } @@ -138,7 +166,7 @@ llvm::raw_string_ostream OS(Str); llvm::yaml::Output Yout(OS); for (Symbol S : Symbols) // copy: Yout<< requires mutability. - Yout<< S; + Yout << S; return OS.str(); } Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -68,6 +68,8 @@ MATCHER_P(Labeled, Label, "") { return arg.label == Label; } MATCHER_P(Kind, K, "") { return arg.kind == K; } MATCHER_P(Filter, F, "") { return arg.filterText == F; } +MATCHER_P(Doc, D, "") { return arg.documentation == D; } +MATCHER_P(Detail, D, "") { return arg.detail == D; } MATCHER_P(PlainText, Text, "") { return arg.insertTextFormat == clangd::InsertTextFormat::PlainText && arg.insertText == Text; @@ -472,6 +474,7 @@ Sym.Name = QName.substr(Pos + 2); Sym.Scope = QName.substr(0, Pos); } + Sym.CompletionPlainInsertText = Sym.Name; Sym.SymInfo.Kind = Pair.second; Slab.insert(Sym); } @@ -561,13 +564,17 @@ Server .addDocument(Context::empty(), getVirtualTestFilePath("foo.cpp"), R"cpp( - namespace ns { class XYZ {}; void foo() {} } + namespace ns { class XYZ {}; void foo(int x) {} } )cpp") .wait(); auto File = getVirtualTestFilePath("bar.cpp"); Annotations Test(R"cpp( - namespace ns { class XXX {}; void fooooo() {} } + namespace ns { + class XXX {}; + /// Doooc + void fooooo() {} + } void f() { ns::^ } )cpp"); Server.addDocument(Context::empty(), File, Test.code()).wait(); @@ -580,7 +587,9 @@ EXPECT_THAT(Results.items, Has("XYZ", CompletionItemKind::Class)); EXPECT_THAT(Results.items, Has("foo", CompletionItemKind::Function)); EXPECT_THAT(Results.items, Has("XXX", CompletionItemKind::Class)); - EXPECT_THAT(Results.items, Has("fooooo", CompletionItemKind::Function)); + EXPECT_THAT(Results.items, Contains(AllOf(Named("fooooo"), Filter("fooooo"), + Kind(CompletionItemKind::Function), + Doc("Doooc"), Detail("void")))); } } // namespace Index: unittests/clangd/SymbolCollectorTests.cpp =================================================================== --- unittests/clangd/SymbolCollectorTests.cpp +++ unittests/clangd/SymbolCollectorTests.cpp @@ -9,7 +9,6 @@ #include "index/SymbolCollector.h" #include "index/SymbolYAML.h" -#include "clang/Index/IndexingAction.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/FileSystemOptions.h" #include "clang/Basic/VirtualFileSystem.h" @@ -26,12 +25,24 @@ #include #include +using testing::AllOf; using testing::Eq; using testing::Field; +using testing::Not; using testing::UnorderedElementsAre; using testing::UnorderedElementsAreArray; // GMock helpers for matching Symbol. +MATCHER_P(Labeled, Label, "") { return arg.CompletionLabel == Label; } +MATCHER(HasDetail, "") { return arg.Detail; } +MATCHER_P(Detail, D, "") { + return arg.Detail && arg.Detail->CompletionDetail == D; +} +MATCHER_P(Doc, D, "") { return arg.Detail && arg.Detail->Documentation == D; } +MATCHER_P(Plain, Text, "") { return arg.CompletionPlainInsertText == Text; } +MATCHER_P(Snippet, S, "") { + return arg.CompletionSnippetInsertText == S; +} MATCHER_P(QName, Name, "") { return (arg.Scope + (arg.Scope.empty() ? "" : "::") + arg.Name).str() == Name; } @@ -147,6 +158,38 @@ QName("foo::baz")})); } +TEST_F(SymbolCollectorTest, SymbolWithDocumentation) { + const std::string Main = R"( + namespace nx { + /// Foo comment. + int ff(int x, double y) { return 0; } + } + )"; + runSymbolCollector(/*Header=*/"", Main); + EXPECT_THAT(Symbols, + UnorderedElementsAre(QName("nx"), + AllOf(QName("nx::ff"), + Labeled("ff(int x, double y)"), + Detail("int"), Doc("Foo comment.")))); +} + +TEST_F(SymbolCollectorTest, PlainAndSnippet) { + const std::string Main = R"( + namespace nx { + void f() {} + int ff(int x, double y) { return 0; } + } + )"; + runSymbolCollector(/*Header=*/"", Main); + EXPECT_THAT( + Symbols, + UnorderedElementsAre( + QName("nx"), + AllOf(QName("nx::f"), Labeled("f()"), Plain("f"), Snippet("f()")), + AllOf(QName("nx::ff"), Labeled("ff(int x, double y)"), Plain("ff"), + Snippet("ff(${1:int x}, ${2:double y})")))); +} + TEST_F(SymbolCollectorTest, YAMLConversions) { const std::string YAML1 = R"( --- @@ -160,6 +203,12 @@ StartOffset: 0 EndOffset: 1 FilePath: /path/foo.h +CompletionLabel: 'Foo1-label' +CompletionFilterText: 'filter' +CompletionPlainInsertText: 'plain' +Detail: + Documentation: 'Foo doc' + CompletionDetail: 'int' ... )"; const std::string YAML2 = R"( @@ -174,15 +223,21 @@ StartOffset: 10 EndOffset: 12 FilePath: /path/foo.h +CompletionLabel: 'Foo2-label' +CompletionFilterText: 'filter' +CompletionPlainInsertText: 'plain' +CompletionSnippetInsertText: 'snippet' ... )"; auto Symbols1 = SymbolFromYAML(YAML1); - EXPECT_THAT(Symbols1, - UnorderedElementsAre(QName("clang::Foo1"))); + EXPECT_THAT(Symbols1, UnorderedElementsAre( + AllOf(QName("clang::Foo1"), Labeled("Foo1-label"), + Doc("Foo doc"), Detail("int")))); auto Symbols2 = SymbolFromYAML(YAML2); - EXPECT_THAT(Symbols2, - UnorderedElementsAre(QName("clang::Foo2"))); + EXPECT_THAT(Symbols2, UnorderedElementsAre(AllOf(QName("clang::Foo2"), + Labeled("Foo2-label"), + Not(HasDetail())))); std::string ConcatenatedYAML = SymbolsToYAML(Symbols1) + SymbolsToYAML(Symbols2);