Index: clangd/CMakeLists.txt =================================================================== --- clangd/CMakeLists.txt +++ clangd/CMakeLists.txt @@ -19,6 +19,7 @@ Protocol.cpp ProtocolHandlers.cpp Trace.cpp + index/FileIndex.cpp index/Index.cpp index/SymbolCollector.cpp Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -34,7 +34,8 @@ bool StorePreamblesInMemory, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, - llvm::Optional CompileCommandsDir); + llvm::Optional CompileCommandsDir, + bool BuildDynamicSymbolIndex); /// Run LSP server loop, receiving input for it from \p In. \p In must be /// opened in binary mode. Output will be written using Out variable passed to Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -278,11 +278,13 @@ bool StorePreamblesInMemory, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, - llvm::Optional CompileCommandsDir) + llvm::Optional CompileCommandsDir, + bool BuildDynamicSymbolIndex) : Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)), - CCOpts(CCOpts), Server(CDB, /*DiagConsumer=*/*this, FSProvider, - AsyncThreadsCount, StorePreamblesInMemory, - /*Logger=*/Out, ResourceDir) {} + CCOpts(CCOpts), + Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, + StorePreamblesInMemory, + /*Logger=*/Out, BuildDynamicSymbolIndex, ResourceDir) {} bool ClangdLSPServer::run(std::istream &In) { assert(!IsDone && "Run was called before"); Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -10,20 +10,19 @@ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDSERVER_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDSERVER_H +#include "ClangdUnit.h" #include "ClangdUnitStore.h" +#include "CodeComplete.h" #include "DraftStore.h" +#include "Function.h" #include "GlobalCompilationDatabase.h" +#include "Protocol.h" +#include "index/Index.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" - -#include "ClangdUnit.h" -#include "CodeComplete.h" -#include "Function.h" -#include "Protocol.h" - #include #include #include @@ -207,10 +206,15 @@ /// clangd are stored in-memory or on disk. /// /// Various messages are logged using \p Logger. + /// + /// If \p BuildDynamicSymbolIndex is true, clangd will build a dynamic symbol + /// index for symbols in opened files. Clangd will use symbol information in + /// index to augment the code completion results. ClangdServer(GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, clangd::Logger &Logger, + bool BuildDynamicSymbolIndex = false, llvm::Optional ResourceDir = llvm::None); /// Set the root path of the workspace. @@ -340,7 +344,11 @@ DiagnosticsConsumer &DiagConsumer; FileSystemProvider &FSProvider; DraftStore DraftMgr; + + std::unique_ptr DynamicIndex; + std::unique_ptr FileSyms; CppFileCollection Units; + std::string ResourceDir; // If set, this represents the workspace path. llvm::Optional RootPath; Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -8,6 +8,7 @@ //===-------------------------------------------------------------------===// #include "ClangdServer.h" +#include "index/FileIndex.h" #include "clang/Format/Format.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" @@ -164,9 +165,15 @@ FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, clangd::Logger &Logger, + bool BuildDynamicSymbolIndex, llvm::Optional ResourceDir) : Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), + DynamicIndex(BuildDynamicSymbolIndex ? new FileSymbolsMemIndex() + : nullptr), + FileSyms(DynamicIndex.get() ? new FileSymbols(DynamicIndex.get()) + : nullptr), + Units(FileSyms.get()), ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()), PCHs(std::make_shared()), StorePreamblesInMemory(StorePreamblesInMemory), @@ -183,8 +190,9 @@ DocVersion Version = DraftMgr.updateDraft(File, Contents); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - std::shared_ptr Resources = Units.getOrCreateFile( - File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); + std::shared_ptr Resources = + Units.getOrCreateFile(File, ResourceDir, CDB, StorePreamblesInMemory, + PCHs, Logger, FileSyms.get()); return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()}, std::move(Resources), std::move(TaggedFS)); } @@ -279,7 +287,7 @@ CompletionList Result = clangd::codeComplete( File, Resources->getCompileCommand(), Preamble ? &Preamble->Preamble : nullptr, Contents, Pos, - TaggedFS.Value, PCHs, CodeCompleteOpts, Logger); + TaggedFS.Value, PCHs, CodeCompleteOpts, Logger, DynamicIndex.get()); Callback(make_tagged(std::move(Result), std::move(TaggedFS.Tag))); }; Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -13,6 +13,7 @@ #include "Function.h" #include "Path.h" #include "Protocol.h" +#include "index/FileIndex.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Frontend/PrecompiledPreamble.h" #include "clang/Serialization/ASTBitCodes.h" @@ -81,6 +82,7 @@ Preprocessor &getPreprocessor(); const Preprocessor &getPreprocessor() const; + std::shared_ptr getPreprocessorPtr(); /// This function returns all top-level decls, including those that come /// from Preamble. Decls, coming from Preamble, have to be deserialized, so @@ -146,12 +148,14 @@ static std::shared_ptr Create(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs, clangd::Logger &Logger); + std::shared_ptr PCHs, clangd::Logger &Logger, + FileSymbols *FileSyms); private: CppFile(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, - std::shared_ptr PCHs, clangd::Logger &Logger); + std::shared_ptr PCHs, clangd::Logger &Logger, + FileSymbols *FileSyms); public: CppFile(CppFile const &) = delete; @@ -254,6 +258,8 @@ std::shared_ptr PCHs; /// Used for logging various messages. clangd::Logger &Logger; + + FileSymbols *FileSyms; }; /// Get the beginning SourceLocation at a specified \p Pos. Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -12,6 +12,7 @@ #include "Compiler.h" #include "Logger.h" #include "Trace.h" +#include "index/SymbolCollector.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" @@ -426,6 +427,23 @@ } }; +/// Retrieve all namespace level symbols from an AST. +std::unique_ptr +indexAST(ASTContext &Ctx, std::shared_ptr PP, + llvm::ArrayRef TopLevelDecls) { + auto Collector = std::make_shared(); + Collector->setPreprocessor(std::move(PP)); + index::IndexingOptions IndexOpts; + IndexOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::All; + IndexOpts.IndexFunctionLocals = false; + + index::indexTopLevelDecls(Ctx, TopLevelDecls, Collector, IndexOpts); + auto Symbols = llvm::make_unique(); + *Symbols = Collector->takeSymbols(); + return Symbols; +} + } // namespace llvm::Optional @@ -571,6 +589,10 @@ Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); } +std::shared_ptr ParsedAST::getPreprocessorPtr() { + return Clang->getPreprocessorPtr(); +} + const Preprocessor &ParsedAST::getPreprocessor() const { return Clang->getPreprocessor(); } @@ -613,19 +635,20 @@ CppFile::Create(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, std::shared_ptr PCHs, - clangd::Logger &Logger) { - return std::shared_ptr(new CppFile(FileName, std::move(Command), - StorePreamblesInMemory, - std::move(PCHs), Logger)); + clangd::Logger &Logger, FileSymbols *FileSyms) { + return std::shared_ptr( + new CppFile(FileName, std::move(Command), StorePreamblesInMemory, + std::move(PCHs), Logger, FileSyms)); } CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command, bool StorePreamblesInMemory, std::shared_ptr PCHs, - clangd::Logger &Logger) + clangd::Logger &Logger, FileSymbols *FileSyms) : FileName(FileName), Command(std::move(Command)), StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0), - RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) { + RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger), + FileSyms(FileSyms) { Logger.log("Opened file " + FileName + " with command [" + this->Command.Directory + "] " + llvm::join(this->Command.CommandLine, " ")); @@ -828,6 +851,12 @@ if (NewAST) { Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(), NewAST->getDiagnostics().end()); + if (That->FileSyms) { + auto Symbols = + indexAST(NewAST->getASTContext(), NewAST->getPreprocessorPtr(), + NewAST->getTopLevelDecls()); + That->FileSyms->update(That->FileName, std::move(Symbols)); + } } else { // Don't report even Preamble diagnostics if we coulnd't build AST. Diagnostics.clear(); Index: clangd/ClangdUnitStore.h =================================================================== --- clangd/ClangdUnitStore.h +++ clangd/ClangdUnitStore.h @@ -15,6 +15,7 @@ #include "ClangdUnit.h" #include "GlobalCompilationDatabase.h" #include "Path.h" +#include "index/FileIndex.h" #include "clang/Tooling/CompilationDatabase.h" namespace clang { @@ -25,11 +26,13 @@ /// Thread-safe mapping from FileNames to CppFile. class CppFileCollection { public: + explicit CppFileCollection(FileSymbols *FileSyms) : FileSyms(FileSyms) {} + std::shared_ptr getOrCreateFile(PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory, std::shared_ptr PCHs, - clangd::Logger &Logger) { + clangd::Logger &Logger, FileSymbols *FileSyms) { std::lock_guard Lock(Mutex); auto It = OpenedFiles.find(File); @@ -37,9 +40,10 @@ auto Command = getCompileCommand(CDB, File, ResourceDir); It = OpenedFiles - .try_emplace(File, CppFile::Create(File, std::move(Command), - StorePreamblesInMemory, - std::move(PCHs), Logger)) + .try_emplace(File, + CppFile::Create(File, std::move(Command), + StorePreamblesInMemory, + std::move(PCHs), Logger, FileSyms)) .first; } return It->second; @@ -86,6 +90,7 @@ std::mutex Mutex; llvm::StringMap> OpenedFiles; + FileSymbols *FileSyms; }; } // namespace clangd } // namespace clang Index: clangd/ClangdUnitStore.cpp =================================================================== --- clangd/ClangdUnitStore.cpp +++ clangd/ClangdUnitStore.cpp @@ -23,6 +23,8 @@ std::shared_ptr Result = It->second; OpenedFiles.erase(It); + if (FileSyms) + FileSyms->remove(File); return Result; } @@ -40,16 +42,17 @@ auto It = OpenedFiles.find(File); if (It == OpenedFiles.end()) { It = OpenedFiles - .try_emplace(File, CppFile::Create(File, std::move(NewCommand), - StorePreamblesInMemory, - std::move(PCHs), Logger)) + .try_emplace(File, + CppFile::Create(File, std::move(NewCommand), + StorePreamblesInMemory, + std::move(PCHs), Logger, FileSyms)) .first; } else if (!compileCommandsAreEqual(It->second->getCompileCommand(), NewCommand)) { Result.RemovedFile = std::move(It->second); It->second = CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory, - std::move(PCHs), Logger); + std::move(PCHs), Logger, FileSyms); } Result.FileInCollection = It->second; return Result; Index: clangd/CodeComplete.h =================================================================== --- clangd/CodeComplete.h +++ clangd/CodeComplete.h @@ -18,6 +18,7 @@ #include "Logger.h" #include "Path.h" #include "Protocol.h" +#include "index/Index.h" #include "clang/Frontend/PrecompiledPreamble.h" #include "clang/Sema/CodeCompleteOptions.h" #include "clang/Tooling/CompilationDatabase.h" @@ -46,9 +47,15 @@ /// Add globals to code completion results. bool IncludeGlobals = true; + /// Add symbols in namespace context (including global namespace) to code + /// completion. + /// FIXME(ioeric): this only affects qualified-id code completion at this + /// point. + unsigned IncludeNamespaceLevelDecls = true; + /// Add brief comments to completion items, if available. - /// FIXME(ibiryukov): it looks like turning this option on significantly slows - /// down completion, investigate if it can be made faster. + /// FIXME(ibiryukov): it looks like turning this option on significantly + /// slows down completion, investigate if it can be made faster. bool IncludeBriefComments = true; /// Include results that are not legal completions in the current context. @@ -61,13 +68,16 @@ }; /// Get code completions at a specified \p Pos in \p FileName. +/// If `Index` is not nullptr, it can be used to augment the code completion +/// results. CompletionList codeComplete(PathRef FileName, const tooling::CompileCommand &Command, PrecompiledPreamble const *Preamble, StringRef Contents, Position Pos, IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, - CodeCompleteOptions Opts, Logger &Logger); + CodeCompleteOptions Opts, Logger &Logger, + const SymbolIndex *Index); /// Get signature help at a specified \p Pos in \p FileName. SignatureHelp Index: clangd/CodeComplete.cpp =================================================================== --- clangd/CodeComplete.cpp +++ clangd/CodeComplete.cpp @@ -15,7 +15,9 @@ //===---------------------------------------------------------------------===// #include "CodeComplete.h" +#include "ClangdServer.h" #include "Compiler.h" +#include "index/Index.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Sema/CodeCompleteConsumer.h" @@ -85,6 +87,53 @@ llvm_unreachable("Unhandled CodeCompletionResult::ResultKind."); } +CompletionItemKind getKindOfSymbol(index::SymbolKind Kind) { + using SK = index::SymbolKind; + switch (Kind){ + case SK::Unknown: + return CompletionItemKind::Missing; + case SK::Module: + case SK::Namespace: + case SK::NamespaceAlias: + return CompletionItemKind::Module; + case SK::Macro: + return CompletionItemKind::Text; + case SK::Enum: + return CompletionItemKind::Enum; + case SK::Struct: + case SK::Class: + case SK::Protocol: + case SK::Extension: + case SK::Union: + return CompletionItemKind::Class; + case SK::TypeAlias: + case SK::Using: + return CompletionItemKind::Reference; + case SK::Function: + case SK::ConversionFunction: + return CompletionItemKind::Function; + case SK::Variable: + case SK::Parameter: + return CompletionItemKind::Variable; + case SK::Field: + return CompletionItemKind::Field; + case SK::EnumConstant: + return CompletionItemKind::Value; + case SK::InstanceMethod: + case SK::ClassMethod: + case SK::StaticMethod: + return CompletionItemKind::Method; + case SK::InstanceProperty: + case SK::ClassProperty: + case SK::StaticProperty: + return CompletionItemKind::Property; + case SK::Constructor: + case SK::Destructor: + return CompletionItemKind::Constructor; + } + llvm_unreachable("Unhandled clang::index::SymbolKind."); +} + std::string escapeSnippet(const llvm::StringRef Text) { std::string Result; Result.reserve(Text.size()); // Assume '$', '}' and '\\' are rare. @@ -163,6 +212,22 @@ return Result; } +// Produces an integer that sorts in the same order as F. +// That is: a < b <==> encodeFloat(a) < encodeFloat(b). +static uint32_t encodeFloat(float F) { + static_assert(std::numeric_limits::is_iec559, ""); + static_assert(sizeof(float) == sizeof(uint32_t), ""); + constexpr uint32_t TopBit = ~(~uint32_t{0} >> 1); + + // Get the bits of the float. Endianness is the same as for integers. + uint32_t U; + memcpy(&U, &F, sizeof(float)); + // IEEE 754 floats compare like sign-magnitude integers. + if (U & TopBit) // Negative float. + return 0 - U; // Map onto the low half of integers, order reversed. + return U + TopBit; // Positive floats map onto the high half of integers. +} + /// A scored code completion result. /// It may be promoted to a CompletionItem if it's among the top-ranked results. struct CompletionCandidate { @@ -210,38 +275,123 @@ } return Score; } +}; - // Produces an integer that sorts in the same order as F. - // That is: a < b <==> encodeFloat(a) < encodeFloat(b). - static uint32_t encodeFloat(float F) { - static_assert(std::numeric_limits::is_iec559, ""); - static_assert(sizeof(float) == sizeof(uint32_t), ""); - constexpr uint32_t TopBit = ~(~uint32_t{0} >> 1); - - // Get the bits of the float. Endianness is the same as for integers. - uint32_t U; - memcpy(&U, &F, sizeof(float)); - // IEEE 754 floats compare like sign-magnitude integers. - if (U & TopBit) // Negative float. - return 0 - U; // Map onto the low half of integers, order reversed. - return U + TopBit; // Positive floats map onto the high half of integers. +/// \brief Information about the scope specifier in the qualfiied-id code +/// completion (e.g. "ns::ab?"). +struct ScopeSpecifierInfo { + static ScopeSpecifierInfo create(Sema &S, const CXXScopeSpec &SS) { + ScopeSpecifierInfo Info; + auto &SM = S.getSourceManager(); + auto SpecifierRange = SS.getRange(); + Info.WrittenSpecifier = + Lexer::getSourceText(CharSourceRange::getCharRange(SpecifierRange), SM, + clang::LangOptions()); + if (SS.isValid()) { + DeclContext *DC = S.computeDeclContext(SS); + if (auto *NS = llvm::dyn_cast(DC)) { + Info.SpecifiedContextName = NS->getQualifiedNameAsString(); + } else if (auto *TU = llvm::dyn_cast(DC)) { + Info.SpecifiedContextName = ""; + // Sema does not include the suffix "::" in the range of SS, so we add + // it back here. + Info.WrittenSpecifier = ""; + } + } + + Info.SpecifierBeginOffset = SM.getFileOffset(SpecifierRange.getBegin()); + Info.SpecifierEndOffset = SM.getFileOffset(SpecifierRange.getEnd()); + llvm::errs() << "create SSInfo: " << Info.SpecifiedContextName << ", " + << Info.WrittenSpecifier << "\n"; + return Info; } + + // The range of scope specifier as written. This does not include the + // filter text following the specifier. For example, for completion at + // "ns::ab?", the range will be "ns". + unsigned int SpecifierBeginOffset; + unsigned int SpecifierEndOffset; + + // The scope specifier as written. For example, for completion "ns::ab?", + // the written scope specifier is "ns". + std::string WrittenSpecifier; + // If this scope specifier is recognized in Sema (e.g. as a namespace + // context), this will be set to the fully qualfied name of the corresponding + // context. + std::string SpecifiedContextName; +}; + +CompletionItem symbolToCompletionItem(const Symbol &Sym, llvm::StringRef Code, + const ScopeSpecifierInfo &SSInfo) { + CompletionItem Item; + bool FullyQualified = + llvm::StringRef(SSInfo.WrittenSpecifier).startswith("::"); + if (FullyQualified) + Item.label = "::"; + Item.label += + Sym.CompletionLabel.empty() ? Sym.QualifiedName : Sym.CompletionLabel; + Item.kind = getKindOfSymbol(Sym.SymInfo.Kind); + Item.detail = Sym.CompletionDetail; + Item.documentation = Sym.Documentation; + Item.insertTextFormat = InsertTextFormat::PlainText; + // FIXME: sort symbols appropriately. + Item.sortText = ""; + + TextEdit Edit; + Edit.newText = + FullyQualified ? ("::" + Sym.QualifiedName) : Sym.QualifiedName; + Edit.range.start = + offsetToPosition(Code, SSInfo.SpecifierBeginOffset); + Edit.range.end = offsetToPosition(Code, SSInfo.SpecifierEndOffset); + Item.textEdit = std::move(Edit); + return Item; +} + +void qualifiedIdCompletionWithIndex(const SymbolIndex &Index, + llvm::StringRef Code, + const ScopeSpecifierInfo &SSInfo, + llvm::StringRef Filter, + CompletionList *Items) { + FuzzyFindRequest Req; + Req.Query = SSInfo.SpecifiedContextName.empty() ? SSInfo.WrittenSpecifier + : SSInfo.SpecifiedContextName; + // FIXME: for now we simply cancatenate specifier with the typed filter. We + // might want to fix the specifier prefix if it is a recognized context (e.g. + // a known namespace in the AST). + Req.Query += "::"; + if (!Filter.empty()) + Req.Query += Filter; + + Items->isIncomplete = !Index.fuzzyFind(Req, [&](const Symbol &Sym) { + Items->items.push_back(symbolToCompletionItem(Sym, Code, SSInfo)); + }); +} + +struct SemaCompletionInfo { + std::string Filter; + + llvm::Optional SSInfo; + // FIXME: add more information for other completion cases that we care about. + // For example, non-qualified id completion. }; class CompletionItemsCollector : public CodeCompleteConsumer { public: CompletionItemsCollector(const CodeCompleteOptions &CodeCompleteOpts, - CompletionList &Items) + CompletionList &Items, SemaCompletionInfo &SCInfo) : CodeCompleteConsumer(CodeCompleteOpts.getClangCompleteOpts(), /*OutputIsBinary=*/false), ClangdOpts(CodeCompleteOpts), Items(Items), Allocator(std::make_shared()), - CCTUInfo(Allocator) {} + CCTUInfo(Allocator), SCInfo(SCInfo) {} void ProcessCodeCompleteResults(Sema &S, CodeCompletionContext Context, CodeCompletionResult *Results, unsigned NumResults) override final { - StringRef Filter = S.getPreprocessor().getCodeCompletionFilter(); + SCInfo.Filter = S.getPreprocessor().getCodeCompletionFilter(); + if (llvm::Optional SS = + Context.getCXXScopeSpecifier()) + SCInfo.SSInfo = ScopeSpecifierInfo::create(S, **SS); std::priority_queue Candidates; for (unsigned I = 0; I < NumResults; ++I) { auto &Result = Results[I]; @@ -249,7 +399,8 @@ (Result.Availability == CXAvailability_NotAvailable || Result.Availability == CXAvailability_NotAccessible)) continue; - if (!Filter.empty() && !fuzzyMatch(S, Context, Filter, Result)) + if (!SCInfo.Filter.empty() && + !fuzzyMatch(S, Context, SCInfo.Filter, Result)) continue; Candidates.emplace(Result); if (ClangdOpts.Limit && Candidates.size() > ClangdOpts.Limit) { @@ -336,7 +487,7 @@ CompletionList &Items; std::shared_ptr Allocator; CodeCompletionTUInfo CCTUInfo; - + SemaCompletionInfo &SCInfo; }; // CompletionItemsCollector bool isInformativeQualifierChunk(CodeCompletionString::Chunk const &Chunk) { @@ -349,8 +500,9 @@ public: PlainTextCompletionItemsCollector(const CodeCompleteOptions &CodeCompleteOpts, - CompletionList &Items) - : CompletionItemsCollector(CodeCompleteOpts, Items) {} + CompletionList &Items, + SemaCompletionInfo &SCInfo) + : CompletionItemsCollector(CodeCompleteOpts, Items, SCInfo) {} private: void ProcessChunks(const CodeCompletionString &CCS, @@ -385,8 +537,9 @@ public: SnippetCompletionItemsCollector(const CodeCompleteOptions &CodeCompleteOpts, - CompletionList &Items) - : CompletionItemsCollector(CodeCompleteOpts, Items) {} + CompletionList &Items, + SemaCompletionInfo &SCInfo) + : CompletionItemsCollector(CodeCompleteOpts, Items, SCInfo) {} private: void ProcessChunks(const CodeCompletionString &CCS, @@ -655,6 +808,7 @@ Result.IncludeCodePatterns = EnableSnippets && IncludeCodePatterns; Result.IncludeMacros = IncludeMacros; Result.IncludeGlobals = IncludeGlobals; + Result.IncludeNamespaceLevelDecls = IncludeNamespaceLevelDecls; Result.IncludeBriefComments = IncludeBriefComments; return Result; @@ -666,19 +820,28 @@ StringRef Contents, Position Pos, IntrusiveRefCntPtr VFS, std::shared_ptr PCHs, - CodeCompleteOptions Opts, Logger &Logger) { + CodeCompleteOptions Opts, Logger &Logger, + const SymbolIndex *Index) { CompletionList Results; std::unique_ptr Consumer; + SemaCompletionInfo SCInfo; if (Opts.EnableSnippets) { - Consumer = - llvm::make_unique(Opts, Results); + Consumer = llvm::make_unique(Opts, Results, + SCInfo); } else { - Consumer = - llvm::make_unique(Opts, Results); + Consumer = llvm::make_unique( + Opts, Results, SCInfo); } invokeCodeComplete(std::move(Consumer), Opts.getClangCompleteOpts(), FileName, Command, Preamble, Contents, Pos, std::move(VFS), std::move(PCHs), Logger); + if (Index && SCInfo.SSInfo) { + // FIXME: log warning with logger if sema code completion have collected + // results. + Results.items.clear(); + qualifiedIdCompletionWithIndex(*Index, Contents, *SCInfo.SSInfo, + SCInfo.Filter, &Results); + } return Results; } Index: clangd/index/FileIndex.h =================================================================== --- /dev/null +++ clangd/index/FileIndex.h @@ -0,0 +1,84 @@ +//===--- FileIndex.h - Symbol index on a set of files.------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H + +#include "../Path.h" +#include "Index.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Error.h" +#include + +namespace clang { +namespace clangd { + +class FileSymbolsMemIndex; + +/// \brief This manages symbols from source files that can be updated or +/// removed. Symbols are used by a symbol index which builds indexes on symbols +/// managed by this class. +/// +/// This implements a snapshot semantics for symbols in a file. Each update to a +/// file will create a new snapshot for all symbols in the file. Snapshots are +/// managed with shared pointers that are shared between this class and the +/// symbol index. For each file, this class only stores a pointer pointing to +/// the newest snapshot, and an outdated snapshot is deleted by the last user of +/// the snapshot, either this class or the symbol index. There should be at most +/// two living snapshots (the latest snapshot and the last one) for the same +/// file at any point. +/// +/// The snapshot semantics keeps critical sections minimal since we only need +/// locking when we swap or obtain refereces to snapshots. +class FileSymbols { +public: + /// \brief Rebuilds the index for \p MemIndex when symbols are updated. + explicit FileSymbols(FileSymbolsMemIndex *MemIndex) : MemIndex(MemIndex) {} + + /// \brief Updates all symbols in a file. + void update(PathRef Path, std::unique_ptr Symbols); + + /// \brief Removes snapshots of \p Path. + void remove(PathRef Path); + +private: + mutable std::mutex Mutex; + + /// \brief Stores the latest snapshots for all active files. + llvm::StringMap> FileToSymbols; + /// \brief The symbol index that builds index on top of symbols from all + /// files. + FileSymbolsMemIndex *MemIndex; +}; + +/// \brief This implements a symbol index for a (relatively small) set of files +/// whose symbols can be easily managed in memory. +class FileSymbolsMemIndex : public SymbolIndex { +public: + /// \brief Re-builds the index with symbols from all files. All existing + /// file snapshots are disgarded in favor of the new snapshots. + void rebuild(llvm::StringMap> FileSnapshots); + + bool fuzzyFind(const FuzzyFindRequest &Req, + std::function Callback) const override; + +private: + /// A map from file name to all symbols in the TU of a file. + llvm::StringMap> FileToSymbols; + /// This is an index built on top of symbols from different source files. + /// For now, this is simply a vector. We may need a more structured index in + /// the future. + std::vector Symbols; + mutable std::mutex Mutex; +}; + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H Index: clangd/index/FileIndex.cpp =================================================================== --- /dev/null +++ clangd/index/FileIndex.cpp @@ -0,0 +1,74 @@ +//===--- ClangdIndex.cpp - Symbol indexes for clangd. ------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===-------------------------------------------------------------------===// + +#include "FileIndex.h" +#include "clang/Index/IndexingAction.h" + +namespace clang { +namespace clangd { + +void FileSymbols::update(PathRef Path, std::unique_ptr Symbols) { + llvm::StringMap> FileSnapshots; + { + std::lock_guard Lock(Mutex); + FileToSymbols[Path] = std::shared_ptr(Symbols.release()); + FileSnapshots = FileToSymbols; + } + + assert(MemIndex && "MemIndex must be set."); + // FIXME: avoid rebuilding the index when symbols haven't changed. + MemIndex->rebuild(std::move(FileSnapshots)); +} + +void FileSymbols::remove(PathRef Path) { + std::lock_guard Lock(Mutex); + FileToSymbols.erase(Path); + // Rely on the next update() to flush out symbols in the removed file from the + // index. +} + +void FileSymbolsMemIndex::rebuild( + llvm::StringMap> FileSnapshots) { + // Build an index, which is simply a vector of symbol references at this + // point. + std::vector TempSymbols; + for (const auto &Pair : FileSnapshots) + for (const auto &Sym : *Pair.second) + TempSymbols.push_back(&Sym.second); + // Swap out the old index and snapshots. + { + std::lock_guard Lock(Mutex); + this->Symbols = std::move(TempSymbols); + this->FileToSymbols = std::move(FileSnapshots); + } +} + +bool FileSymbolsMemIndex::fuzzyFind( + const FuzzyFindRequest &Req, + std::function Callback) const { + std::string LoweredQuery = llvm::StringRef(Req.Query).lower(); + unsigned Matched = 0; + { + std::lock_guard Lock(Mutex); + for (const auto *Sym : Symbols) { + // Find all symbols that contain the query, igoring cases. + // FIXME: use better matching algorithm, e.g. fuzzy matcher. + if (StringRef(StringRef(Sym->QualifiedName).lower()) + .contains(LoweredQuery)) { + Callback(*Sym); + if (++Matched > Req.MaxCandidateCount) + return false; + } + } + } + return true; +} + +} // namespace clangd +} // namespace clang Index: clangd/index/Index.h =================================================================== --- clangd/index/Index.h +++ clangd/index/Index.h @@ -80,7 +80,17 @@ // FIXME: add definition location of the symbol. // FIXME: add all occurrences support. // FIXME: add extra fields for index scoring signals. - // FIXME: add code completion information. + + // Documentation including comment for the symbol declaration. + std::string Documentation; + // A brief description of the symbol that can be displayed in the completion + // candidate list. For example, "a::b::Foo(X x, Y y) const" is a labal for + // a function. + std::string CompletionLabel; + // Detail about the symbol. For example, the result type of a function. + std::string CompletionDetail; + // The placeholder text for function parameters in order. + std::vector Params; }; // A symbol container that stores a set of symbols. The container will maintain @@ -110,6 +120,33 @@ llvm::DenseMap Symbols; }; +struct FuzzyFindRequest { + /// \brief A query string for the fuzzy find. This is matched against symbols' + /// qualfified names. + std::string Query; + /// \brief The maxinum number of candidates to return. + size_t MaxCandidateCount = UINT_MAX; +}; + +/// \brief Interface for symbol indexes that can be used for searching or +/// matching symbols among a set of symbols based on names or unique IDs. +class SymbolIndex { +public: + virtual ~SymbolIndex() = default; + + /// \brief Matches symbols in the index fuzzily and applies \p Callback on + /// each matched symbol. + /// + /// Returns true if all candidates are matched. + virtual bool + fuzzyFind(const FuzzyFindRequest &Req, + std::function Callback) const = 0; + + // FIXME: add interfaces for more index use cases: + // - Symbol getSymbolInfo(llvm::StringRef USR); + // - getAllOccurrences(llvm::StringRef USR); +}; + } // namespace clangd } // namespace clang Index: clangd/index/Index.cpp =================================================================== --- clangd/index/Index.cpp +++ clangd/index/Index.cpp @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// #include "Index.h" - +#include "clang/Sema/CodeCompleteConsumer.h" #include "llvm/Support/SHA1.h" namespace clang { @@ -20,6 +20,7 @@ } } // namespace + SymbolID::SymbolID(llvm::StringRef USR) : HashValue(llvm::SHA1::hash(toArrayRef(USR))) {} Index: clangd/index/SymbolCollector.h =================================================================== --- clangd/index/SymbolCollector.h +++ clangd/index/SymbolCollector.h @@ -24,6 +24,12 @@ public: SymbolCollector() = default; + void initialize(ASTContext &Ctx) override { ASTCtx = &Ctx; } + + void setPreprocessor(std::shared_ptr PP) override { + this->PP = std::move(PP); + } + bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, ArrayRef Relations, FileID FID, @@ -37,6 +43,9 @@ private: // All Symbols collected from the AST. SymbolSlab Symbols; + + ASTContext *ASTCtx; + std::shared_ptr PP; }; } // namespace clangd Index: clangd/index/SymbolCollector.cpp =================================================================== --- clangd/index/SymbolCollector.cpp +++ clangd/index/SymbolCollector.cpp @@ -8,8 +8,8 @@ //===----------------------------------------------------------------------===// #include "SymbolCollector.h" - #include "clang/AST/ASTContext.h" +#include "clang/Sema/CodeCompleteConsumer.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/Basic/SourceManager.h" @@ -57,6 +57,107 @@ } return AbsolutePath.str(); } + +std::string getDocumentation(const CodeCompletionString &CCS) { + // Things like __attribute__((nonnull(1,3))) and [[noreturn]]. Present this + // information in the documentation field. + std::string Result; + const unsigned AnnotationCount = CCS.getAnnotationCount(); + if (AnnotationCount > 0) { + Result += "Annotation"; + if (AnnotationCount == 1) { + Result += ": "; + } else /* AnnotationCount > 1 */ { + Result += "s: "; + } + for (unsigned I = 0; I < AnnotationCount; ++I) { + Result += CCS.getAnnotation(I); + Result.push_back(I == AnnotationCount - 1 ? '\n' : ' '); + } + } + // Add brief documentation (if there is any). + if (CCS.getBriefComment() != nullptr) { + if (!Result.empty()) { + // This means we previously added annotations. Add an extra newline + // character to make the annotations stand out. + Result.push_back('\n'); + } + Result += CCS.getBriefComment(); + } + return Result; +} + +void ProcessChunks(const CodeCompletionString &CCS, Symbol *Sym) { + for (const auto &Chunk : CCS) { + // Informative qualifier chunks only clutter completion results, skip + // them. + if (Chunk.Kind == CodeCompletionString::CK_Informative && + StringRef(Chunk.Text).endswith("::")) + continue; + + switch (Chunk.Kind) { + case CodeCompletionString::CK_TypedText: + // There's always exactly one CK_TypedText chunk. + Sym->CompletionLabel += Chunk.Text; + break; + case CodeCompletionString::CK_ResultType: + Sym->CompletionDetail = Chunk.Text; + break; + case CodeCompletionString::CK_Optional: + break; + case CodeCompletionString::CK_Placeholder: + // A string that acts as a placeholder for, e.g., a function call + // argument. + Sym->Params.push_back(Chunk.Text); + LLVM_FALLTHROUGH; + default: + Sym->CompletionLabel += Chunk.Text; + break; + } + } +} + +inline std::string +joinNamespaces(const llvm::SmallVectorImpl &Namespaces) { + if (Namespaces.empty()) + return ""; + std::string Result = Namespaces.front(); + for (auto I = Namespaces.begin() + 1, E = Namespaces.end(); I != E; ++I) + Result += ("::" + *I).str(); + return Result; +} + +// Given "a::b::c", returns {"a", "b", "c"}. +llvm::SmallVector splitSymbolName(llvm::StringRef Name) { + llvm::SmallVector Splitted; + Name.split(Splitted, "::", /*MaxSplit=*/-1, + /*KeepEmpty=*/false); + return Splitted; +} + +void addSymbolCompletionInfo(ASTContext &Ctx, Preprocessor &PP, + const NamedDecl *ND, Symbol *Sym) { + CodeCompletionResult SymbolCompletion(ND, 0); + auto Allocator = std::make_shared(); + CodeCompletionTUInfo TUInfo(Allocator); + const auto *CCS = SymbolCompletion.CreateCodeCompletionString( + Ctx, PP, CodeCompletionContext::CCC_Name, *Allocator, TUInfo, + /*IncludeBriefComments*/ true); + Sym->Documentation = getDocumentation(*CCS); + + ProcessChunks(*CCS, Sym); + // Since symbol names in CCS labels are not qualified, we prepend a namespace + // qualfifier. + std::string QualifiedName = ND->getQualifiedNameAsString(); + auto SplittedNames = splitSymbolName(QualifiedName); + if (SplittedNames.size() > 1) { + std::string LabelPrefix = joinNamespaces(SmallVector( + SplittedNames.begin(), SplittedNames.end() - 1)); + Sym->CompletionLabel = LabelPrefix + "::" + Sym->CompletionLabel; + } +} + + } // namespace // Always return true to continue indexing. @@ -87,8 +188,14 @@ SymbolLocation Location = { makeAbsolutePath(SM, SM.getFilename(D->getLocation())), SM.getFileOffset(D->getLocStart()), SM.getFileOffset(D->getLocEnd())}; - Symbols.insert({std::move(ID), ND->getQualifiedNameAsString(), - index::getSymbolInfo(D), std::move(Location)}); + Symbol Sym; + Sym.ID = std::move(ID); + Sym.QualifiedName = ND->getQualifiedNameAsString(); + Sym.SymInfo = index::getSymbolInfo(D); + Sym.CanonicalDeclaration = std::move(Location); + assert(ASTCtx && PP.get() && "ASTContext and Preprocessor must be set."); + addSymbolCompletionInfo(*ASTCtx, *PP, ND, &Sym); + Symbols.insert(std::move(Sym)); } return true; Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -90,6 +90,15 @@ "Trace internal events and timestamps in chrome://tracing JSON format"), llvm::cl::init(""), llvm::cl::Hidden); +static llvm::cl::opt EnableIndexBasedCompletion( + "enable-index-based-completion", + llvm::cl::desc( + "Enable index-based global code completion (experimental). Clangd will " + "use symbols built from ASTs of opened files and additional indexes " + "(e.g. offline built codebase-wide symbol table) to complete partial " + "symbols."), + llvm::cl::init(false)); + int main(int argc, char *argv[]) { llvm::cl::ParseCommandLineOptions(argc, argv, "clangd"); @@ -170,9 +179,15 @@ clangd::CodeCompleteOptions CCOpts; CCOpts.EnableSnippets = EnableSnippets; CCOpts.IncludeIneligibleResults = IncludeIneligibleResults; + if (EnableIndexBasedCompletion) { + // Disable sema code completion for qualified code completion and use global + // symbol index instead. + CCOpts.IncludeNamespaceLevelDecls = false; + } // Initialize and run ClangdLSPServer. ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, - CCOpts, ResourceDirRef, CompileCommandsDirPath); + CCOpts, ResourceDirRef, CompileCommandsDirPath, + EnableIndexBasedCompletion); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;