diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -61,6 +61,7 @@ Quality.cpp RIFF.cpp Selection.cpp + SemanticHighlighting.cpp SourceCode.cpp Threading.cpp Trace.cpp diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -31,7 +31,8 @@ /// MessageHandler binds the implemented LSP methods (e.g. onInitialize) to /// corresponding JSON-RPC methods ("initialize"). /// The server also supports $/cancelRequest (MessageHandler provides this). -class ClangdLSPServer : private DiagnosticsConsumer { +class ClangdLSPServer : private DiagnosticsConsumer, + private SemanticHighlightingConsumer { public: /// If \p CompileCommandsDir has a value, compile_commands.json will be /// loaded only from \p CompileCommandsDir. Otherwise, clangd will look @@ -56,6 +57,9 @@ void onDiagnosticsReady(PathRef File, std::vector Diagnostics) override; void onFileUpdated(PathRef File, const TUStatus &Status) override; + void onSemanticHighlightingReady( + PathRef File, std::vector Tokens) override; + // LSP methods. Notifications have signature void(const Params&). // Calls have signature void(const Params&, Callback). void onInitialize(const InitializeParams &, Callback); diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -9,6 +9,7 @@ #include "ClangdLSPServer.h" #include "Diagnostics.h" #include "Protocol.h" +#include "SemanticHighlighting.h" #include "SourceCode.h" #include "Trace.h" #include "URI.h" @@ -342,6 +343,7 @@ CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags, ClangdServerOpts.ResourceDir); Server.emplace(*CDB, FSProvider, static_cast(*this), + static_cast(*this), ClangdServerOpts); applyConfiguration(Params.initializationOptions.ConfigSettings); @@ -396,7 +398,9 @@ ExecuteCommandParams::CLANGD_APPLY_TWEAK}}, }}, {"typeHierarchyProvider", true}, - }}}}; + {"semanticHighlighting", + llvm::json::Object{ + {"scopes", getSemanticHighlightingScopes()}}}}}}}; if (NegotiatedOffsetEncoding) Result["offsetEncoding"] = *NegotiatedOffsetEncoding; Reply(std::move(Result)); @@ -1034,6 +1038,15 @@ publishDiagnostics(URI, std::move(LSPDiagnostics)); } +void ClangdLSPServer::onSemanticHighlightingReady( + PathRef File, std::vector Tokens) { + TextDocumentIdentifier TDI{URIForFile::canonicalize(File, /*TUPath=*/File)}; + auto SHInfos = encodeSHTokens(Tokens); + notify( + "textDocument/semanticHighlighting", + llvm::json::Object{{"textDocument", TDI}, {"lines", std::move(SHInfos)}}); +} + void ClangdLSPServer::onFileUpdated(PathRef File, const TUStatus &Status) { if (!SupportFileStatus) return; diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -136,7 +136,8 @@ /// synchronize access to shared state. ClangdServer(const GlobalCompilationDatabase &CDB, const FileSystemProvider &FSProvider, - DiagnosticsConsumer &DiagConsumer, const Options &Opts); + DiagnosticsConsumer &DiagConsumer, + SemanticHighlightingConsumer &SHConsumer, const Options &Opts); /// Add a \p File to the list of tracked C++ files or update the contents if /// \p File is already tracked. Also schedules parsing of the AST for it on a diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -45,8 +45,9 @@ // Update the FileIndex with new ASTs and plumb the diagnostics responses. struct UpdateIndexCallbacks : public ParsingCallbacks { - UpdateIndexCallbacks(FileIndex *FIndex, DiagnosticsConsumer &DiagConsumer) - : FIndex(FIndex), DiagConsumer(DiagConsumer) {} + UpdateIndexCallbacks(FileIndex *FIndex, DiagnosticsConsumer &DiagConsumer, + SemanticHighlightingConsumer &SHConsumer) + : FIndex(FIndex), DiagConsumer(DiagConsumer), SHConsumer(SHConsumer) {} void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr PP, @@ -64,6 +65,11 @@ DiagConsumer.onDiagnosticsReady(File, std::move(Diags)); } + virtual void onSemanticHighlighting( + PathRef File, std::vector Tokens) override { + SHConsumer.onSemanticHighlightingReady(File, Tokens); + } + void onFileUpdated(PathRef File, const TUStatus &Status) override { DiagConsumer.onFileUpdated(File, Status); } @@ -71,6 +77,7 @@ private: FileIndex *FIndex; DiagnosticsConsumer &DiagConsumer; + SemanticHighlightingConsumer &SHConsumer; }; } // namespace @@ -85,6 +92,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, const FileSystemProvider &FSProvider, DiagnosticsConsumer &DiagConsumer, + SemanticHighlightingConsumer &SHConsumer, const Options &Opts) : FSProvider(FSProvider), DynamicIdx(Opts.BuildDynamicSymbolIndex @@ -99,8 +107,8 @@ // FIXME(ioeric): this can be slow and we may be able to index on less // critical paths. WorkScheduler(CDB, Opts.AsyncThreadsCount, Opts.StorePreamblesInMemory, - llvm::make_unique(DynamicIdx.get(), - DiagConsumer), + llvm::make_unique( + DynamicIdx.get(), DiagConsumer, SHConsumer), Opts.UpdateDebounce, Opts.RetentionPolicy) { // Adds an index to the stack, at higher priority than existing indexes. auto AddIndex = [&](SymbolIndex *Idx) { diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -633,6 +633,20 @@ }; llvm::json::Value toJSON(const Diagnostic &); +/// Represents semantic highlighting information that has to be applied on a +/// specific line of the text document. +struct SemanticHighlightingInformation { + /// The zero-based line position in the text document. + int line; + + /// A base64 encoded string representing every single highlighting character + /// with its start position, length, and the "lookup table" index of the + /// semantic highlighting TextMate scopes. + /// If empty, then no highlighted positions are available for the line. + llvm::Optional tokens; +}; +llvm::json::Value toJSON(const SemanticHighlightingInformation &); + /// A LSP-specific comparator used to find diagnostic in a container like /// std:map. /// We only use the required fields of Diagnostic to do the comparsion to avoid diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -458,6 +458,13 @@ return true; } +llvm::json::Value toJSON(const SemanticHighlightingInformation &SHI) { + llvm::json::Object Result{{"line", SHI.line}}; + if (SHI.tokens) + Result["tokens"] = *SHI.tokens; + return std::move(Result); +} + bool fromJSON(const llvm::json::Value &Params, CodeActionContext &R) { llvm::json::ObjectMapper O(Params); return O && O.map("diagnostics", R.diagnostics); diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -0,0 +1,86 @@ +//===--- SemanticHighlighting.h ----------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTIC_HIGHLIGHTING_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTIC_HIGHLIGHTING_H + +#include "ClangdUnit.h" +#include "Path.h" +#include "Protocol.h" + +namespace clang { +namespace clangd { + +enum class SemanticHighlightingKind : uint16_t { + Class, + Enum, + EnumClass, + Enumerator, + Field, + Function, + FunctionDeclaration, + GlobalVariable, + Label, + LocalVariable, + LocalVariableDeclaration, + Macro, + MacroDefinition, + Method, + MethodDeclaration, + Namespace, + ParameterVariable, + StaticField, + StaticMethod, + StaticMethodDeclaration, + TemplateParameter, + Typedef, + Special_LastKind, + // TODO: OverloadedOperator, VariablePassedByNonconstRef +}; + +/// Return the supported semantic highlighting scopes. +/// Every top-level element of the returned array corresponds to +/// one of the token kinds in SemanticHighlightingKind, and +/// contains one or more TextMate scope names corresponding to +/// that token kind. +std::vector> getSemanticHighlightingScopes(); + +struct SemanticHighlightingToken { + Range R; + SemanticHighlightingKind Kind; + + friend bool operator==(const SemanticHighlightingToken &A, + const SemanticHighlightingToken &B) { + return A.R == B.R && A.Kind == B.Kind; + } + friend bool operator<(const SemanticHighlightingToken &A, + const SemanticHighlightingToken &B) { + return (A.R < B.R) || (A.R == B.R && A.Kind < B.Kind); + } +}; + +std::vector +computeSemanticHighlightings(ParsedAST &AST); + +class SemanticHighlightingConsumer { +public: + virtual ~SemanticHighlightingConsumer() = default; + + virtual void onSemanticHighlightingReady( + PathRef File, std::vector Tokens) = 0; +}; + +/// Encode semantic highlighting tokens for an AST into the representation +/// used in the wire protocol. +std::vector +encodeSHTokens(const std::vector &Tokens); + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTIC_HIGHLIGHTING_H diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -0,0 +1,393 @@ +//===--- SemanticHighlighting.cpp --------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SemanticHighlighting.h" + +#include "AST.h" +#include "clang/AST/RecursiveASTVisitor.h" + +namespace clang { +namespace clangd { + +std::vector> getSemanticHighlightingScopes() { + // For now, just use one scope per token kind. The protocol supports + // multiple, expected to be arranged from most specific to least specific, + // which could be added later. + // TODO: Revise scope names to make consistent with other languages / editors? + return { + {"entity.name.class.cpp"}, // Class + {"entity.name.enum.cpp"}, // Enum + {"entity.name.enum.class.cpp"}, // EnumClass + {"entity.name.enumerator.cpp"}, // Enumerator + {"entity.name.variable.member.cpp"}, // Field + {"entity.name.function.cpp"}, // Function + {"entity.name.function.declaration.cpp"}, // FunctionDeclaration + {"entity.name.variable.global.cpp"}, // GlobalVariable + {"entity.name.label.cpp"}, // Label + {"entity.name.variable.local.cpp"}, // LocalVariable + {"entity.name.variable.local.declaration.cpp"}, // LocalVariableDeclaration + {"entity.name.macro.cpp"}, // Macro + {"entity.name.macro.definition.cpp"}, // MacroDefinition + {"entity.name.function.member.cpp"}, // Method + {"entity.name.function.member.declaration.cpp"}, // MethodDeclaration + {"entity.name.namespace.cpp"}, // Namespace + {"entity.name.variable.parameter.cpp"}, // ParameterVariable + {"entity.name.variable.member.static.cpp"}, // StaticField + {"entity.name.function.member.static.cpp"}, // StaticMethod + {"entity.name.function.member.static.declaration.cpp"}, // StaticMethodDeclaration + {"entity.name.templateparameter.cpp"}, // TemplateParameter + {"entity.name.typedef.cpp"}, // Typedef + }; +} + +namespace { + +Optional classifyDecl(Decl *D, bool IsDeclaration) { + if (dyn_cast(D)) { + return SemanticHighlightingKind::ParameterVariable; + } + if (VarDecl *VD = dyn_cast(D)) { + if (VD->isStaticDataMember()) { + return SemanticHighlightingKind::StaticField; + } + if (VD->isLocalVarDecl()) { + return IsDeclaration ? SemanticHighlightingKind::LocalVariableDeclaration + : SemanticHighlightingKind::LocalVariable; + } + return SemanticHighlightingKind::GlobalVariable; + } + if (FieldDecl *FD = dyn_cast(D)) { + return SemanticHighlightingKind::Field; + } + if (CXXMethodDecl *MD = dyn_cast(D)) { + if (MD->isStatic()) { + return IsDeclaration ? SemanticHighlightingKind::StaticMethodDeclaration + : SemanticHighlightingKind::StaticMethod; + } + return IsDeclaration ? SemanticHighlightingKind::MethodDeclaration + : SemanticHighlightingKind::Method; + } + if (dyn_cast(D)) { + return IsDeclaration ? SemanticHighlightingKind::FunctionDeclaration + : SemanticHighlightingKind::Function; + } + if (dyn_cast(D)) { + return SemanticHighlightingKind::Class; + } + if (EnumDecl *ED = dyn_cast(D)) { + return ED->isScoped() ? SemanticHighlightingKind::EnumClass + : SemanticHighlightingKind::Enum; + } + if (dyn_cast(D)) { + return SemanticHighlightingKind::Enumerator; + } + if (dyn_cast(D) || + dyn_cast(D) || + dyn_cast(D)) { + return SemanticHighlightingKind::TemplateParameter; + } + if (dyn_cast(D)) { + return SemanticHighlightingKind::Typedef; + } + if (dyn_cast(D)) { + return SemanticHighlightingKind::Namespace; + } + // TODO: Label, Macro, MacroDefinition. + return llvm::None; +} + +bool isAnonymous(Decl *D) { + if (NamedDecl *ND = dyn_cast(D)) { + return ND->getDeclName().isEmpty(); + } + return false; +} + +class SemanticHighlightingVisitor + : public RecursiveASTVisitor { + using Base = RecursiveASTVisitor; + +public: + SemanticHighlightingVisitor(ASTContext &Context) : Context{Context} {} + + std::vector collectTokens() { + // FIXME: Having to set the scope here is required to get the + // VariableTemplates test to pass. This is either a clangd bug or a clang + // AST bug. + // Context.setTraversalScope({Context.getTranslationUnitDecl()}); + + TraverseAST(Context); + + // Deduplicate tokens. + // This takes care of avoiding duplicates in some cases that would be + // difficult to handle during the AST traversal itself, such as + // the template-name in the declaration of a class template specialization + // being traversed both as a Decl and as a TypeLoc. + std::sort(Tokens.begin(), Tokens.end()); + auto Last = std::unique(Tokens.begin(), Tokens.end()); + Tokens.erase(Last, Tokens.end()); + + return std::move(Tokens); + } + + // RecursiveASTVisitor overrides + // TODO: We're probably missing some. + + // Declarations + bool VisitNamedDecl(NamedDecl *ND) { + // Don't emit a highlighting for an empty name. + if (isAnonymous(ND)) { + return true; + } + if (auto Kind = classifyDecl(ND, /*IsDeclaration=*/true)) { + addToken(*Kind, findNameLoc(ND)); + } + return true; + } + + // Expressions + bool VisitDeclRefExpr(DeclRefExpr *E) { + return handleReference(E->getDecl(), E->getLocation()); + } + bool VisitMemberExpr(MemberExpr *E) { + return handleReference(E->getMemberDecl(), E->getMemberLoc()); + } + + // Types + bool VisitTagTypeLoc(TagTypeLoc L) { + return handleReference(L.getDecl(), L.getNameLoc()); + } + bool VisitTypedefTypeLoc(TypedefTypeLoc L) { + return handleReference(L.getTypedefNameDecl(), L.getNameLoc()); + } + bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) { + // Color the template name. The visitor takes care of visiting the + // arguments. + // TODO: Could we do this via TraverseTemplateName? + // Unclear how we would get the location. + auto TemplateNameLoc = L.getTemplateNameLoc(); + if (TemplateDecl *TD = + L.getTypePtr()->getTemplateName().getAsTemplateDecl()) { + return handleReference(TD->getTemplatedDecl(), TemplateNameLoc); + } + // If dependent, fall back to Class. + addToken(SemanticHighlightingKind::Class, TemplateNameLoc); + return true; + } + bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc L) { + return handleReference(L.getDecl(), L.getNameLoc()); + } + bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) { + // For now, assume it's a class. + // TODO: Perform heuristic resolution to try to make a better guess. + addToken(SemanticHighlightingKind::Class, L.getNameLoc()); + return true; + } + bool VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc L) { + addToken(SemanticHighlightingKind::Class, L.getNameLoc()); + return true; + } + + // Other node types, only available via Traverse*() + bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLoc) { + if (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier()) { + if (NNS->getKind() == NestedNameSpecifier::Namespace) { + addToken(SemanticHighlightingKind::Namespace, + NNSLoc.getLocalBeginLoc()); + } else if (NNS->getKind() == NestedNameSpecifier::Identifier) { + // This seems to occur when a nested-name-specifier has a + // dependent segment that can't be resolved to a type yet. + // Assume it resolves to a class. + // TODO: Perform heuristic resolution to try to make a better guess. + addToken(SemanticHighlightingKind::Class, NNSLoc.getLocalBeginLoc()); + } + } + return Base::TraverseNestedNameSpecifierLoc(NNSLoc); + } + bool TraverseConstructorInitializer(CXXCtorInitializer *Init) { + addToken(SemanticHighlightingKind::Field, Init->getMemberLocation()); + return Base::TraverseConstructorInitializer(Init); + } + + // Dependent expressions + bool VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) { + addToken(SemanticHighlightingKind::Method, E->getNameLoc()); + return true; + } + bool VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) { + addToken(E->getNamingClass() ? SemanticHighlightingKind::StaticMethod + : SemanticHighlightingKind::Function, + E->getNameLoc()); + return true; + } + bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) { + // For now, assume it's a static field. + // TODO: Perform heuristic resolution to try to make a better guess. + addToken(SemanticHighlightingKind::StaticField, E->getLocation()); + return true; + } + bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) { + addToken(E->getQualifier() ? SemanticHighlightingKind::StaticField + : SemanticHighlightingKind::Field, + E->getMemberLoc()); + return true; + } + bool VisitSizeOfPackExpr(SizeOfPackExpr *E) { + addToken(SemanticHighlightingKind::TemplateParameter, E->getPackLoc()); + return true; + } + +private: + ASTContext &Context; + std::vector Tokens; + + bool handleReference(Decl *D, SourceLocation Loc) { + // Don't emit a highlighting for an empty name. + if (!D || isAnonymous(D)) { + return true; + } + if (NamedDecl *ND = dyn_cast(D)) { + // Don't emit a highlighting for an operator name. + // TODO: We probably do want to emit a highlighting in the case where + // the "operator" token is used explicitly. Not sure how to + // distinguish this case. + if (ND->getDeclName().getNameKind() == DeclarationName::CXXOperatorName) { + return true; + } + } + if (auto Kind = classifyDecl(D, /*IsDeclaration=*/false)) { + addToken(*Kind, Loc); + } + return true; + } + + void addToken(SemanticHighlightingKind Kind, SourceLocation Loc) { + if (auto Range = toRange(Loc)) { + Tokens.push_back({*Range, Kind}); + } + } + + llvm::Optional toRange(SourceLocation Loc) { + // TODO: There is similar logic in declToSym() that could be factored out + // somewhere. + auto &SM = Context.getSourceManager(); + if (!SM.isWrittenInMainFile(Loc)) { + return llvm::None; + } + return Range{ + sourceLocToPosition(SM, Loc), + sourceLocToPosition( + SM, Lexer::getLocForEndOfToken(Loc, 0, SM, Context.getLangOpts()))}; + } +}; + +} // namespace + +std::vector +computeSemanticHighlightings(ParsedAST &AST) { + SemanticHighlightingVisitor Visitor{AST.getASTContext()}; + return Visitor.collectTokens(); +} + +namespace { + +// Encode binary data into base64. +// This was copied from compiler-rt/lib/fuzzer/FuzzerUtil.cpp. +// TOOD: Factor this out into llvm/Support? +std::string encodeBase64(const llvm::SmallVectorImpl &U) { + static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + std::string Res; + size_t i; + for (i = 0; i + 2 < U.size(); i += 3) { + uint32_t x = (U[i] << 16) + (U[i + 1] << 8) + U[i + 2]; + Res += Table[(x >> 18) & 63]; + Res += Table[(x >> 12) & 63]; + Res += Table[(x >> 6) & 63]; + Res += Table[x & 63]; + } + if (i + 1 == U.size()) { + uint32_t x = (U[i] << 16); + Res += Table[(x >> 18) & 63]; + Res += Table[(x >> 12) & 63]; + Res += "=="; + } else if (i + 2 == U.size()) { + uint32_t x = (U[i] << 16) + (U[i + 1] << 8); + Res += Table[(x >> 18) & 63]; + Res += Table[(x >> 12) & 63]; + Res += Table[(x >> 6) & 63]; + Res += "="; + } + return Res; +} + +// The protocol format requires integers to be encoded as big-endian. +void write32be(uint32_t I, llvm::raw_ostream &OS) { + char Buf[4]; + llvm::support::endian::write32be(Buf, I); + OS.write(Buf, sizeof(Buf)); +} +void write16be(uint16_t I, llvm::raw_ostream &OS) { + char Buf[2]; + llvm::support::endian::write16be(Buf, I); + OS.write(Buf, sizeof(Buf)); +} + +std::string +encodeSHTokensForLine(const std::vector &Tokens) { + llvm::SmallVector BinaryData; + llvm::raw_svector_ostream OS(BinaryData); + for (size_t i = 0; i < Tokens.size(); ++i) { + const auto &Token = Tokens[i]; + // The caller has checked that the token's range is valid and on a single + // line. + uint32_t start = Token.R.start.character; + uint16_t length = Token.R.end.character - Token.R.start.character; + uint16_t scope = static_cast(Token.Kind); + write32be(start, OS); + write16be(length, OS); + write16be(scope, OS); + } + return encodeBase64(BinaryData); +} + +} // namespace + +std::vector +encodeSHTokens(const std::vector &Tokens) { + // Group the tokens by line. + llvm::DenseMap> TokensPerLine; + for (const auto &Token : Tokens) { + int Line = Token.R.start.line; + if (Token.R.end.line != Line) { + // Token's range spans multiple lines. The protocol does not support this. + // TODO: Give some sort of error or warning. + continue; + } + if (Token.R.end.character < Token.R.start.character) { + // Token range is invalid. TODO: Give an error or warning. + continue; + } + TokensPerLine[Line].push_back(Token); + } + + // Encode each line's tokens into the protocol representation. + std::vector Infos; + for (const auto &Line : TokensPerLine) { + SemanticHighlightingInformation Info; + Info.line = Line.first; + Info.tokens = encodeSHTokensForLine(Line.second); + Infos.push_back(std::move(Info)); + } + + return Infos; +} + +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h --- a/clang-tools-extra/clangd/TUScheduler.h +++ b/clang-tools-extra/clangd/TUScheduler.h @@ -12,6 +12,7 @@ #include "ClangdUnit.h" #include "Function.h" #include "GlobalCompilationDatabase.h" +#include "SemanticHighlighting.h" #include "Threading.h" #include "index/CanonicalIncludes.h" #include "llvm/ADT/Optional.h" @@ -113,6 +114,11 @@ /// Called whenever the diagnostics for \p File are produced. virtual void onDiagnostics(PathRef File, std::vector Diags) {} + /// Called whenever the semantic highlightings for \p File are produced. + virtual void + onSemanticHighlighting(PathRef File, + std::vector Tokens) {} + /// Called whenever the TU status is updated. virtual void onFileUpdated(PathRef File, const TUStatus &Status) {} }; diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -46,6 +46,7 @@ #include "Compiler.h" #include "GlobalCompilationDatabase.h" #include "Logger.h" +#include "SemanticHighlighting.h" #include "Trace.h" #include "index/CanonicalIncludes.h" #include "clang/Frontend/CompilerInvocation.h" @@ -491,6 +492,13 @@ if (ReportDiagnostics) Callbacks.onDiagnostics(FileName, (*AST)->getDiagnostics()); } + + // Compute semantic highlighting. + // TODO: Should this be inside the `DiagsMu` critical section, + // or in a critical section of its own? + auto Highlightings = computeSemanticHighlightings(**AST); + Callbacks.onSemanticHighlighting(FileName, Highlightings); + trace::Span Span("Running main AST callback"); Callbacks.onMainAST(FileName, **AST); DiagsWereReported = true; diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -52,6 +52,7 @@ RenameTests.cpp RIFFTests.cpp SelectionTests.cpp + SemanticHighlightingTests.cpp SerializationTests.cpp SourceCodeTests.cpp SymbolCollectorTests.cpp diff --git a/clang-tools-extra/clangd/unittests/ClangdTests.cpp b/clang-tools-extra/clangd/unittests/ClangdTests.cpp --- a/clang-tools-extra/clangd/unittests/ClangdTests.cpp +++ b/clang-tools-extra/clangd/unittests/ClangdTests.cpp @@ -145,8 +145,10 @@ bool ExpectErrors = false) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); for (const auto &FileWithContents : ExtraFiles) FS.Files[testPath(FileWithContents.first)] = FileWithContents.second; @@ -197,8 +199,10 @@ TEST_F(ClangdVFSTest, Reparse) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); const auto SourceContents = R"cpp( #include "foo.h" @@ -232,8 +236,10 @@ TEST_F(ClangdVFSTest, ReparseOnHeaderChange) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); const auto SourceContents = R"cpp( #include "foo.h" @@ -284,9 +290,11 @@ int Got; } DiagConsumer; MockCompilationDatabase CDB; + NoopSemanticHighlightingConsumer SHConsumer; // Verify that the context is plumbed to the FS provider and diagnostics. - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); { WithContextValue Entrypoint(Secret, 42); Server.addDocument(testPath("foo.cpp"), "void main(){}"); @@ -302,12 +310,14 @@ // Checks that searches for GCC installation is done through vfs. MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; CDB.ExtraClangFlags.insert(CDB.ExtraClangFlags.end(), {"-xc++", "-target", "x86_64-linux-unknown", "-m64", "--gcc-toolchain=/randomusr", "-stdlib=libstdc++"}); - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); // Just a random gcc version string SmallString<8> Version("4.9.3"); @@ -351,8 +361,10 @@ TEST_F(ClangdVFSTest, ForceReparseCompileCommand) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); const auto SourceContents1 = R"cpp( @@ -387,8 +399,10 @@ TEST_F(ClangdVFSTest, ForceReparseCompileCommandDefines) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); const auto SourceContents = R"cpp( @@ -440,7 +454,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; MultipleErrorCheckingDiagConsumer DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); auto BarCpp = testPath("bar.cpp"); @@ -483,8 +499,10 @@ TEST_F(ClangdVFSTest, MemoryUsage) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); Path FooCpp = testPath("foo.cpp"); const auto SourceContents = R"cpp( @@ -518,9 +536,11 @@ TEST_F(ClangdVFSTest, InvalidCompileCommand) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); // clang cannot create CompilerInvocation if we pass two files in the @@ -635,9 +655,11 @@ ReqStats.emplace_back(); TestDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; { MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); // Prepare some random distributions for the test. std::random_device RandGen; @@ -770,8 +792,10 @@ TEST_F(ClangdVFSTest, CheckSourceHeaderSwitch) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto SourceContents = R"cpp( #include "foo.h" @@ -895,8 +919,10 @@ std::future StartSecond = StartSecondPromise.get_future(); NoConcurrentAccessDiagConsumer DiagConsumer(std::move(StartSecondPromise)); + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); Server.addDocument(FooCpp, SourceContentsWithErrors); StartSecond.wait(); Server.addDocument(FooCpp, SourceContentsWithoutErrors); @@ -907,8 +933,10 @@ TEST_F(ClangdVFSTest, FormatCode) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto Path = testPath("foo.cpp"); std::string Code = R"cpp( @@ -936,8 +964,10 @@ TEST_F(ClangdVFSTest, ChangedHeaderFromISystem) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto SourcePath = testPath("source/foo.cpp"); auto HeaderPath = testPath("headers/foo.h"); @@ -1011,8 +1041,10 @@ llvm::StringMap CountStats; ListenStatsFSProvider FS(CountStats); ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto SourcePath = testPath("foo.cpp"); auto HeaderPath = testPath("foo.h"); @@ -1040,6 +1072,7 @@ TEST_F(ClangdVFSTest, FlagsWithPlugins) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; CDB.ExtraClangFlags = { "-Xclang", @@ -1048,7 +1081,8 @@ "random-plugin", }; OverlayCDB OCDB(&CDB); - ClangdServer Server(OCDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(OCDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); const auto SourceContents = "int main() { return 0; }"; @@ -1062,11 +1096,13 @@ TEST_F(ClangdVFSTest, FallbackWhenPreambleIsNotReady) { MockFSProvider FS; ErrorCheckingDiagConsumer DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); - Annotations Code(R"cpp( + Annotations Code(R"cpp( namespace ns { int xyz; } using namespace ns; int main() { @@ -1127,7 +1163,9 @@ Notification CanReturnCommand; DelayedCompilationDatabase CDB(CanReturnCommand); - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); Annotations Code(R"cpp( diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp --- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp +++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp @@ -136,7 +136,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); return completions(Server, Text, std::move(IndexSymbols), std::move(Opts), FilePath); } @@ -582,7 +584,9 @@ FS.Files[BarHeader] = ""; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto BarURI = URI::create(BarHeader).toString(); Symbol Sym = cls("ns::X"); Sym.CanonicalDeclaration.FileURI = BarURI.c_str(); @@ -599,10 +603,10 @@ CodeCompleteOptions NoInsertion; NoInsertion.InsertIncludes = CodeCompleteOptions::NeverInsert; Results = completions(Server, - R"cpp( + R"cpp( int main() { ns::^ } )cpp", - {Sym}, NoInsertion); + {Sym}, NoInsertion); EXPECT_THAT(Results.Completions, ElementsAre(AllOf(Named("X"), Not(InsertInclude())))); // Duplicate based on inclusions in preamble. @@ -621,7 +625,9 @@ MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); Symbol SymX = cls("ns::X"); Symbol SymY = cls("ns::Y"); std::string BarHeader = testPath("bar.h"); @@ -649,7 +655,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); FS.Files[testPath("bar.h")] = R"cpp(namespace ns { struct preamble { int member; }; })cpp"; @@ -699,9 +707,10 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; ClangdServer::Options Opts = ClangdServer::optsForTest(); Opts.BuildDynamicSymbolIndex = true; - ClangdServer Server(CDB, FS, DiagConsumer, Opts); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, Opts); FS.Files[testPath("foo_header.h")] = R"cpp( #pragma once @@ -732,9 +741,10 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; auto Opts = ClangdServer::optsForTest(); Opts.BuildDynamicSymbolIndex = true; - ClangdServer Server(CDB, FS, DiagConsumer, Opts); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, Opts); FS.Files[testPath("foo.h")] = R"cpp( namespace ns { class XYZ {}; void foo(int x) {} } @@ -910,10 +920,11 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; ClangdServer::Options Opts = ClangdServer::optsForTest(); Opts.StaticIndex = Index.get(); - ClangdServer Server(CDB, FS, DiagConsumer, Opts); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, Opts); auto File = testPath("foo.cpp"); runAddDocument(Server, File, Text); return llvm::cantFail(runSignatureHelp(Server, File, Point)); @@ -1333,7 +1344,9 @@ MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); Annotations Source(R"cpp( #include "foo.h" @@ -1368,7 +1381,9 @@ MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); Annotations Source(R"cpp( // We ignore namespace comments, for rationale see CodeCompletionStrings.h. @@ -1433,10 +1448,12 @@ MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockFSProvider FS; FS.Files[FooCpp] = "// empty file"; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); // Run completion outside the file range. Position Pos; Pos.line = 100; @@ -1557,7 +1574,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); CodeCompleteOptions Opts; Opts.IncludeFixIts = true; @@ -1597,7 +1616,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); CodeCompleteOptions Opts; Opts.IncludeFixIts = true; @@ -1677,7 +1698,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); constexpr const char *TestCodes[] = { R"cpp( @@ -1835,9 +1858,10 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; ClangdServer::Options Opts = ClangdServer::optsForTest(); Opts.BuildDynamicSymbolIndex = true; - ClangdServer Server(CDB, FS, DiagConsumer, Opts); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, Opts); FS.Files[testPath("foo.h")] = R"cpp( struct Foo { @@ -1969,19 +1993,19 @@ TEST(GuessCompletionPrefix, Filters) { for (llvm::StringRef Case : { - "[[scope::]][[ident]]^", - "[[]][[]]^", - "\n[[]][[]]^", - "[[]][[ab]]^", - "x.[[]][[ab]]^", - "x.[[]][[]]^", - "[[x::]][[ab]]^", - "[[x::]][[]]^", - "[[::x::]][[ab]]^", - "some text [[scope::more::]][[identif]]^ier", - "some text [[scope::]][[mor]]^e::identifier", - "weird case foo::[[::bar::]][[baz]]^", - }) { + "[[scope::]][[ident]]^", + "[[]][[]]^", + "\n[[]][[]]^", + "[[]][[ab]]^", + "x.[[]][[ab]]^", + "x.[[]][[]]^", + "[[x::]][[ab]]^", + "[[x::]][[]]^", + "[[::x::]][[ab]]^", + "some text [[scope::more::]][[identif]]^ier", + "some text [[scope::]][[mor]]^e::identifier", + "weird case foo::[[::bar::]][[baz]]^", + }) { Annotations F(Case); auto Offset = cantFail(positionToOffset(F.code(), F.point())); auto ToStringRef = [&](Range R) { @@ -2004,7 +2028,9 @@ MockFSProvider FS; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto File = testPath("foo.cpp"); Annotations Test(R"cpp( @@ -2066,7 +2092,9 @@ FS.Files[FooHeader] = ""; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); std::string DeclFile = URI::create(testPath("foo")).toString(); Symbol sym = func("Func"); @@ -2097,7 +2125,9 @@ std::string FooHeader = testPath("foo.h"); FS.Files[FooHeader] = "#define CLANGD_PREAMBLE_HEADER x\n"; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto Results = completions( R"cpp(#include "foo.h" #define CLANGD_PREAMBLE_MAIN x @@ -2223,7 +2253,9 @@ std::string BarHeader = testPath("sub/bar.h"); FS.Files[BarHeader] = ""; IgnoreDiagnostics DiagConsumer; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto Results = completions(Server, R"cpp( #include "^" diff --git a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp --- a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp +++ b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp @@ -58,7 +58,7 @@ class WorkspaceSymbolsTest : public ::testing::Test { public: WorkspaceSymbolsTest() - : Server(CDB, FSProvider, DiagConsumer, optsForTests()) { + : Server(CDB, FSProvider, DiagConsumer, SHConsumer, optsForTests()) { // Make sure the test root directory is created. FSProvider.Files[testPath("unused")] = ""; CDB.ExtraClangFlags = {"-xc++"}; @@ -68,6 +68,7 @@ MockFSProvider FSProvider; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; ClangdServer Server; int Limit = 0; @@ -321,12 +322,13 @@ class DocumentSymbolsTest : public ::testing::Test { public: DocumentSymbolsTest() - : Server(CDB, FSProvider, DiagConsumer, optsForTests()) {} + : Server(CDB, FSProvider, DiagConsumer, SHConsumer, optsForTests()) {} protected: MockFSProvider FSProvider; MockCompilationDatabase CDB; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; ClangdServer Server; std::vector getSymbols(PathRef File) { @@ -414,21 +416,22 @@ AllOf(WithName("KInt"), WithKind(SymbolKind::Variable), Children()), AllOf(WithName("kStr"), WithKind(SymbolKind::Variable), Children()), AllOf(WithName("f1"), WithKind(SymbolKind::Function), Children()), - AllOf(WithName("foo"), WithKind(SymbolKind::Namespace), - Children( - AllOf(WithName("int32"), WithKind(SymbolKind::Class), - Children()), - AllOf(WithName("int32_t"), WithKind(SymbolKind::Class), - Children()), - AllOf(WithName("v1"), WithKind(SymbolKind::Variable), - Children()), - AllOf(WithName("bar"), WithKind(SymbolKind::Namespace), - Children(AllOf(WithName("v2"), - WithKind(SymbolKind::Variable), - Children()))), - AllOf(WithName("baz"), WithKind(SymbolKind::Namespace), - Children()), - AllOf(WithName("v2"), WithKind(SymbolKind::Namespace))))})); + AllOf( + WithName("foo"), WithKind(SymbolKind::Namespace), + Children( + AllOf(WithName("int32"), WithKind(SymbolKind::Class), + Children()), + AllOf(WithName("int32_t"), WithKind(SymbolKind::Class), + Children()), + AllOf(WithName("v1"), WithKind(SymbolKind::Variable), + Children()), + AllOf(WithName("bar"), WithKind(SymbolKind::Namespace), + Children(AllOf(WithName("v2"), + WithKind(SymbolKind::Variable), + Children()))), + AllOf(WithName("baz"), WithKind(SymbolKind::Namespace), + Children()), + AllOf(WithName("v2"), WithKind(SymbolKind::Namespace))))})); } TEST_F(DocumentSymbolsTest, DeclarationDefinition) { diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -0,0 +1,407 @@ +//===-- SemanticHighlightingTests.cpp----------------------*- C++ -*-------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "Annotations.h" +#include "SemanticHighlighting.h" +#include "TestTU.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include + +namespace clang { +namespace clangd { + +namespace { + +std::string kindAsString(SemanticHighlightingKind Kind) { + switch (Kind) { + case SemanticHighlightingKind::Class: + return "Class"; + case SemanticHighlightingKind::Enum: + return "Enum"; + case SemanticHighlightingKind::EnumClass: + return "EnumClass"; + case SemanticHighlightingKind::Enumerator: + return "Enumerator"; + case SemanticHighlightingKind::Field: + return "Field"; + case SemanticHighlightingKind::Function: + return "Function"; + case SemanticHighlightingKind::FunctionDeclaration: + return "FunctionDeclaration"; + case SemanticHighlightingKind::GlobalVariable: + return "GlobalVariable"; + case SemanticHighlightingKind::Label: + return "Label"; + case SemanticHighlightingKind::LocalVariable: + return "LocalVariable"; + case SemanticHighlightingKind::LocalVariableDeclaration: + return "LocalVariableDeclaration"; + case SemanticHighlightingKind::Macro: + return "Macro"; + case SemanticHighlightingKind::MacroDefinition: + return "MacroDefinition"; + case SemanticHighlightingKind::Method: + return "Method"; + case SemanticHighlightingKind::MethodDeclaration: + return "MethodDeclaration"; + case SemanticHighlightingKind::Namespace: + return "Namespace"; + case SemanticHighlightingKind::ParameterVariable: + return "ParameterVariable"; + case SemanticHighlightingKind::StaticField: + return "StaticField"; + case SemanticHighlightingKind::StaticMethod: + return "StaticMethod"; + case SemanticHighlightingKind::StaticMethodDeclaration: + return "StaticMethodDeclaration"; + case SemanticHighlightingKind::TemplateParameter: + return "TemplateParameter"; + case SemanticHighlightingKind::Typedef: + return "Typedef"; + case SemanticHighlightingKind::Special_LastKind: + return ""; + } + return ""; +} + +} // namespace + +// Teach gtest how to print a SemanticHighlightingToken to make test debugging +// easier. These need to not be in an anonymous namespace for gtest to find +// them. +std::ostream &operator<<(std::ostream &OS, const Position &Pos) { + return OS << Pos.line << ":" << Pos.character; +} +std::ostream &operator<<(std::ostream &OS, const Range &R) { + return OS << "[" << R.start << "," << R.end << "]"; +} +std::ostream &operator<<(std::ostream &OS, + const SemanticHighlightingToken &Token) { + return OS << kindAsString(Token.Kind) << " @ " << Token.R; +} + +namespace { + +using Highlightings = std::vector; + +Highlightings collectExpectedHighlightings(const Annotations &Source) { + Highlightings Result; + for (int Index = 0; Index < (int)SemanticHighlightingKind::Special_LastKind; + ++Index) { + auto Kind = static_cast(Index); + for (const auto &Range : Source.ranges(kindAsString(Kind))) { + Result.push_back({Range, Kind}); + } + } + return Result; +} + +void compareHighlightings(Highlightings &Actual, Highlightings &Expected) { + std::sort(Actual.begin(), Actual.end()); + std::sort(Expected.begin(), Expected.end()); + EXPECT_EQ(Actual, Expected); +} + +void checkHighlightings(llvm::StringRef AnnotatedSource) { + Annotations Source(AnnotatedSource); + TestTU TU = TestTU::withCode(Source.code()); + auto AST = TU.build(); + // dumpAST(AST, llvm::errs()); + // Check that the AST has no errors. This is mostly for catching mistakes. + // If we later want to test highlighting of invalid code, we can add an + // option to bypass this check. + for (auto D : AST.getDiagnostics()) { + llvm::errs() << D << "\n"; // For debugging if the assert below fails. + } + ASSERT_TRUE(AST.getDiagnostics().empty()); + auto ActualHighlightings = computeSemanticHighlightings(AST); + auto ExpectedHighlightings = collectExpectedHighlightings(Source); + compareHighlightings(ActualHighlightings, ExpectedHighlightings); +} + +TEST(SemanticHighlightings, Basics) { + checkHighlightings(R"cpp( +class $Class[[Waldo]] { + int $MethodDeclaration[[find]](int $ParameterVariable[[Param]]); +}; +$Class[[Waldo]] $GlobalVariable[[W]]; +)cpp"); +} + +TEST(SemanticHighlightings, UnnamedParameters) { + // Test that we do not get highlightings for unnamed parameters. + checkHighlightings(R"cpp( + void $FunctionDeclaration[[foo]](int); // unnamed function parameter + template class $Class[[Foo]] {}; // unnamed template parameter +)cpp"); +} + +TEST(SemanticHighlightings, AnonymousTypes) { + // Test that we do not get highlightings for anonymous types. + checkHighlightings(R"cpp( + struct { } $GlobalVariable[[S]]; // unnamed structure +)cpp"); +} + +TEST(SemanticHighlightings, Templates) { + checkHighlightings(R"cpp( +template +void $FunctionDeclaration[[foo]]($TemplateParameter[[T]]); +class $Class[[C]] { + template + void $MethodDeclaration[[bar]]($TemplateParameter[[T]]); +}; +)cpp"); +} + +TEST(SemanticHighlightings, DependentMethodCall) { + checkHighlightings(R"cpp( +class $Class[[C]] { + template + void $MethodDeclaration[[bar]]($TemplateParameter[[T]]); +}; + +template +void $FunctionDeclaration[[foo]]($TemplateParameter[[U]] $ParameterVariable[[u]]) { + $Class[[C]]().$Method[[bar]]($ParameterVariable[[u]]); +} +)cpp"); +} + +TEST(SemanticHighlightings, DependentFunctionCall) { + checkHighlightings(R"cpp( +template +void $FunctionDeclaration[[foo]]($TemplateParameter[[T]] $ParameterVariable[[t]]) { + $Function[[bar]]($ParameterVariable[[t]]); +} +)cpp"); +} + +TEST(SemanticHighlightings, AlignmentSpecifier) { + checkHighlightings(R"cpp( +struct $Class[[S]] { double $Field[[x]]; }; +alignas($Class[[S]]) int $GlobalVariable[[y]]; +)cpp"); +} + +TEST(SemanticHighlightings, AliasTemplates) { + checkHighlightings(R"cpp( +template +struct $Class[[Pair]] { }; + +template +using $Typedef[[PairIntX]] = $Class[[Pair]]; + +struct $Class[[Waldo]] { }; + +int $FunctionDeclaration[[main]]() { + $Typedef[[PairIntX]]<$Class[[Waldo]]> $LocalVariableDeclaration[[pair]]; +} +)cpp"); +} + +TEST(SemanticHighlightings, QualifiedEnum) { + checkHighlightings(R"cpp( +namespace $Namespace[[N]] { + struct $Class[[C]] { + enum $Enum[[E1]] { }; + enum class $EnumClass[[EC1]] { }; + }; + $Class[[C]]::$Enum[[E1]] $GlobalVariable[[e1]]; + $Class[[C]]::$EnumClass[[EC1]] $GlobalVariable[[ec1]]; + enum $Enum[[E2]] { }; + enum class $EnumClass[[EC2]] { }; +} +$Namespace[[N]]::$Class[[C]]::$Enum[[E1]] $GlobalVariable[[e1]]; +$Namespace[[N]]::$Class[[C]]::$EnumClass[[EC1]] $GlobalVariable[[ec1]]; +$Namespace[[N]]::$Enum[[E2]] $GlobalVariable[[e2]]; +$Namespace[[N]]::$EnumClass[[EC2]] $GlobalVariable[[ec2]]; +)cpp"); +} + +TEST(SemanticHighlightings, InheritingConstructor) { + checkHighlightings(R"cpp( +class $Class[[Base]] { }; +class $Class[[Derived]] : $Class[[Base]] { + // FIXME: Would like the second one to be Method. + using $Class[[Base]]::$Class[[Base]]; +}; +)cpp"); +} + +TEST(SemanticHighlightings, LambdaCaptures) { + checkHighlightings(R"cpp( +void $FunctionDeclaration[[foo]](int $ParameterVariable[[param]]) { + int $LocalVariableDeclaration[[local]]; + [$LocalVariable[[local]], $ParameterVariable[[param]]](){}(); +} +)cpp"); +} + +TEST(SemanticHighlightings, DependentType) { + checkHighlightings(R"cpp( +template +void $FunctionDeclaration[[foo]](typename $TemplateParameter[[T]]::$Class[[Type]] $ParameterVariable[[param]] + = $TemplateParameter[[T]]::$StaticField[[val]]); +)cpp"); +} + +// Disabled as this requires heuristic resolution. +TEST(SemanticHighlightings, DISABLED_DependentEnum) { + checkHighlightings(R"cpp( +template +struct $Class[[Base]] { + enum $Enum[[E]] { $Enumerator[[A]] }; + enum class $EnumClass[[F]] { $Enumerator[[B]] }; +}; +template +struct $Class[[Derived]] : $Class[[Base]]<$TemplateParameter[[T]]> { + static typename $Class[[Base]]<$TemplateParameter[[T]]>::$Enum[[E]] $StaticField[[x]] + = $Class[[Base]]<$TemplateParameter[[T]]>::$Enumerator[[A]]; + static typename $Class[[Base]]<$TemplateParameter[[T]]>::$EnumClass[[F]] $StaticField[[y]] + = $Class[[Base]]<$TemplateParameter[[T]]>::$EnumClass[[F]]::$Enumerator[[B]]; +}; +)cpp"); +} + +TEST(SemanticHighlightings, ExplicitSpec) { + checkHighlightings(R"cpp( +template class $Class[[C]]; +template <> class $Class[[C]] {}; +)cpp"); +} + +TEST(SemanticHighlightings, TemplateMetaprogramming) { + checkHighlightings(R"cpp( +template +struct $Class[[IndexTuple]] { + using $Typedef[[Next]] = $Class[[IndexTuple]]<$TemplateParameter[[Indexes]]..., sizeof...($TemplateParameter[[Indexes]])>; +}; +template +struct $Class[[BuildIndexTuple]] { + using $Typedef[[Type]] = typename $Class[[BuildIndexTuple]]<$TemplateParameter[[N]] - 1>::$Class[[Type]]::$Class[[Next]]; +}; +template <> +struct $Class[[BuildIndexTuple]]<0> { + using $Typedef[[Type]] = $Class[[IndexTuple]]<>; +}; +)cpp"); +} + +// Disabled due to a clang bug +// (see http://lists.llvm.org/pipermail/clangd-dev/2019-April/000400.html). +TEST(SemanticHighlightings, DISABLED_VariableTemplates) { + checkHighlightings(R"cpp( +template +const bool $GlobalVariable[[templ]] = true; +struct $Class[[A]] { }; +bool $GlobalVariable[[x]] = $GlobalVariable[[templ]]<$Class[[A]]>; +struct $Class[[S]] { + template + static const bool $StaticField[[templ]] = true; + void $MethodDeclaration[[bar]]() { + bool $LocalVariableDeclaration[[y]] = $StaticField[[templ]]<$Class[[A]]>; + } +}; +)cpp"); +} + +TEST(SemanticHighlightings, FunctionTemplateSpec) { + checkHighlightings(R"cpp( +struct $Class[[S]] { }; +template void $FunctionDeclaration[[Waldo]]() { } +template <> void $FunctionDeclaration[[Waldo]]<$Class[[S]]>() { } +)cpp"); +} + +TEST(SemanticHighlightings, OverloadedOperator) { + checkHighlightings(R"cpp( +struct $Class[[S]] { + int $Field[[waldo]]; +}; +struct $Class[[Iter]] { + // FIXME: Would like the operator symbol included in the highlighting + // for the method declaration, not just the operator keyword. + $Class[[S]] $MethodDeclaration[[operator]]*(); +}; +int $FunctionDeclaration[[main]]() { + $Class[[Iter]] $LocalVariableDeclaration[[it]]; + // FIXME: Once we support highlighting of overloaded operators, + // there should be an overloded operator highlighting on the next line. + (void)(1 + (*$LocalVariable[[it]]).$Field[[waldo]]); +} +)cpp"); +} + +TEST(SemanticHighlightings, MemInitializerList) { + checkHighlightings(R"cpp( +struct $Class[[S]] { + int $Field[[field]]; + + $MethodDeclaration[[S]](int $ParameterVariable[[field]]) + : $Field[[field]]($ParameterVariable[[field]]) {} +}; + )cpp"); +} + +TEST(SemanticHighlightings, InjectedClassName) { + checkHighlightings(R"cpp( +template +struct $Class[[S]] { + void $MethodDeclaration[[foo]]($Class[[S]]); +}; + )cpp"); +} + +TEST(SemanticHighlightings, DependentField) { + checkHighlightings(R"cpp( +template +void $FunctionDeclaration[[foo]]($TemplateParameter[[T]] $ParameterVariable[[t]]) { + $ParameterVariable[[t]].$Field[[field]]; +} + )cpp"); +} + +TEST(SemanticHighlightings, DependentStaticField) { + checkHighlightings(R"cpp( +template +int $FunctionDeclaration[[foo]]() { + return $TemplateParameter[[T]]::$StaticField[[field]]; +} + +template +struct $Class[[S]] { + void $MethodDeclaration[[bar]]() { + // For some reason, clang parses this differently than the free function case + // (CXXDependentScopeMemberExpr vs. DependentScopeDeclRefExpr). + return $TemplateParameter[[T]]::$StaticField[[field]]; + } +}; + )cpp"); +} + +TEST(SemanticHighlightings, StaticMethodWithDependentArguments) { + checkHighlightings(R"cpp( +struct $Class[[A]] { + template + static void $StaticMethodDeclaration[[foo]]($TemplateParameter[[T]]); +}; + +template +struct $Class[[B]] { + void $MethodDeclaration[[bar]]() { + $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]()); + } +}; + )cpp"); +} + +} // namespace +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -684,7 +684,9 @@ } CaptureTUStatus; MockFSProvider FS; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, CaptureTUStatus, ClangdServer::optsForTest()); + NoopSemanticHighlightingConsumer SHConsumer; + ClangdServer Server(CDB, FS, CaptureTUStatus, SHConsumer, + ClangdServer::optsForTest()); Annotations Code("int m^ain () {}"); // We schedule the following tasks in the queue: diff --git a/clang-tools-extra/clangd/unittests/TestFS.h b/clang-tools-extra/clangd/unittests/TestFS.h --- a/clang-tools-extra/clangd/unittests/TestFS.h +++ b/clang-tools-extra/clangd/unittests/TestFS.h @@ -57,6 +57,12 @@ StringRef RelPathPrefix; }; +class NoopSemanticHighlightingConsumer : public SemanticHighlightingConsumer { +public: + void onSemanticHighlightingReady( + PathRef, std::vector) override {} +}; + // Returns an absolute (fake) test directory for this OS. const char *testRoot(); diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp --- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp +++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp @@ -524,8 +524,10 @@ MockCompilationDatabase CDB(BuildDir, RelPathPrefix); IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockFSProvider FS; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); // Fill the filesystem. auto FooCpp = testPath("src/foo.cpp"); @@ -1194,8 +1196,10 @@ TEST(GoToInclude, All) { MockFSProvider FS; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); const char *SourceContents = R"cpp( @@ -1269,8 +1273,10 @@ // good preamble. MockFSProvider FS; IgnoreDiagnostics DiagConsumer; + NoopSemanticHighlightingConsumer SHConsumer; MockCompilationDatabase CDB; - ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest()); + ClangdServer Server(CDB, FS, DiagConsumer, SHConsumer, + ClangdServer::optsForTest()); auto FooCpp = testPath("foo.cpp"); // The trigger locations must be the same.