diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -1397,14 +1397,15 @@ void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params, Callback CB) { + auto File = Params.textDocument.uri.file(); Server->semanticHighlights( Params.textDocument.uri.file(), - [this, File(Params.textDocument.uri.file().str()), CB(std::move(CB))]( + [this, File(File.str()), CB(std::move(CB)), Code(Server->getDraft(File))]( llvm::Expected> HT) mutable { if (!HT) return CB(HT.takeError()); SemanticTokens Result; - Result.tokens = toSemanticTokens(*HT); + Result.tokens = toSemanticTokens(*HT, *Code); { std::lock_guard Lock(SemanticTokensMutex); auto &Last = LastSemanticTokens[File]; @@ -1420,14 +1421,15 @@ void ClangdLSPServer::onSemanticTokensDelta( const SemanticTokensDeltaParams &Params, Callback CB) { + auto File = Params.textDocument.uri.file(); Server->semanticHighlights( Params.textDocument.uri.file(), - [this, PrevResultID(Params.previousResultId), - File(Params.textDocument.uri.file().str()), CB(std::move(CB))]( + [this, PrevResultID(Params.previousResultId), File(File.str()), + CB(std::move(CB)), Code(Server->getDraft(File))]( llvm::Expected> HT) mutable { if (!HT) return CB(HT.takeError()); - std::vector Toks = toSemanticTokens(*HT); + std::vector Toks = toSemanticTokens(*HT, *Code); SemanticTokensOrDelta Result; { diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -21,6 +21,7 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H #include "Protocol.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/raw_ostream.h" namespace clang { @@ -101,7 +102,8 @@ // main AST. std::vector getSemanticHighlightings(ParsedAST &AST); -std::vector toSemanticTokens(llvm::ArrayRef); +std::vector toSemanticTokens(llvm::ArrayRef, + llvm::StringRef Code); llvm::StringRef toSemanticTokenType(HighlightingKind Kind); llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier); std::vector diffTokens(llvm::ArrayRef Before, diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -30,7 +30,9 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" #include namespace clang { @@ -918,7 +920,8 @@ } std::vector -toSemanticTokens(llvm::ArrayRef Tokens) { +toSemanticTokens(llvm::ArrayRef Tokens, + llvm::StringRef Code) { assert(std::is_sorted(Tokens.begin(), Tokens.end())); std::vector Result; const HighlightingToken *Last = nullptr; @@ -939,8 +942,22 @@ Out.deltaLine = Tok.R.start.line; Out.deltaStart = Tok.R.start.character; } - assert(Tok.R.end.line == Tok.R.start.line); - Out.length = Tok.R.end.character - Tok.R.start.character; + if (Tok.R.end.line == Tok.R.start.line) { + Out.length = Tok.R.end.character - Tok.R.start.character; + } else { + // If a tokens length is past the end of the line, it should be treated as + // if the token ends at the end of the line and will not wrap onto the + // next line. + // This is slow, but code rarely has multiline tokens. + // FIXME: There's a client capability for supporting multiline tokens, + // respect that. + auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start)); + auto LineEnd = Code.find('\n', TokStartOffset); + if (LineEnd == Code.npos) + Out.length = Code.size() - TokStartOffset; + else + Out.length = LineEnd - TokStartOffset; + } Out.tokenType = static_cast(Tok.Kind); Out.tokenModifiers = Tok.Modifiers; diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -944,7 +944,7 @@ )"); Tokens.front().Modifiers |= unsigned(HighlightingModifier::Declaration); Tokens.front().Modifiers |= unsigned(HighlightingModifier::Readonly); - auto Results = toSemanticTokens(Tokens); + auto Results = toSemanticTokens(Tokens, /*Code=*/""); ASSERT_THAT(Results, SizeIs(3)); EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable)); @@ -972,13 +972,15 @@ auto Before = toSemanticTokens(tokens(R"( [[foo]] [[bar]] [[baz]] [[one]] [[two]] [[three]] - )")); + )"), + /*Code=*/""); EXPECT_THAT(diffTokens(Before, Before), IsEmpty()); auto After = toSemanticTokens(tokens(R"( [[foo]] [[hello]] [[world]] [[baz]] [[one]] [[two]] [[three]] - )")); + )"), + /*Code=*/""); // Replace [bar, baz] with [hello, world, baz] auto Diff = diffTokens(Before, After); @@ -1000,6 +1002,23 @@ EXPECT_EQ(3u, Diff.front().tokens[2].length); } +TEST(SemanticHighlighting, MultilineTokens) { + llvm::StringRef AnnotatedCode = R"cpp( +[[fo +o]] [[bar]])cpp"; + auto Toks = toSemanticTokens(tokens(AnnotatedCode), + Annotations(AnnotatedCode).code()); + ASSERT_THAT(Toks, SizeIs(2)); + // foo + EXPECT_EQ(Toks[0].deltaLine, 1u); + EXPECT_EQ(Toks[0].deltaStart, 0u); + EXPECT_EQ(Toks[0].length, 2u); + + // bar + EXPECT_EQ(Toks[1].deltaLine, 1u); + EXPECT_EQ(Toks[1].deltaStart, 2u); + EXPECT_EQ(Toks[1].length, 3u); +} } // namespace } // namespace clangd } // namespace clang