diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -136,6 +136,8 @@ DiagnosticToReplacementMap; /// Caches FixIts per file and diagnostics llvm::StringMap FixItsMap; + std::mutex HighlightingsMutex; + llvm::StringMap> FileToHighlightings; // Most code should not deal with Transport directly. // MessageHandler deals with incoming messages, use call() etc for outgoing. diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -452,6 +452,12 @@ void ClangdLSPServer::onDocumentDidOpen( const DidOpenTextDocumentParams &Params) { PathRef File = Params.textDocument.uri.file(); + { + // If a file was closed before highlightings were generated for the last + // edit there are stale previous highlightings. + std::lock_guard Lock(HighlightingsMutex); + FileToHighlightings.erase(File); + } const std::string &Contents = Params.textDocument.text; @@ -601,6 +607,10 @@ std::lock_guard Lock(FixItsMutex); FixItsMap.erase(File); } + { + std::lock_guard HLock(HighlightingsMutex); + FileToHighlightings.erase(File); + } // clangd will not send updates for this file anymore, so we empty out the // list of diagnostics shown on the client (e.g. in the "Problems" pane of // VSCode). Note that this cannot race with actual diagnostics responses @@ -1099,9 +1109,21 @@ void ClangdLSPServer::onHighlightingsReady( PathRef File, std::vector Highlightings) { + std::vector Old; + { + std::lock_guard Lock(HighlightingsMutex); + Old = FileToHighlightings[File]; + } + // LSP allows us to send incremental edits of highlightings. Also need to diff + // to remove highlightings from tokens that should no longer have them. + std::vector Diffed = diffHighlightings(Highlightings, Old); publishSemanticHighlighting( {{URIForFile::canonicalize(File, /*TUPath=*/File)}, - toSemanticHighlightingInformation(Highlightings)}); + toSemanticHighlightingInformation(Diffed)}); + { + std::lock_guard Lock(HighlightingsMutex); + FileToHighlightings[File] = std::move(Highlightings); + } } void ClangdLSPServer::onDiagnosticsReady(PathRef File, diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -43,7 +43,16 @@ Range R; }; -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs); +bool operator==(const HighlightingToken &L, const HighlightingToken &R); +bool operator<(const HighlightingToken &L, const HighlightingToken &R); + +/// Contains all information about highlightings on a single line. +struct LineHighlightings { + int Line; + std::vector Tokens; +}; + +bool operator==(const LineHighlightings &L, const LineHighlightings &R); // Returns all HighlightingTokens from an AST. Only generates highlights for the // main AST. @@ -53,9 +62,22 @@ /// (https://manual.macromates.com/en/language_grammars). llvm::StringRef toTextMateScope(HighlightingKind Kind); -// Convert to LSP's semantic highlighting information. +/// Convert to LSP's semantic highlighting information. std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens); +toSemanticHighlightingInformation(llvm::ArrayRef Tokens); + +// Return a line-by-line diff between two highlightings. +// - if the tokens on a line are the same in both hightlightings this line is +// omitted. +// - if a line exists in NewHighligtings but not in OldHighligtings the tokens +// on this line is emitted., we emit the tokens on this line +// - if a line not exists in NewHighligtings but in OldHighligtings an empty +// line is emitted (to tell client to clear the previous highlightings on this +// line) +/// REQUIRED: OldHighlightings and NewHighlightings are sorted. +std::vector +diffHighlightings(ArrayRef NewHighlightings, + ArrayRef OldHighlightings); } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -33,10 +33,7 @@ TraverseAST(Ctx); // Initializer lists can give duplicates of tokens, therefore all tokens // must be deduplicated. - llvm::sort(Tokens, - [](const HighlightingToken &L, const HighlightingToken &R) { - return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); - }); + llvm::sort(Tokens); auto Last = std::unique(Tokens.begin(), Tokens.end()); Tokens.erase(Last, Tokens.end()); return Tokens; @@ -260,10 +257,58 @@ llvm::support::endian::write16be(Buf.data(), I); OS.write(Buf.data(), Buf.size()); } + +// Get the highlightings on \c Line where the first entry of line is at \c +// StartLineIt. If it is not at \c StartLineIt an empty vector is returned. +ArrayRef +takeLine(ArrayRef AllTokens, + ArrayRef::iterator StartLineIt, int Line) { + return ArrayRef(StartLineIt, AllTokens.end()) + .take_while([Line](const HighlightingToken &Token) { + return Token.R.start.line == Line; + }); +} } // namespace -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) { - return Lhs.Kind == Rhs.Kind && Lhs.R == Rhs.R; +std::vector +diffHighlightings(ArrayRef NewHighlightings, + ArrayRef OldHighlightings) { + // FIXME: There's an edge case when tokens span multiple lines. If the first + // token on the line started on a line above the current one and the rest of + // the line is the equal to the previous one than we will remove all + // highlights but the ones for the token spanning multiple lines. This means + // that when we get into the LSP layer the only highlights that will be + // visible are the ones for the token spanning multiple lines. + // Example: + // EndOfMultilineToken Token Token Token + // If "Token Token Token" don't differ from previously the line is + // incorrectly removed. + std::vector LineTokens; + // ArrayRefs to the current line in the highlights. + ArrayRef NewLine(NewHighlightings.begin(), + /*length*/0UL); + ArrayRef OldLine(OldHighlightings.begin(), + /*length*/0UL); + auto NewEnd = NewHighlightings.end(); + auto OldEnd = OldHighlightings.end(); + for (int Line = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd; ++Line) { + NewLine = takeLine(NewHighlightings, NewLine.end(), Line); + OldLine = takeLine(OldHighlightings, OldLine.end(), Line); + if (NewLine != OldLine) + LineTokens.push_back({Line, NewLine}); + } + + return LineTokens; +} + +bool operator==(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind); +} +bool operator<(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); +} +bool operator==(const LineHighlightings &L, const LineHighlightings &R) { + return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens); } std::vector getSemanticHighlightings(ParsedAST &AST) { @@ -271,22 +316,18 @@ } std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { +toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { if (Tokens.size() == 0) return {}; // FIXME: Tokens might be multiple lines long (block comments) in this case // this needs to add multiple lines for those tokens. - std::map> TokenLines; - for (const HighlightingToken &Token : Tokens) - TokenLines[Token.R.start.line].push_back(Token); - std::vector Lines; - Lines.reserve(TokenLines.size()); - for (const auto &Line : TokenLines) { + Lines.reserve(Tokens.size()); + for (const auto &Line : Tokens) { llvm::SmallVector LineByteTokens; llvm::raw_svector_ostream OS(LineByteTokens); - for (const auto &Token : Line.second) { + for (const auto &Token : Line.Tokens) { // Writes the token to LineByteTokens in the byte format specified by the // LSP proposal. Described below. // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->| @@ -297,7 +338,7 @@ write16be(static_cast(Token.Kind), OS); } - Lines.push_back({Line.first, encodeBase64(LineByteTokens)}); + Lines.push_back({Line.Line, encodeBase64(LineByteTokens)}); } return Lines; diff --git a/clang-tools-extra/clangd/test/semantic-highlighting.test b/clang-tools-extra/clangd/test/semantic-highlighting.test --- a/clang-tools-extra/clangd/test/semantic-highlighting.test +++ b/clang-tools-extra/clangd/test/semantic-highlighting.test @@ -49,6 +49,40 @@ # CHECK-NEXT: } # CHECK-NEXT:} --- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 0, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo2.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 6},"end": {"line": 0,"character": 6}},"rangeLength": 0,"text": "\nint y = 2;"}]}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- {"jsonrpc":"2.0","id":3,"method":"shutdown"} --- {"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -29,9 +29,12 @@ return Tokens; } -void checkHighlightings(llvm::StringRef Code) { - Annotations Test(Code); +std::vector getActualTokens(Annotations &Test) { auto AST = TestTU::withCode(Test.code()).build(); + return getSemanticHighlightings(AST); +} + +std::vector getExpectedTokens(Annotations &Test) { static const std::map KindToString{ {HighlightingKind::Variable, "Variable"}, {HighlightingKind::Function, "Function"}, @@ -48,9 +51,43 @@ Test.ranges(KindString.second), KindString.first); ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end()); } + llvm::sort(ExpectedTokens); + return ExpectedTokens; +} - auto ActualTokens = getSemanticHighlightings(AST); - EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens)); +std::vector getExpectedEmptyLines(Annotations &Test) { + auto EmptyRanges = Test.ranges("Empty"); + std::vector EmptyLines; + EmptyLines.reserve(EmptyRanges.size()); + for (const auto &EmptyRange : EmptyRanges) + EmptyLines.push_back(EmptyRange.start.line); + return EmptyLines; +} + +void checkHighlightings(llvm::StringRef Code) { + Annotations Test(Code); + EXPECT_THAT(getActualTokens(Test), + getExpectedTokens(Test)); +} + +void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) { + Annotations OldTest(OldCode); + Annotations NewTest(NewCode); + std::vector ExpectedLinePairHighlighting; + for (int Line : getExpectedEmptyLines(NewTest)) + ExpectedLinePairHighlighting.push_back({Line, {}}); + std::map> ExpectedLines; + for (const HighlightingToken &Token : getExpectedTokens(NewTest)) + ExpectedLines[Token.R.start.line].push_back(Token); + for (auto &LineTokens : ExpectedLines) { + ExpectedLinePairHighlighting.push_back( + {LineTokens.first, LineTokens.second}); + } + + std::vector ActualDiffed = + diffHighlightings(getActualTokens(NewTest), getActualTokens(OldTest)); + EXPECT_THAT(ActualDiffed, + testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting)); } TEST(SemanticHighlighting, GetsCorrectTokens) { @@ -251,21 +288,110 @@ return Pos; }; - std::vector Tokens{ - {HighlightingKind::Variable, - Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, - {HighlightingKind::Function, - Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, - {HighlightingKind::Variable, - Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}; + std::vector Tokens{ + {3, + {{HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}}}, + {1, + {{HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}}}; std::vector ActualResults = toSemanticHighlightingInformation(Tokens); std::vector ExpectedResults = { - {1, "AAAAAQAEAAA="}, - {3, "AAAACAAEAAAAAAAEAAMAAQ=="}}; + {3, "AAAACAAEAAAAAAAEAAMAAQ=="}, {1, "AAAAAQAEAAA="}}; EXPECT_EQ(ActualResults, ExpectedResults); } +TEST(SemanticHighlighting, HighlightingDiffer) { + struct { + llvm::StringRef OldCode; + llvm::StringRef NewCode; + } TestCases[]{{ + R"cpp( + int A + double B; + struct C {}; + )cpp", + R"cpp( + int A; + double B; + struct C {}; + )cpp"}, + { + R"cpp( + struct Alpha { + double SomeVariable = 9483.301; + }; + struct Beta {}; + int A = 121; + Alpha Var; + )cpp", + R"cpp( + struct Alpha { + double SomeVariable = 9483.301; + }; + struct Beta {}; // Some comment + $Variable[[intA]] = 121; + $Class[[Beta]] $Variable[[Var]]; + )cpp"}, + { + R"cpp( + int A = 121; int B; + )cpp", + R"cpp( + $Variable[[intA]] = 121; int $Variable[[B]]; + )cpp"}, + { + R"cpp( + int A; + )cpp", + R"cpp( + struct $Class[[A]]; + )cpp"}, + { + R"cpp( + int ABBA = 23; + double C = 123.3 * 5; + )cpp", + R"cpp( + int ABBA = 23; + $Empty[[]]//double C = 123.3 * 5; + float $Variable[[H]]; + )cpp"}, + { + R"cpp( + int a; + int b; + int c; + )cpp", + R"cpp( + int a; + int $Variable[[new_b]]; + int c; + )cpp" + }, + { + R"cpp( + int a; + int b; + int c; + int d; + )cpp", + R"cpp( + int a; +/* $Empty[[]] int b; + $Empty[[]]int c;*/ + int d; + )cpp" + }}; + + for (const auto &Test : TestCases) { + checkDiffedHighlights(Test.OldCode, Test.NewCode); + } +} + } // namespace } // namespace clangd } // namespace clang