diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -136,6 +136,8 @@ DiagnosticToReplacementMap; /// Caches FixIts per file and diagnostics llvm::StringMap FixItsMap; + std::mutex HighlightingsMutex; + llvm::StringMap> FileToHighlightings; // Most code should not deal with Transport directly. // MessageHandler deals with incoming messages, use call() etc for outgoing. diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -452,6 +452,12 @@ void ClangdLSPServer::onDocumentDidOpen( const DidOpenTextDocumentParams &Params) { PathRef File = Params.textDocument.uri.file(); + { + // If a file was closed before highlightings were generated for the last + // edit there are stale previous highlightings. + std::lock_guard Lock(HighlightingsMutex); + FileToHighlightings.erase(File); + } const std::string &Contents = Params.textDocument.text; @@ -601,6 +607,10 @@ std::lock_guard Lock(FixItsMutex); FixItsMap.erase(File); } + { + std::lock_guard HLock(HighlightingsMutex); + FileToHighlightings.erase(File); + } // clangd will not send updates for this file anymore, so we empty out the // list of diagnostics shown on the client (e.g. in the "Problems" pane of // VSCode). Note that this cannot race with actual diagnostics responses @@ -1099,9 +1109,21 @@ void ClangdLSPServer::onHighlightingsReady( PathRef File, std::vector Highlightings) { + std::vector Old; + { + std::lock_guard Lock(HighlightingsMutex); + Old = FileToHighlightings[File]; + } + // LSP allows us to send incremental edits of highlightings. Also need to diff + // to remove highlightings from tokens that should no longer have them. + std::vector Diffed = diffHighlightings(Highlightings, Old); publishSemanticHighlighting( {{URIForFile::canonicalize(File, /*TUPath=*/File)}, - toSemanticHighlightingInformation(Highlightings)}); + toSemanticHighlightingInformation(Diffed)}); + { + std::lock_guard Lock(HighlightingsMutex); + FileToHighlightings[File] = std::move(Highlightings); + } } void ClangdLSPServer::onDiagnosticsReady(PathRef File, diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -42,7 +42,16 @@ Range R; }; -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs); +bool operator==(const HighlightingToken &L, const HighlightingToken &R); +bool operator<(const HighlightingToken &L, const HighlightingToken &R); + +/// Contains all information about highlightings on a single line. +struct LineHighlightings { + int Line; + std::vector Tokens; +}; + +bool operator==(const LineHighlightings &L, const LineHighlightings &R); // Returns all HighlightingTokens from an AST. Only generates highlights for the // main AST. @@ -52,9 +61,19 @@ /// (https://manual.macromates.com/en/language_grammars). llvm::StringRef toTextMateScope(HighlightingKind Kind); -// Convert to LSP's semantic highlighting information. +/// Convert to LSP's semantic highlighting information. std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens); +toSemanticHighlightingInformation(llvm::ArrayRef Tokens); + +/// Removes every line where \c Highlightings is the same as \c +/// PrevHighlightings. If \c PrevHighlightings has lines that does not exist +/// in \c Highlightings an empty line is added. Returns the resulting +/// HighlightingTokens grouped by their line number. +/// +/// REQUIRED: OldHighlightings and NewHighlightings are sorted. +std::vector +diffHighlightings(ArrayRef NewHighlightings, + ArrayRef OldHighlightings); } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -33,10 +33,7 @@ TraverseAST(Ctx); // Initializer lists can give duplicates of tokens, therefore all tokens // must be deduplicated. - llvm::sort(Tokens, - [](const HighlightingToken &L, const HighlightingToken &R) { - return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); - }); + llvm::sort(Tokens); auto Last = std::unique(Tokens.begin(), Tokens.end()); Tokens.erase(Last, Tokens.end()); return Tokens; @@ -119,6 +116,8 @@ private: void addToken(SourceLocation Loc, const NamedDecl *D) { + if (D->isInvalidDecl()) + return; if (D->getDeclName().isIdentifier() && D->getName().empty()) // Don't add symbols that don't have any length. return; @@ -225,10 +224,58 @@ llvm::support::endian::write16be(Buf.data(), I); OS.write(Buf.data(), Buf.size()); } + +// Get the highlightings on \c Line where the first entry of line is at \c +// StartLineIt. If it is not at \c StartLineIt an empty vector is returned. +ArrayRef +takeLine(ArrayRef AllTokens, + ArrayRef::iterator StartLineIt, int Line) { + return ArrayRef(StartLineIt, AllTokens.end()) + .take_while([Line](const HighlightingToken &Token) { + return Token.R.start.line == Line; + }); +} } // namespace -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) { - return Lhs.Kind == Rhs.Kind && Lhs.R == Rhs.R; +std::vector +diffHighlightings(ArrayRef NewHighlightings, + ArrayRef OldHighlightings) { + // FIXME: There's an edge case when tokens span multiple lines. If the first + // token on the line started on a line above the current one and the rest of + // the line is the equal to the previous one than we will remove all + // highlights but the ones for the token spanning multiple lines. This means + // that when we get into the LSP layer the only highlights that will be + // visible are the ones for the token spanning multiple lines. + // Example: + // EndOfMultilineToken Token Token Token + // If "Token Token Token" don't differ from previously the line is + // incorrectly removed. + std::vector LineTokens; + // ArrayRefs to the current line in the highlights. + ArrayRef NewLine(NewHighlightings.begin(), + /*length*/0UL); + ArrayRef OldLine(OldHighlightings.begin(), + /*length*/0UL); + auto NewEnd = NewHighlightings.end(); + auto OldEnd = OldHighlightings.end(); + for (int Line = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd; ++Line) { + NewLine = takeLine(NewHighlightings, NewLine.end(), Line); + OldLine = takeLine(OldHighlightings, OldLine.end(), Line); + if (NewLine != OldLine) + LineTokens.push_back({Line, NewLine}); + } + + return LineTokens; +} + +bool operator==(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind); +} +bool operator<(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); +} +bool operator==(const LineHighlightings &L, const LineHighlightings &R) { + return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens); } std::vector getSemanticHighlightings(ParsedAST &AST) { @@ -236,22 +283,18 @@ } std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { +toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { if (Tokens.size() == 0) return {}; // FIXME: Tokens might be multiple lines long (block comments) in this case // this needs to add multiple lines for those tokens. - std::map> TokenLines; - for (const HighlightingToken &Token : Tokens) - TokenLines[Token.R.start.line].push_back(Token); - std::vector Lines; - Lines.reserve(TokenLines.size()); - for (const auto &Line : TokenLines) { + Lines.reserve(Tokens.size()); + for (const auto &Line : Tokens) { llvm::SmallVector LineByteTokens; llvm::raw_svector_ostream OS(LineByteTokens); - for (const auto &Token : Line.second) { + for (const auto &Token : Line.Tokens) { // Writes the token to LineByteTokens in the byte format specified by the // LSP proposal. Described below. // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->| @@ -262,7 +305,7 @@ write16be(static_cast(Token.Kind), OS); } - Lines.push_back({Line.first, encodeBase64(LineByteTokens)}); + Lines.push_back({Line.Line, encodeBase64(LineByteTokens)}); } return Lines; diff --git a/clang-tools-extra/clangd/test/semantic-highlighting.test b/clang-tools-extra/clangd/test/semantic-highlighting.test --- a/clang-tools-extra/clangd/test/semantic-highlighting.test +++ b/clang-tools-extra/clangd/test/semantic-highlighting.test @@ -46,6 +46,40 @@ # CHECK-NEXT: } # CHECK-NEXT:} --- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 0, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo2.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 6},"end": {"line": 0,"character": 6}},"rangeLength": 0,"text": "\nint y = 2;"}]}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- {"jsonrpc":"2.0","id":3,"method":"shutdown"} --- {"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -29,9 +29,12 @@ return Tokens; } -void checkHighlightings(llvm::StringRef Code) { - Annotations Test(Code); +std::vector getActualTokens(Annotations &Test) { auto AST = TestTU::withCode(Test.code()).build(); + return getSemanticHighlightings(AST); +} + +std::vector getExpectedTokens(Annotations &Test) { static const std::map KindToString{ {HighlightingKind::Variable, "Variable"}, {HighlightingKind::Function, "Function"}, @@ -47,11 +50,50 @@ Test.ranges(KindString.second), KindString.first); ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end()); } + return ExpectedTokens; +} - auto ActualTokens = getSemanticHighlightings(AST); +std::vector getExpectedEmptyLines(Annotations &Test) { + auto EmptyRanges = Test.ranges("Empty"); + std::vector EmptyLines(EmptyRanges.size()); + for (unsigned I = 0, End = EmptyRanges.size(); I < End; ++I) + EmptyLines[I] = EmptyRanges[I].start.line; + return EmptyLines; +} + +void checkHighlightings(llvm::StringRef Code) { + Annotations Test(Code); + std::vector ActualTokens = getActualTokens(Test); + std::vector ExpectedTokens = getExpectedTokens(Test); EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens)); } +void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) { + Annotations OldTest(OldCode); + Annotations NewTest(NewCode); + std::vector OldActualTokens = getActualTokens(OldTest); + std::vector NewActualTokens = getActualTokens(NewTest); + std::vector ExpectedTokens = getExpectedTokens(NewTest); + std::vector EmptyLines = getExpectedEmptyLines(NewTest); + std::vector ActualDiffed = + diffHighlightings(NewActualTokens, OldActualTokens); + + std::map> ExpectedLines; + for (const HighlightingToken &Token : ExpectedTokens) + ExpectedLines[Token.R.start.line].push_back(Token); + std::vector ExpectedLinePairHighlighting; + for (int Line : EmptyLines) + ExpectedLinePairHighlighting.push_back({Line, {}}); + for (auto &LineTokens : ExpectedLines) { + llvm::sort(LineTokens.second); + ExpectedLinePairHighlighting.push_back( + {LineTokens.first, LineTokens.second}); + } + + EXPECT_THAT(ActualDiffed, + testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting)); +} + TEST(SemanticHighlighting, GetsCorrectTokens) { const char *TestCases[] = { R"cpp( @@ -211,21 +253,82 @@ return Pos; }; - std::vector Tokens{ - {HighlightingKind::Variable, - Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, - {HighlightingKind::Function, - Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, - {HighlightingKind::Variable, - Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}; + std::vector Tokens{ + {3, + {{HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}}}, + {1, + {{HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}}}; std::vector ActualResults = toSemanticHighlightingInformation(Tokens); std::vector ExpectedResults = { - {1, "AAAAAQAEAAA="}, - {3, "AAAACAAEAAAAAAAEAAMAAQ=="}}; + {3, "AAAACAAEAAAAAAAEAAMAAQ=="}, {1, "AAAAAQAEAAA="}}; EXPECT_EQ(ActualResults, ExpectedResults); } +TEST(SemanticHighlighting, HighlightingDiffer) { + // The first entry is the old code. The second entry is the new code. + std::vector> TestCases{{ + R"cpp( + int A + double B; + struct C {}; + )cpp", + R"cpp( + int A; + double B; + struct C {}; + )cpp"}, + { + R"cpp( + struct Alpha { + double SomeVariable = 9483.301; + }; + struct Beta {}; + int A = 121; + Alpha Var; + )cpp", + R"cpp( + struct Alpha { + double SomeVariable = 9483.301; + }; + struct Beta {}; // Some comment + $Empty[[]]intA = 121; + $Class[[Beta]] $Variable[[Var]]; + )cpp"}, + { + R"cpp( + int A = 121; int B; + )cpp", + R"cpp( + intA = 121; int $Variable[[B]]; + )cpp"}, + { + R"cpp( + int A; + )cpp", + R"cpp( + struct $Class[[A]]; + )cpp"}, + { + R"cpp( + int ABBA = 23; + double C = 123.3 * 5; + )cpp", + R"cpp( + int ABBA = 23; + $Empty[[]]//double C = 123.3 * 5; + float $Variable[[H]]; + )cpp"}}; + + for (auto Test : TestCases) { + checkDiffedHighlights(Test.first, Test.second); + } +} + } // namespace } // namespace clangd } // namespace clang