Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.h +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h @@ -55,9 +55,9 @@ // Implement DiagnosticsConsumer. void onDiagnosticsReady(PathRef File, std::vector Diagnostics) override; void onFileUpdated(PathRef File, const TUStatus &Status) override; - void - onHighlightingsReady(PathRef File, - std::vector Highlightings) override; + void onHighlightingsReady(PathRef File, + std::vector Highlightings, + int NumLines) override; // LSP methods. Notifications have signature void(const Params&). // Calls have signature void(const Params&, Callback). @@ -138,6 +138,8 @@ DiagnosticToReplacementMap; /// Caches FixIts per file and diagnostics llvm::StringMap FixItsMap; + std::mutex HighlightingsMutex; + llvm::StringMap> FileToHighlightings; // Most code should not deal with Transport directly. // MessageHandler deals with incoming messages, use call() etc for outgoing. Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp @@ -615,6 +615,10 @@ std::lock_guard Lock(FixItsMutex); FixItsMap.erase(File); } + { + std::lock_guard HLock(HighlightingsMutex); + FileToHighlightings.erase(File); + } // clangd will not send updates for this file anymore, so we empty out the // list of diagnostics shown on the client (e.g. in the "Problems" pane of // VSCode). Note that this cannot race with actual diagnostics responses @@ -1113,10 +1117,21 @@ } void ClangdLSPServer::onHighlightingsReady( - PathRef File, std::vector Highlightings) { + PathRef File, std::vector Highlightings, int NumLines) { + std::vector Old; + std::vector HighlightingsCopy = Highlightings; + { + std::lock_guard Lock(HighlightingsMutex); + Old = std::move(FileToHighlightings[File]); + FileToHighlightings[File] = std::move(HighlightingsCopy); + } + // LSP allows us to send incremental edits of highlightings. Also need to diff + // to remove highlightings from tokens that should no longer have them. + std::vector Diffed = + diffHighlightings(Highlightings, Old, NumLines); publishSemanticHighlighting( {{URIForFile::canonicalize(File, /*TUPath=*/File)}, - toSemanticHighlightingInformation(Highlightings)}); + toSemanticHighlightingInformation(Diffed)}); } void ClangdLSPServer::onDiagnosticsReady(PathRef File, Index: clang-tools-extra/trunk/clangd/ClangdServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.h +++ clang-tools-extra/trunk/clangd/ClangdServer.h @@ -52,9 +52,12 @@ virtual void onFileUpdated(PathRef File, const TUStatus &Status){}; /// Called by ClangdServer when some \p Highlightings for \p File are ready. + /// \p NumLines are the number of lines in the file where the highlightings + /// where generated from. virtual void onHighlightingsReady(PathRef File, - std::vector Highlightings) {} + std::vector Highlightings, + int NumLines) {} }; /// When set, used by ClangdServer to get clang-tidy options for each particular Index: clang-tools-extra/trunk/clangd/ClangdServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdServer.cpp @@ -71,10 +71,19 @@ if (SemanticHighlighting) Highlightings = getSemanticHighlightings(AST); + // FIXME: We need a better way to send the maximum line number to the + // differ. + // The differ needs the information about the max number of lines + // to not send diffs that are outside the file. + const SourceManager &SM = AST.getSourceManager(); + FileID MainFileID = SM.getMainFileID(); + int NumLines = SM.getBufferData(MainFileID).count('\n') + 1; + Publish([&]() { DiagConsumer.onDiagnosticsReady(Path, std::move(Diagnostics)); if (SemanticHighlighting) - DiagConsumer.onHighlightingsReady(Path, std::move(Highlightings)); + DiagConsumer.onHighlightingsReady(Path, std::move(Highlightings), + NumLines); }); } Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.h =================================================================== --- clang-tools-extra/trunk/clangd/SemanticHighlighting.h +++ clang-tools-extra/trunk/clangd/SemanticHighlighting.h @@ -43,7 +43,16 @@ Range R; }; -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs); +bool operator==(const HighlightingToken &L, const HighlightingToken &R); +bool operator<(const HighlightingToken &L, const HighlightingToken &R); + +/// Contains all information about highlightings on a single line. +struct LineHighlightings { + int Line; + std::vector Tokens; +}; + +bool operator==(const LineHighlightings &L, const LineHighlightings &R); // Returns all HighlightingTokens from an AST. Only generates highlights for the // main AST. @@ -53,9 +62,22 @@ /// (https://manual.macromates.com/en/language_grammars). llvm::StringRef toTextMateScope(HighlightingKind Kind); -// Convert to LSP's semantic highlighting information. +/// Convert to LSP's semantic highlighting information. std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens); +toSemanticHighlightingInformation(llvm::ArrayRef Tokens); + +/// Return a line-by-line diff between two highlightings. +/// - if the tokens on a line are the same in both hightlightings, this line is +/// omitted. +/// - if a line exists in New but not in Old the tokens on this line are +/// emitted. +/// - if a line does not exists in New but exists in Old an empty line is +/// emitted (to tell client to clear the previous highlightings on this line). +/// \p NewMaxLine is the maximum line number from the new file. +/// REQUIRED: Old and New are sorted. +std::vector +diffHighlightings(ArrayRef New, + ArrayRef Old, int NewMaxLine); } // namespace clangd } // namespace clang Index: clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp =================================================================== --- clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp +++ clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp @@ -12,6 +12,7 @@ #include "SourceCode.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RecursiveASTVisitor.h" +#include namespace clang { namespace clangd { @@ -33,10 +34,7 @@ TraverseAST(Ctx); // Initializer lists can give duplicates of tokens, therefore all tokens // must be deduplicated. - llvm::sort(Tokens, - [](const HighlightingToken &L, const HighlightingToken &R) { - return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); - }); + llvm::sort(Tokens); auto Last = std::unique(Tokens.begin(), Tokens.end()); Tokens.erase(Last, Tokens.end()); return Tokens; @@ -260,10 +258,74 @@ llvm::support::endian::write16be(Buf.data(), I); OS.write(Buf.data(), Buf.size()); } + +// Get the highlightings on \c Line where the first entry of line is at \c +// StartLineIt. If it is not at \c StartLineIt an empty vector is returned. +ArrayRef +takeLine(ArrayRef AllTokens, + ArrayRef::iterator StartLineIt, int Line) { + return ArrayRef(StartLineIt, AllTokens.end()) + .take_while([Line](const HighlightingToken &Token) { + return Token.R.start.line == Line; + }); +} } // namespace -bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) { - return Lhs.Kind == Rhs.Kind && Lhs.R == Rhs.R; +std::vector +diffHighlightings(ArrayRef New, + ArrayRef Old, int NewMaxLine) { + assert(std::is_sorted(New.begin(), New.end()) && "New must be a sorted vector"); + assert(std::is_sorted(Old.begin(), Old.end()) && "Old must be a sorted vector"); + + // FIXME: There's an edge case when tokens span multiple lines. If the first + // token on the line started on a line above the current one and the rest of + // the line is the equal to the previous one than we will remove all + // highlights but the ones for the token spanning multiple lines. This means + // that when we get into the LSP layer the only highlights that will be + // visible are the ones for the token spanning multiple lines. + // Example: + // EndOfMultilineToken Token Token Token + // If "Token Token Token" don't differ from previously the line is + // incorrectly removed. Suggestion to fix is to separate any multiline tokens + // into one token for every line it covers. This requires reading from the + // file buffer to figure out the length of each line though. + std::vector DiffedLines; + // ArrayRefs to the current line in the highlightings. + ArrayRef NewLine(New.begin(), + /*length*/0UL); + ArrayRef OldLine(Old.begin(), + /*length*/ 0UL); + auto NewEnd = New.end(); + auto OldEnd = Old.end(); + auto NextLineNumber = [&]() { + int NextNew = NewLine.end() != NewEnd ? NewLine.end()->R.start.line + : std::numeric_limits::max(); + int NextOld = OldLine.end() != OldEnd ? OldLine.end()->R.start.line + : std::numeric_limits::max(); + return std::min(NextNew, NextOld); + }; + + // If the New file has fewer lines than the Old file we don't want to send + // highlightings beyond the end of the file. + for (int LineNumber = 0; LineNumber < NewMaxLine; + LineNumber = NextLineNumber()) { + NewLine = takeLine(New, NewLine.end(), LineNumber); + OldLine = takeLine(Old, OldLine.end(), LineNumber); + if (NewLine != OldLine) + DiffedLines.push_back({LineNumber, NewLine}); + } + + return DiffedLines; +} + +bool operator==(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) == std::tie(R.R, R.Kind); +} +bool operator<(const HighlightingToken &L, const HighlightingToken &R) { + return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind); +} +bool operator==(const LineHighlightings &L, const LineHighlightings &R) { + return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens); } std::vector getSemanticHighlightings(ParsedAST &AST) { @@ -271,22 +333,18 @@ } std::vector -toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { +toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { if (Tokens.size() == 0) return {}; // FIXME: Tokens might be multiple lines long (block comments) in this case // this needs to add multiple lines for those tokens. - std::map> TokenLines; - for (const HighlightingToken &Token : Tokens) - TokenLines[Token.R.start.line].push_back(Token); - std::vector Lines; - Lines.reserve(TokenLines.size()); - for (const auto &Line : TokenLines) { + Lines.reserve(Tokens.size()); + for (const auto &Line : Tokens) { llvm::SmallVector LineByteTokens; llvm::raw_svector_ostream OS(LineByteTokens); - for (const auto &Token : Line.second) { + for (const auto &Token : Line.Tokens) { // Writes the token to LineByteTokens in the byte format specified by the // LSP proposal. Described below. // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->| @@ -297,7 +355,7 @@ write16be(static_cast(Token.Kind), OS); } - Lines.push_back({Line.first, encodeBase64(LineByteTokens)}); + Lines.push_back({Line.Line, encodeBase64(LineByteTokens)}); } return Lines; Index: clang-tools-extra/trunk/clangd/test/semantic-highlighting.test =================================================================== --- clang-tools-extra/trunk/clangd/test/semantic-highlighting.test +++ clang-tools-extra/trunk/clangd/test/semantic-highlighting.test @@ -49,6 +49,50 @@ # CHECK-NEXT: } # CHECK-NEXT:} --- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 0, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo2.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 1, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 1,"character": 10}},"rangeLength": 11,"text": ""}]}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- {"jsonrpc":"2.0","id":3,"method":"shutdown"} --- {"jsonrpc":"2.0","method":"exit"} Index: clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp =================================================================== --- clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp +++ clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp @@ -29,9 +29,7 @@ return Tokens; } -void checkHighlightings(llvm::StringRef Code) { - Annotations Test(Code); - auto AST = TestTU::withCode(Test.code()).build(); +std::vector getExpectedTokens(Annotations &Test) { static const std::map KindToString{ {HighlightingKind::Variable, "Variable"}, {HighlightingKind::Function, "Function"}, @@ -48,10 +46,47 @@ Test.ranges(KindString.second), KindString.first); ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end()); } + llvm::sort(ExpectedTokens); + return ExpectedTokens; +} + +void checkHighlightings(llvm::StringRef Code) { + Annotations Test(Code); + auto AST = TestTU::withCode(Test.code()).build(); + std::vector ActualTokens = getSemanticHighlightings(AST); + EXPECT_THAT(ActualTokens, getExpectedTokens(Test)); +} - auto ActualTokens = getSemanticHighlightings(AST); - EXPECT_THAT(ActualTokens, testing::UnorderedElementsAreArray(ExpectedTokens)) - << "Inputs is:\n" << Code; +// Any annotations in OldCode and NewCode are converted into their corresponding +// HighlightingToken. The tokens are diffed against each other. Any lines where +// the tokens should diff must be marked with a ^ somewhere on that line in +// NewCode. If there are diffs that aren't marked with ^ the test fails. The +// test also fails if there are lines marked with ^ that don't differ. +void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) { + Annotations OldTest(OldCode); + Annotations NewTest(NewCode); + std::vector OldTokens = getExpectedTokens(OldTest); + std::vector NewTokens = getExpectedTokens(NewTest); + + llvm::DenseMap> ExpectedLines; + for (const Position &Point : NewTest.points()) { + ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens + // are inserted on these lines later. + } + std::vector ExpectedLinePairHighlighting; + for (const HighlightingToken &Token : NewTokens) { + auto It = ExpectedLines.find(Token.R.start.line); + if (It != ExpectedLines.end()) + It->second.push_back(Token); + } + for (auto &LineTokens : ExpectedLines) + ExpectedLinePairHighlighting.push_back( + {LineTokens.first, LineTokens.second}); + + std::vector ActualDiffed = + diffHighlightings(NewTokens, OldTokens, NewCode.count('\n')); + EXPECT_THAT(ActualDiffed, + testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting)); } TEST(SemanticHighlighting, GetsCorrectTokens) { @@ -226,8 +261,9 @@ std::atomic Count = {0}; void onDiagnosticsReady(PathRef, std::vector) override {} - void onHighlightingsReady( - PathRef File, std::vector Highlightings) override { + void onHighlightingsReady(PathRef File, + std::vector Highlightings, + int NLines) override { ++Count; } }; @@ -252,21 +288,124 @@ return Pos; }; - std::vector Tokens{ - {HighlightingKind::Variable, - Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, - {HighlightingKind::Function, - Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, - {HighlightingKind::Variable, - Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}; + std::vector Tokens{ + {3, + {{HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}}}, + {1, + {{HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}}}; std::vector ActualResults = toSemanticHighlightingInformation(Tokens); std::vector ExpectedResults = { - {1, "AAAAAQAEAAA="}, - {3, "AAAACAAEAAAAAAAEAAMAAQ=="}}; + {3, "AAAACAAEAAAAAAAEAAMAAQ=="}, {1, "AAAAAQAEAAA="}}; EXPECT_EQ(ActualResults, ExpectedResults); } +TEST(SemanticHighlighting, HighlightingDiffer) { + struct { + llvm::StringRef OldCode; + llvm::StringRef NewCode; + } TestCases[]{{ + R"( + $Variable[[A]] + $Class[[B]] + $Function[[C]] + )", + R"( + $Variable[[A]] + $Class[[D]] + $Function[[C]] + )"}, + { + R"( + $Class[[C]] + $Field[[F]] + $Variable[[V]] + $Class[[C]] $Variable[[V]] $Field[[F]] + )", + R"( + $Class[[C]] + $Field[[F]] + ^$Function[[F]] + $Class[[C]] $Variable[[V]] $Field[[F]] + )"}, + { + R"( + + $Class[[A]] + $Variable[[A]] + )", + R"( + + ^ + ^$Class[[A]] + ^$Variable[[A]] + )"}, + { + R"( + $Class[[C]] + $Field[[F]] + $Variable[[V]] + $Class[[C]] $Variable[[V]] $Field[[F]] + )", + R"( + $Class[[C]] + ^ + ^ + $Class[[C]] $Variable[[V]] $Field[[F]] + )"}, + { + R"( + $Class[[A]] + $Variable[[A]] + $Variable[[A]] + )", + R"( + $Class[[A]] + ^$Variable[[AA]] + $Variable[[A]] + )"}, + { + R"( + $Class[[A]] + $Variable[[A]] + $Class[[A]] + $Variable[[A]] + )", + R"( + $Class[[A]] + $Variable[[A]] + )"}, + { + R"( + $Class[[A]] + $Variable[[A]] + )", + R"( + $Class[[A]] + $Variable[[A]] + ^$Class[[A]] + ^$Variable[[A]] + )"}, + { + R"( + $Variable[[A]] + $Variable[[A]] + $Variable[[A]] + )", + R"( + ^$Class[[A]] + ^$Class[[A]] + ^$Class[[A]] + )"}}; + + for (const auto &Test : TestCases) + checkDiffedHighlights(Test.OldCode, Test.NewCode); +} + } // namespace } // namespace clangd } // namespace clang