Added SemanticSymbolASTCollector for collecting semantic symbols Currently collects variable and function declarations

[jvikstrom]

Summary

[clangd] Added SemanticSymbolASTCollector that collects variable and function declarations

[jvikstrom]

Added dots at the end of comments

Updating D63479: Added SemanticSymbolASTCollector for collecting semantic symbols

Currently collects variable and function declarations

[jvikstrom]

Moved SemanticSymbolASTVisitor to an anonymous namespace and added a function for getting highlights that returns highlights line by line

Updating D63479: Added SemanticSymbolASTCollector for collecting semantic symbols

Currently collects variable and function declarations

[jvikstrom]

Resubmitting later with "complete" prototype in CL

clang-tools-extra/clangd/CMakeLists.txt

add_clang_library(clangDaemon
   Headers.cpp
   IncludeFixer.cpp
   JSONTransport.cpp
   Logger.cpp
   Protocol.cpp
   Quality.cpp
   RIFF.cpp
   Selection.cpp
+  SemanticHighlight.cpp
   SourceCode.cpp
   Threading.cpp
   Trace.cpp
   TUScheduler.cpp
   URI.cpp
   XRefs.cpp
 
   index/Background.cpp

clang-tools-extra/clangd/SemanticHighlight.h

+//===--- SemanticSymbolASTCollector.h - Manipulating source code as strings
+//-----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Code for collecting semantic symbols from the C++ AST using the
+// RecursiveASTVisitor
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSYMBOLASTCOLLECTOR_H
+
+#include "AST.h"
+#include "Headers.h"
+#include "Protocol.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Lex/Lexer.h"
+
+namespace clang {
+namespace clangd {
+
+// ScopeIndex represents the mapping from the scopes list to a type of
+// expression.
+enum class SemanticScope : int {
+  VariableDeclaration = 0,
+  FunctionDeclaration = 1,
+};
+
+// Contains all information needed for the highlighting a symbol.
+struct SemanticSymbol {
+  SemanticSymbol() {}
+  SemanticSymbol(SemanticScope Scope, Position StartPosition, unsigned int Len)
+      : Scope(Scope), StartPosition(StartPosition), Len(Len) {}
+  SemanticScope Scope;
+  Position StartPosition;
+  unsigned int Len;
+};
+
+bool operator==(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs);
+bool operator!=(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs);
+
+// Contains all highlights in a single line.
+struct LineHighlight {
+  LineHighlight(unsigned int Line = 0, std::vector<SemanticSymbol> Tokens = {})
+      : Line(Line), Tokens(Tokens) {}
+  unsigned int Line;
+  std::vector<SemanticSymbol> Tokens;
+};
+
+bool operator==(const LineHighlight &Lhs, const LineHighlight &Rhs);
+bool operator!=(const LineHighlight &Lhs, const LineHighlight &Rhs);
+
+std::vector<LineHighlight> getASTHighlights(ASTContext &AST);
+
+} // namespace clangd
+} // namespace clang
+
+#endif

clang-tools-extra/clangd/SemanticHighlight.cpp

+#include "SemanticHighlight.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+// Collects all semantic symbols in an ASTContext. Symbols on line i are always
+// in front of symbols on line i+1
+class SemanticSymbolASTCollector
+    : public RecursiveASTVisitor<SemanticSymbolASTCollector> {
+  std::vector<SemanticSymbol> Symbols;
+  const ASTContext &AST;
+  const SourceManager &SM;
+
+public:
+  SemanticSymbolASTCollector(const ASTContext &AST)
+      : AST(AST), SM(AST.getSourceManager()) {}
+
+  std::vector<SemanticSymbol> getSymbols() { return Symbols; }
+
+  bool VisitVarDecl(VarDecl *Var) {
+    addSymbol(Var, SemanticScope::VariableDeclaration);
+    return true;
+  }
+
+  bool VisitFunctionDecl(FunctionDecl *Func) {
+    addSymbol(Func, SemanticScope::FunctionDeclaration);
+    return true;
+  }
+
+private:
+  void addSymbol(Decl *D, SemanticScope Scope) {
+    auto Loc = D->getLocation();
+    SemanticSymbol S;
+    auto LSPLoc = sourceLocToPosition(SM, Loc);
+
+    S.Len = clang::Lexer::MeasureTokenLength(Loc, SM, AST.getLangOpts());
+    if (S.Len == 0) {
+      // Don't add symbols that don't have any length.
+      return;
+    }
+
+    S.StartPosition.character = LSPLoc.character;
+    S.StartPosition.line = LSPLoc.line;
+    S.Scope = Scope;
+
+    Symbols.push_back(S);
+  }
+};
+
+} // namespace
+
+bool operator==(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs) {
+  return Lhs.Scope == Rhs.Scope && Lhs.StartPosition == Rhs.StartPosition &&
+         Lhs.Len == Rhs.Len;
+}
+bool operator!=(const SemanticSymbol &Lhs, const SemanticSymbol &Rhs) {
+  return !(Lhs == Rhs);
+}
+
+bool operator==(const LineHighlight &Lhs, const LineHighlight &Rhs) {
+  return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens;
+}
+bool operator!=(const LineHighlight &Lhs, const LineHighlight &Rhs) {
+  return !(Lhs == Rhs);
+}
+
+std::vector<LineHighlight> getASTHighlights(ASTContext &AST) {
+  SemanticSymbolASTCollector Collector(AST);
+  Collector.TraverseAST(AST);
+  auto Symbols = Collector.getSymbols();
+  std::vector<LineHighlight> Lines;
+  int LastLine = -1;
+  // Split the vector of symbols into lines
+  for (const auto &Symbol : Symbols) {
+    if (Symbol.StartPosition.line != LastLine) {
+      Lines.push_back(LineHighlight(Symbol.StartPosition.line));
+      LastLine = Symbol.StartPosition.line;
+    }
+
+    Lines.back().Tokens.push_back(Symbol);
+  }
+
+  return Lines;
+}
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/unittests/CMakeLists.txt

add_unittest(ClangdUnitTests ClangdTests
   IndexActionTests.cpp
   IndexTests.cpp
   JSONTransportTests.cpp
   PrintASTTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
   SelectionTests.cpp
+  SemanticHighlightTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
   SymbolInfoTests.cpp
   SyncAPI.cpp
   TUSchedulerTests.cpp
   TestFS.cpp
   TestIndex.cpp

clang-tools-extra/clangd/unittests/SemanticHighlightTests.cpp

+//===-- SemanticSymbolASTCollectorTests.cpp - SemanticSymbolASTCollector tests
+//------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Protocol.h"
+#include "SemanticHighlight.h"
+#include "SourceCode.h"
+#include "TestTU.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+Position createPosition(int Line, int Character) {
+  Position Pos;
+  Pos.character = Character;
+  Pos.line = Line;
+  return Pos;
+}
+
+TEST(SemanticSymbolASTCollector, GetBeginningOfIdentifier) {
+  std::string Preamble = R"cpp(
+    struct A {
+      double SomeMember;
+    };
+    void foo(int a) {
+      auto VeryLongVariableName = 12312;
+      A     aa;
+    }
+  )cpp";
+
+  Annotations TestCase(Preamble);
+  std::vector<LineHighlight> CorrectLines = std::vector<LineHighlight>{
+      LineHighlight(4, {SemanticSymbol(SemanticScope::FunctionDeclaration,
+                                       createPosition(4, 9), 3),
+                        SemanticSymbol(SemanticScope::VariableDeclaration,
+                                       createPosition(4, 17), 1)}),
+      LineHighlight(5, {SemanticSymbol(SemanticScope::VariableDeclaration,
+                                       createPosition(5, 11), 20)}),
+      LineHighlight(6, {SemanticSymbol(SemanticScope::VariableDeclaration,
+                                       createPosition(6, 12), 2)})
+
+  };
+
+  auto AST = TestTU::withCode(TestCase.code()).build();
+  auto Lines = getASTHighlights(AST.getASTContext());
+  EXPECT_THAT(Lines, ElementsAreArray(CorrectLines));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang