[clangd] Add highlighting for macro expansions.

[jvikstrom]

Summary

https://github.com/clangd/clangd/issues/134

[jvikstrom]

Removed SourceManager field from HighlightingTokenCollector.

[ilya-biryukov]

clang-tools-extra/clangd/SemanticHighlighting.cpp

37

NIT: uncapitalize and use 'macro expansions' instead of `Macro Expansions'

470

Is there a place on the internet with conventional textmate scope names?
Or do we come up with these on our own?

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

394–395

Similar to how we highlight everything else, we should also highlight macro declarations and usages (i.e. expansions in the same way).
Could we add the corresponding highlightings?

#define $Macro[[DEF_CLASS]](T) class T T{}
$Macro[[DEF_CLASS]](A);

[hokein]

clang-tools-extra/clangd/SemanticHighlighting.cpp

470

This is not standard TX scope names, we only have guidelines. In general, we use the names using in vscode as references (you could see the TM name via vscode's "Insepct TM Scopes" functionality)

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

394–395

+1, I think we could name the HighlightingKind as Macro instead of MacroExpansion (even through we don't highlighting the macro def yet).

[jvikstrom]

Address comments.

[jvikstrom]

clang-tools-extra/clangd/SemanticHighlighting.cpp

470

This is from the vscode c++ extension. (we mostly use the same scopes as in that extension, but in a few cases we have our own scopes as there aren't any in vscode)

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

394–395

We don't seem to collect information about where macro declarations are in the code in ParsedAST.
Added a fixme to add it.

[jvikstrom]

Renamed MacroExpansion to Macro.

[hokein]

LGTM from my side.

clang-tools-extra/clangd/SemanticHighlighting.cpp

470

I'd drop the expansion, just use entity.name.function.preprocessor.cpp.

clang-tools-extra/clangd/SemanticHighlighting.h

enum class HighlightingKind {
   Field,
   StaticField,
   Class,
   Enum,
   EnumConstant,
   Namespace,
   TemplateParameter,
   Primitive,
+  MacroExpansion,
 
   NumKinds,
 };
 
 // Contains all information needed for the highlighting a token.
 struct HighlightingToken {
   HighlightingKind Kind;
   Range R;

clang-tools-extra/clangd/SemanticHighlighting.cpp

 namespace clang {
 namespace clangd {
 namespace {
 
 // Collects all semantic tokens in an ASTContext.
 class HighlightingTokenCollector
     : public RecursiveASTVisitor<HighlightingTokenCollector> {
   std::vector<HighlightingToken> Tokens;
-  ASTContext &Ctx;
+  ParsedAST &AST;
   const SourceManager &SM;
 
 public:
   HighlightingTokenCollector(ParsedAST &AST)
-      : Ctx(AST.getASTContext()), SM(AST.getSourceManager()) {}
+      : AST(AST), SM(AST.getSourceManager()) {}
 
   std::vector<HighlightingToken> collectTokens() {
     Tokens.clear();
-    TraverseAST(Ctx);
+    TraverseAST(AST.getASTContext());
+    // Add highlightings for Macro Expansions as they are not traversed by the
+    // visitor.
+    for (const SourceLocation &L : AST.getMainFileExpansions())
+      addToken(L, HighlightingKind::MacroExpansion);
     // Initializer lists can give duplicates of tokens, therefore all tokens
     // must be deduplicated.
     llvm::sort(Tokens);
     auto Last = std::unique(Tokens.begin(), Tokens.end());
     Tokens.erase(Last, Tokens.end());
     // Macros can give tokens that have the same source range but conflicting
     // kinds. In this case all tokens sharing this source range should be
     // removed.
void addToken(SourceLocation Loc, HighlightingKind Kind) {
     // Non top level decls that are included from a header are not filtered by
     // topLevelDecls. (example: method declarations being included from another
     // file for a class from another file)
     // There are also cases with macros where the spelling loc will not be in
     // the main file and the highlighting would be incorrect.
     if (!isInsideMainFile(Loc, SM))
       return;
 
-    auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
+    auto R = getTokenRange(SM, AST.getASTContext().getLangOpts(), Loc);
     if (!R) {
       // R should always have a value, if it doesn't something is very wrong.
       elog("Tried to add semantic token with an invalid range");
       return;
     }
 
     Tokens.push_back({Kind, R.getValue()});
   }
llvm::StringRef toTextMateScope(HighlightingKind Kind) {
   case HighlightingKind::EnumConstant:
     return "variable.other.enummember.cpp";
   case HighlightingKind::Namespace:
     return "entity.name.namespace.cpp";
   case HighlightingKind::TemplateParameter:
     return "entity.name.type.template.cpp";
   case HighlightingKind::Primitive:
     return "storage.type.primitive.cpp";
+  case HighlightingKind::MacroExpansion:
+    return "entity.name.function.preprocessor.expansion.cpp";
   case HighlightingKind::NumKinds:
     llvm_unreachable("must not pass NumKinds to the function");
   }
   llvm_unreachable("unhandled HighlightingKind");
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/test/semantic-highlighting.test

 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.namespace.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.type.template.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "storage.type.primitive.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "entity.name.function.preprocessor.expansion.cpp"
 # CHECK-NEXT:          ]
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

static const std::map<HighlightingKind, std::string> KindToString{
       {HighlightingKind::Enum, "Enum"},
       {HighlightingKind::Namespace, "Namespace"},
       {HighlightingKind::EnumConstant, "EnumConstant"},
       {HighlightingKind::Field, "Field"},
       {HighlightingKind::StaticField, "StaticField"},
       {HighlightingKind::Method, "Method"},
       {HighlightingKind::StaticMethod, "StaticMethod"},
       {HighlightingKind::TemplateParameter, "TemplateParameter"},
-      {HighlightingKind::Primitive, "Primitive"}};
+      {HighlightingKind::Primitive, "Primitive"},
+      {HighlightingKind::MacroExpansion, "MacroExpansion"}};
   std::vector<HighlightingToken> ExpectedTokens;
   for (const auto &KindString : KindToString) {
     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
         Test.ranges(KindString.second), KindString.first);
     ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
   llvm::sort(ExpectedTokens);
   return ExpectedTokens;
)cpp",
         $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
         $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
         $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
     )cpp",
       // Tokens that share a source range but have conflicting Kinds are not
       // highlighted.
       R"cpp(
       #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
       #define DEF_CLASS(T) class T {};
-      DEF_MULTIPLE(XYZ);
-      DEF_MULTIPLE(XYZW);
-      DEF_CLASS($Class[[A]])
+      $MacroExpansion[[DEF_MULTIPLE]](XYZ);
+      $MacroExpansion[[DEF_MULTIPLE]](XYZW);
+      $MacroExpansion[[DEF_CLASS]]($Class[[A]])
       #define MACRO_CONCAT(X, V, T) T foo##X = V
       #define DEF_VAR(X, V) int X = V
       #define DEF_VAR_T(T, X, V) T X = V
       #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
       #define CPY(X) X
       #define DEF_VAR_TYPE(X, Y) X Y
       #define SOME_NAME variable
       #define SOME_NAME_SET variable2 = 123
       #define INC_VAR(X) X += 2
       $Primitive[[void]] $Function[[foo]]() {
-        DEF_VAR($LocalVariable[[X]],  123);
-        DEF_VAR_REV(908, $LocalVariable[[XY]]);
-        $Primitive[[int]] CPY( $LocalVariable[[XX]] );
-        DEF_VAR_TYPE($Class[[A]], $LocalVariable[[AA]]);
-        $Primitive[[double]] SOME_NAME;
-        $Primitive[[int]] SOME_NAME_SET;
+        $MacroExpansion[[DEF_VAR]]($LocalVariable[[X]],  123);
+        $MacroExpansion[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
+        $Primitive[[int]] $MacroExpansion[[CPY]]( $LocalVariable[[XX]] );
+        $MacroExpansion[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
+        $Primitive[[double]] $MacroExpansion[[SOME_NAME]];
+        $Primitive[[int]] $MacroExpansion[[SOME_NAME_SET]];
         $LocalVariable[[variable]] = 20.1;
-        MACRO_CONCAT(var, 2, $Primitive[[float]]);
-        DEF_VAR_T($Class[[A]], CPY(CPY($LocalVariable[[Nested]])),
-              CPY($Class[[A]]()));
-        INC_VAR($LocalVariable[[variable]]);
-      }
-      $Primitive[[void]] SOME_NAME();
-      DEF_VAR($Variable[[XYZ]], 567);
-      DEF_VAR_REV(756, $Variable[[AB]]);
+        $MacroExpansion[[MACRO_CONCAT]](var, 2, $Primitive[[float]]);
+        $MacroExpansion[[DEF_VAR_T]]($Class[[A]], $MacroExpansion[[CPY]](
+              $MacroExpansion[[CPY]]($LocalVariable[[Nested]])),
+            $MacroExpansion[[CPY]]($Class[[A]]()));
+        $MacroExpansion[[INC_VAR]]($LocalVariable[[variable]]);
+      }
+      $Primitive[[void]] $MacroExpansion[[SOME_NAME]]();
+      $MacroExpansion[[DEF_VAR]]($Variable[[XYZ]], 567);
+      $MacroExpansion[[DEF_VAR_REV]](756, $Variable[[AB]]);
 
       #define CALL_FN(F) F();
       #define DEF_FN(F) void F ()
-      DEF_FN($Function[[g]]) {
-        CALL_FN($Function[[foo]]);
+      $MacroExpansion[[DEF_FN]]($Function[[g]]) {
+        $MacroExpansion[[CALL_FN]]($Function[[foo]]);
       }
     )cpp",
       R"cpp(
       #define fail(expr) expr
       #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); }
       $Primitive[[int]] $Variable[[x]];
       $Primitive[[int]] $Variable[[y]];
       $Primitive[[int]] $Function[[f]]();
       $Primitive[[void]] $Function[[foo]]() {
-        assert($Variable[[x]] != $Variable[[y]]);
-        assert($Variable[[x]] != $Function[[f]]());
+        $MacroExpansion[[assert]]($Variable[[x]] != $Variable[[y]]);
+        $MacroExpansion[[assert]]($Variable[[x]] != $Function[[f]]());
       }
     )cpp"};
   for (const auto &TestCase : TestCases) {
     checkHighlightings(TestCase);
   }
 
   checkHighlightings(R"cpp(
     class $Class[[A]] {
       #include "imp.h"
     };
     #endif
   )cpp",
                      {{"imp.h", R"cpp(
     int someMethod();
     void otherMethod();
   )cpp"}});
 
   // A separate test for macros in headers.
   checkHighlightings(R"cpp(
     #include "imp.h"
-    DEFINE_Y
-    DXYZ_Y(A);
+    $MacroExpansion[[DEFINE_Y]]
+    $MacroExpansion[[DXYZ_Y]](A);
   )cpp",
                      {{"imp.h", R"cpp(
     #define DXYZ(X) class X {};
     #define DXYZ_Y(Y) DXYZ(x##Y)
     #define DEFINE(X) int X;
     #define DEFINE_Y DEFINE(Y)
   )cpp"}});
 }
EXPECT_THAT(ActualDiff,
               testing::UnorderedElementsAre(
                   testing::AllOf(LineNumber(3), EmptyHighlightings()),
                   testing::AllOf(LineNumber(4), EmptyHighlightings())));
 }
 
 } // namespace
 } // namespace clangd
 } // namespace clang
 No newline at end of file