[clangd] Added semantic highlighting support for primitives.

[jvikstrom]

Summary

Adds a new HighlightingKind "Primitive". Adds a special case for TypeLocs that have an underlying TypePtr that is are builtin types, adding them as primitives.
The primary reason for this change is because otherwise typedefs that typedef primitives typedef int A would not get highlighted (so in the example A would not get any highlightings.)

[hokein]

nice, just a nit.

clang-tools-extra/clangd/SemanticHighlighting.cpp

138

nit: we could merge this to the if below.

clang-tools-extra/clangd/SemanticHighlighting.h

enum class HighlightingKind {
   Function,
   Method,
   Field,
   Class,
   Enum,
   EnumConstant,
   Namespace,
   TemplateParameter,
+  Primitive,
 
   NumKinds,
 };
 
 // Contains all information needed for the highlighting a token.
 struct HighlightingToken {
   HighlightingKind Kind;
   Range R;

clang-tools-extra/clangd/SemanticHighlighting.cpp

if (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier())
         addToken(NNSLoc.getLocalBeginLoc(), HighlightingKind::Namespace);
 
     return RecursiveASTVisitor<
         HighlightingTokenCollector>::TraverseNestedNameSpecifierLoc(NNSLoc);
   }
 
 private:
   void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
+    // Builtins must be special cased as they do not have a TagDecl.
+    if (const auto *T = TL.getTypePtr())
+      if (T->isBuiltinType())
+        addToken(Loc, HighlightingKind::Primitive);
+
     if (const Type *TP = TL.getTypePtr())
       if (const TagDecl *TD = TP->getAsTagDecl())
         addToken(Loc, TD);
   }
 
   void addToken(SourceLocation Loc, const NamedDecl *D) {
     if (D->getDeclName().isIdentifier() && D->getName().empty())
       // Don't add symbols that don't have any length.
llvm::StringRef toTextMateScope(HighlightingKind Kind) {
   case HighlightingKind::Enum:
     return "entity.name.type.enum.cpp";
   case HighlightingKind::EnumConstant:
     return "variable.other.enummember.cpp";
   case HighlightingKind::Namespace:
     return "entity.name.namespace.cpp";
   case HighlightingKind::TemplateParameter:
     return "entity.name.type.template.cpp";
+  case HighlightingKind::Primitive:
+    return "storage.type.primitive.cpp";
   case HighlightingKind::NumKinds:
     llvm_unreachable("must not pass NumKinds to the function");
   }
   llvm_unreachable("unhandled HighlightingKind");
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/test/semantic-highlighting.test

 # CHECK-NEXT:          [
 # CHECK-NEXT:            "variable.other.enummember.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.namespace.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.type.template.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "storage.type.primitive.cpp"
 # CHECK-NEXT:          ]
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 0,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 0,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 1,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo2.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 1,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:   ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

static const std::map<HighlightingKind, std::string> KindToString{
       {HighlightingKind::Variable, "Variable"},
       {HighlightingKind::Function, "Function"},
       {HighlightingKind::Class, "Class"},
       {HighlightingKind::Enum, "Enum"},
       {HighlightingKind::Namespace, "Namespace"},
       {HighlightingKind::EnumConstant, "EnumConstant"},
       {HighlightingKind::Field, "Field"},
       {HighlightingKind::Method, "Method"},
-      {HighlightingKind::TemplateParameter, "TemplateParameter"}};
+      {HighlightingKind::TemplateParameter, "TemplateParameter"},
+      {HighlightingKind::Primitive, "Primitive"}};
   std::vector<HighlightingToken> ExpectedTokens;
   for (const auto &KindString : KindToString) {
     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
         Test.ranges(KindString.second), KindString.first);
     ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
   llvm::sort(ExpectedTokens);
   return ExpectedTokens;
void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
   EXPECT_THAT(ActualDiffed,
               testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
     R"cpp(
       struct $Class[[AS]] {
-        double $Field[[SomeMember]];
+        $Primitive[[double]] $Field[[SomeMember]];
       };
       struct {
       } $Variable[[S]];
-      void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
+      $Primitive[[void]] $Function[[foo]]($Primitive[[int]] $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
         auto $Variable[[VeryLongVariableName]] = 12312;
         $Class[[AS]]     $Variable[[AA]];
         auto $Variable[[L]] = $Variable[[AA]].$Field[[SomeMember]] + $Variable[[A]];
-        auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
+        auto $Variable[[FN]] = [ $Variable[[AA]]]($Primitive[[int]] $Variable[[A]]) -> $Primitive[[void]] {};
         $Variable[[FN]](12312);
       }
     )cpp",
     R"cpp(
-      void $Function[[foo]](int);
-      void $Function[[Gah]]();
-      void $Function[[foo]]() {
+      $Primitive[[void]] $Function[[foo]]($Primitive[[int]]);
+      $Primitive[[void]] $Function[[Gah]]();
+      $Primitive[[void]] $Function[[foo]]() {
         auto $Variable[[Bou]] = $Function[[Gah]];
       }
       struct $Class[[A]] {
-        void $Method[[abc]]();
+        $Primitive[[void]] $Method[[abc]]();
       };
     )cpp",
     R"cpp(
       namespace $Namespace[[abc]] {
         template<typename $TemplateParameter[[T]]>
         struct $Class[[A]] {
           $TemplateParameter[[T]] $Field[[t]];
         };
       }
       template<typename $TemplateParameter[[T]]>
       struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
         typename $TemplateParameter[[T]]::A* $Field[[D]];
       };
-      $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
-      typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
+      $Namespace[[abc]]::$Class[[A]]<$Primitive[[int]]> $Variable[[AA]];
+      typedef $Namespace[[abc]]::$Class[[A]]<$Primitive[[int]]> $Class[[AAA]];
       struct $Class[[B]] {
         $Class[[B]]();
         ~$Class[[B]]();
-        void operator<<($Class[[B]]);
+        $Primitive[[void]] operator<<($Class[[B]]);
         $Class[[AAA]] $Field[[AA]];
       };
       $Class[[B]]::$Class[[B]]() {}
       $Class[[B]]::~$Class[[B]]() {}
-      void $Function[[f]] () {
+      $Primitive[[void]] $Function[[f]] () {
         $Class[[B]] $Variable[[BB]] = $Class[[B]]();
         $Variable[[BB]].~$Class[[B]]();
         $Class[[B]]();
       }
     )cpp",
     R"cpp(
       enum class $Enum[[E]] {
         $EnumConstant[[A]],
         $EnumConstant[[B]],
       };
       enum $Enum[[EE]] {
         $EnumConstant[[Hi]],
       };
       struct $Class[[A]] {
         $Enum[[E]] $Field[[EEE]];
         $Enum[[EE]] $Field[[EEEE]];
       };
-      int $Variable[[I]] = $EnumConstant[[Hi]];
+      $Primitive[[int]] $Variable[[I]] = $EnumConstant[[Hi]];
       $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
     )cpp",
     R"cpp(
       namespace $Namespace[[abc]] {
         namespace {}
         namespace $Namespace[[bcd]] {
           struct $Class[[A]] {};
           namespace $Namespace[[cde]] {
R"cpp(
       $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
       $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
             $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
       ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
       ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
     )cpp",
     R"cpp(
       struct $Class[[D]] {
-        double $Field[[C]];
+        $Primitive[[double]] $Field[[C]];
       };
       struct $Class[[A]] {
-        double $Field[[B]];
+        $Primitive[[double]] $Field[[B]];
         $Class[[D]] $Field[[E]];
-        static double $Variable[[S]];
-        void $Method[[foo]]() {
+        static $Primitive[[double]] $Variable[[S]];
+        $Primitive[[void]] $Method[[foo]]() {
           $Field[[B]] = 123;
           this->$Field[[B]] = 156;
           this->$Method[[foo]]();
           $Method[[foo]]();
           $Variable[[S]] = 90.1;
         }
       };
-      void $Function[[foo]]() {
+      $Primitive[[void]] $Function[[foo]]() {
         $Class[[A]] $Variable[[AA]];
         $Variable[[AA]].$Field[[B]] += 2;
         $Variable[[AA]].$Method[[foo]]();
         $Variable[[AA]].$Field[[E]].$Field[[C]];
         $Class[[A]]::$Variable[[S]] = 90;
       }
     )cpp",
     R"cpp(
       struct $Class[[AA]] {
-        int $Field[[A]];
+        $Primitive[[int]] $Field[[A]];
       }
-      int $Variable[[B]];
+      $Primitive[[int]] $Variable[[B]];
       $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
     )cpp",
     R"cpp(
       namespace $Namespace[[a]] {
         struct $Class[[A]] {};
+        typedef $Primitive[[char]] $Primitive[[C]];
       }
       typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
       using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
       enum class $Enum[[E]] {};
       typedef $Enum[[E]] $Enum[[C]];
       typedef $Enum[[C]] $Enum[[CC]];
       using $Enum[[CD]] = $Enum[[CC]];
       $Enum[[CC]] $Function[[f]]($Class[[B]]);
       $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+      typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
+      typedef $Primitive[[float]] $Primitive[[F]];
     )cpp",
     R"cpp(
-      template<typename $TemplateParameter[[T]], typename = void>
+      template<typename $TemplateParameter[[T]], typename = $Primitive[[void]]>
       class $Class[[A]] {
         $TemplateParameter[[T]] $Field[[AA]];
         $TemplateParameter[[T]] $Method[[foo]]();
       };
       template<class $TemplateParameter[[TT]]>
       class $Class[[B]] {
         $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
       };
       template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
       class $Class[[BB]] {};
       template<class $TemplateParameter[[T]]>
-      class $Class[[BB]]<$TemplateParameter[[T]], int> {};
+      class $Class[[BB]]<$TemplateParameter[[T]], $Primitive[[int]]> {};
       template<class $TemplateParameter[[T]]>
       class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
 
       template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
       $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
 
       template<typename>
       class $Class[[Foo]] {};
 
       template<typename $TemplateParameter[[T]]>
-      void $Function[[foo]]($TemplateParameter[[T]] ...);
+      $Primitive[[void]] $Function[[foo]]($TemplateParameter[[T]] ...);
     )cpp",
     R"cpp(
       template <class $TemplateParameter[[T]]>
       struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
-      extern template struct $Class[[Tmpl]]<float>;
-      template struct $Class[[Tmpl]]<double>;
+      extern template struct $Class[[Tmpl]]<$Primitive[[float]]>;
+      template struct $Class[[Tmpl]]<$Primitive[[double]]>;
     )cpp"};
   for (const auto &TestCase : TestCases) {
     checkHighlightings(TestCase);
   }
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
   class HighlightingsCounterDiagConsumer : public DiagnosticsConsumer {

clang-tools-extra/clangd/unittests/TweakTests.cpp

TEST(TweaksTest, ExtractVariable) {
   }
 }
 
 TEST(TweaksTest, AnnotateHighlightings) {
   llvm::StringLiteral ID = "AnnotateHighlightings";
   checkAvailable(ID, "^vo^id^ ^f(^) {^}^"); // available everywhere.
   checkAvailable(ID, "[[int a; int b;]]");
   const char *Input = "void ^f() {}";
-  const char *Output = "void /* entity.name.function.cpp */f() {}";
+  const char *Output = "/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f() {}";
   checkTransform(ID, Input, Output);
 
   checkTransform(ID,
   R"cpp(
 [[void f1();
 void f2();]]
 )cpp",
   R"cpp(
-void /* entity.name.function.cpp */f1();
-void /* entity.name.function.cpp */f2();
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f1();
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f2();
 )cpp");
 
    checkTransform(ID,
   R"cpp(
 void f1();
 void f2() {^};
 )cpp",
 
   R"cpp(
 void f1();
-void /* entity.name.function.cpp */f2() {};
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f2() {};
 )cpp");
 }
 
 TWEAK_TEST(ExpandMacro);
 TEST_F(ExpandMacroTest, Test) {
   Header = R"cpp(
     #define FOO 1 2 3
     #define FUNC(X) X+X+X