[clangd] Added semantic highlighting support for primitives.

[jvikstrom]

Summary

Adds a new HighlightingKind "Primitive". Adds a special case for TypeLocs that have an underlying TypePtr that is are builtin types, adding them as primitives.
The primary reason for this change is because otherwise typedefs that typedef primitives typedef int A would not get highlighted (so in the example A would not get any highlightings.)

[hokein]

nice, just a nit.

clang-tools-extra/clangd/SemanticHighlighting.cpp

138 ↗

(On Diff #214121)

nit: we could merge this to the if below.

clang-tools-extra/trunk/clangd/SemanticHighlighting.h

enum class HighlightingKind {
   Function,
   Method,
   Field,
   Class,
   Enum,
   EnumConstant,
   Namespace,
   TemplateParameter,
+  Primitive,
 
   NumKinds,
 };
 
 // Contains all information needed for the highlighting a token.
 struct HighlightingToken {
   HighlightingKind Kind;
   Range R;

clang-tools-extra/trunk/clangd/SemanticHighlighting.cpp

if (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier())
         addToken(NNSLoc.getLocalBeginLoc(), HighlightingKind::Namespace);
 
     return RecursiveASTVisitor<
         HighlightingTokenCollector>::TraverseNestedNameSpecifierLoc(NNSLoc);
   }
 
 private:
   void addTypeLoc(SourceLocation Loc, const TypeLoc &TL) {
-    if (const Type *TP = TL.getTypePtr())
+    if (const Type *TP = TL.getTypePtr()) {
       if (const TagDecl *TD = TP->getAsTagDecl())
         addToken(Loc, TD);
+      if (TP->isBuiltinType())
+        // Builtins must be special cased as they do not have a TagDecl.
+        addToken(Loc, HighlightingKind::Primitive);
+    }
   }
 
   void addToken(SourceLocation Loc, const NamedDecl *D) {
     if (D->getDeclName().isIdentifier() && D->getName().empty())
       // Don't add symbols that don't have any length.
       return;
     // We highlight class decls, constructor decls and destructor decls as
     // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we will
llvm::StringRef toTextMateScope(HighlightingKind Kind) {
   case HighlightingKind::Enum:
     return "entity.name.type.enum.cpp";
   case HighlightingKind::EnumConstant:
     return "variable.other.enummember.cpp";
   case HighlightingKind::Namespace:
     return "entity.name.namespace.cpp";
   case HighlightingKind::TemplateParameter:
     return "entity.name.type.template.cpp";
+  case HighlightingKind::Primitive:
+    return "storage.type.primitive.cpp";
   case HighlightingKind::NumKinds:
     llvm_unreachable("must not pass NumKinds to the function");
   }
   llvm_unreachable("unhandled HighlightingKind");
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/trunk/clangd/test/semantic-highlighting.test

 # CHECK-NEXT:          [
 # CHECK-NEXT:            "variable.other.enummember.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.namespace.cpp"
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.type.template.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "storage.type.primitive.cpp"
 # CHECK-NEXT:          ]
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 0,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","version":1,"text":"int x = 2;\nint y = 2;"}}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 0,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 1,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo2.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp","version":2},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
 #      CHECK:  "method": "textDocument/semanticHighlighting",
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
 # CHECK-NEXT:        "line": 1,
-# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:        "tokens": "AAAAAAADAAkAAAAEAAEAAA=="
 # CHECK-NEXT:      }
 # CHECK-NEXT:   ],
 # CHECK-NEXT:    "textDocument": {
 # CHECK-NEXT:      "uri": "file:///clangd-test/foo.cpp"
 # CHECK-NEXT:    }
 # CHECK-NEXT:  }
 # CHECK-NEXT:}
 ---

clang-tools-extra/trunk/clangd/unittests/SemanticHighlightingTests.cpp

static const std::map<HighlightingKind, std::string> KindToString{
       {HighlightingKind::Variable, "Variable"},
       {HighlightingKind::Function, "Function"},
       {HighlightingKind::Class, "Class"},
       {HighlightingKind::Enum, "Enum"},
       {HighlightingKind::Namespace, "Namespace"},
       {HighlightingKind::EnumConstant, "EnumConstant"},
       {HighlightingKind::Field, "Field"},
       {HighlightingKind::Method, "Method"},
-      {HighlightingKind::TemplateParameter, "TemplateParameter"}};
+      {HighlightingKind::TemplateParameter, "TemplateParameter"},
+      {HighlightingKind::Primitive, "Primitive"}};
   std::vector<HighlightingToken> ExpectedTokens;
   for (const auto &KindString : KindToString) {
     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
         Test.ranges(KindString.second), KindString.first);
     ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
   }
   llvm::sort(ExpectedTokens);
   return ExpectedTokens;
void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
   EXPECT_THAT(ActualDiffed,
               testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting));
 }
 
 TEST(SemanticHighlighting, GetsCorrectTokens) {
   const char *TestCases[] = {
     R"cpp(
       struct $Class[[AS]] {
-        double $Field[[SomeMember]];
+        $Primitive[[double]] $Field[[SomeMember]];
       };
       struct {
       } $Variable[[S]];
-      void $Function[[foo]](int $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
+      $Primitive[[void]] $Function[[foo]]($Primitive[[int]] $Variable[[A]], $Class[[AS]] $Variable[[As]]) {
         auto $Variable[[VeryLongVariableName]] = 12312;
         $Class[[AS]]     $Variable[[AA]];
         auto $Variable[[L]] = $Variable[[AA]].$Field[[SomeMember]] + $Variable[[A]];
-        auto $Variable[[FN]] = [ $Variable[[AA]]](int $Variable[[A]]) -> void {};
+        auto $Variable[[FN]] = [ $Variable[[AA]]]($Primitive[[int]] $Variable[[A]]) -> $Primitive[[void]] {};
         $Variable[[FN]](12312);
       }
     )cpp",
     R"cpp(
-      void $Function[[foo]](int);
-      void $Function[[Gah]]();
-      void $Function[[foo]]() {
+      $Primitive[[void]] $Function[[foo]]($Primitive[[int]]);
+      $Primitive[[void]] $Function[[Gah]]();
+      $Primitive[[void]] $Function[[foo]]() {
         auto $Variable[[Bou]] = $Function[[Gah]];
       }
       struct $Class[[A]] {
-        void $Method[[abc]]();
+        $Primitive[[void]] $Method[[abc]]();
       };
     )cpp",
     R"cpp(
       namespace $Namespace[[abc]] {
         template<typename $TemplateParameter[[T]]>
         struct $Class[[A]] {
           $TemplateParameter[[T]] $Field[[t]];
         };
       }
       template<typename $TemplateParameter[[T]]>
       struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
         typename $TemplateParameter[[T]]::A* $Field[[D]];
       };
-      $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
-      typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
+      $Namespace[[abc]]::$Class[[A]]<$Primitive[[int]]> $Variable[[AA]];
+      typedef $Namespace[[abc]]::$Class[[A]]<$Primitive[[int]]> $Class[[AAA]];
       struct $Class[[B]] {
         $Class[[B]]();
         ~$Class[[B]]();
-        void operator<<($Class[[B]]);
+        $Primitive[[void]] operator<<($Class[[B]]);
         $Class[[AAA]] $Field[[AA]];
       };
       $Class[[B]]::$Class[[B]]() {}
       $Class[[B]]::~$Class[[B]]() {}
-      void $Function[[f]] () {
+      $Primitive[[void]] $Function[[f]] () {
         $Class[[B]] $Variable[[BB]] = $Class[[B]]();
         $Variable[[BB]].~$Class[[B]]();
         $Class[[B]]();
       }
     )cpp",
     R"cpp(
       enum class $Enum[[E]] {
         $EnumConstant[[A]],
         $EnumConstant[[B]],
       };
       enum $Enum[[EE]] {
         $EnumConstant[[Hi]],
       };
       struct $Class[[A]] {
         $Enum[[E]] $Field[[EEE]];
         $Enum[[EE]] $Field[[EEEE]];
       };
-      int $Variable[[I]] = $EnumConstant[[Hi]];
+      $Primitive[[int]] $Variable[[I]] = $EnumConstant[[Hi]];
       $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
     )cpp",
     R"cpp(
       namespace $Namespace[[abc]] {
         namespace {}
         namespace $Namespace[[bcd]] {
           struct $Class[[A]] {};
           namespace $Namespace[[cde]] {
R"cpp(
       $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
       $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
             $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
       ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
       ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
     )cpp",
     R"cpp(
       struct $Class[[D]] {
-        double $Field[[C]];
+        $Primitive[[double]] $Field[[C]];
       };
       struct $Class[[A]] {
-        double $Field[[B]];
+        $Primitive[[double]] $Field[[B]];
         $Class[[D]] $Field[[E]];
-        static double $Variable[[S]];
-        void $Method[[foo]]() {
+        static $Primitive[[double]] $Variable[[S]];
+        $Primitive[[void]] $Method[[foo]]() {
           $Field[[B]] = 123;
           this->$Field[[B]] = 156;
           this->$Method[[foo]]();
           $Method[[foo]]();
           $Variable[[S]] = 90.1;
         }
       };
-      void $Function[[foo]]() {
+      $Primitive[[void]] $Function[[foo]]() {
         $Class[[A]] $Variable[[AA]];
         $Variable[[AA]].$Field[[B]] += 2;
         $Variable[[AA]].$Method[[foo]]();
         $Variable[[AA]].$Field[[E]].$Field[[C]];
         $Class[[A]]::$Variable[[S]] = 90;
       }
     )cpp",
     R"cpp(
       struct $Class[[AA]] {
-        int $Field[[A]];
+        $Primitive[[int]] $Field[[A]];
       }
-      int $Variable[[B]];
+      $Primitive[[int]] $Variable[[B]];
       $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
     )cpp",
     R"cpp(
       namespace $Namespace[[a]] {
         struct $Class[[A]] {};
+        typedef $Primitive[[char]] $Primitive[[C]];
       }
       typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
       using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
       enum class $Enum[[E]] {};
       typedef $Enum[[E]] $Enum[[C]];
       typedef $Enum[[C]] $Enum[[CC]];
       using $Enum[[CD]] = $Enum[[CC]];
       $Enum[[CC]] $Function[[f]]($Class[[B]]);
       $Enum[[CD]] $Function[[f]]($Class[[BB]]);
+      typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
+      typedef $Primitive[[float]] $Primitive[[F]];
     )cpp",
     R"cpp(
-      template<typename $TemplateParameter[[T]], typename = void>
+      template<typename $TemplateParameter[[T]], typename = $Primitive[[void]]>
       class $Class[[A]] {
         $TemplateParameter[[T]] $Field[[AA]];
         $TemplateParameter[[T]] $Method[[foo]]();
       };
       template<class $TemplateParameter[[TT]]>
       class $Class[[B]] {
         $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
       };
       template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
       class $Class[[BB]] {};
       template<class $TemplateParameter[[T]]>
-      class $Class[[BB]]<$TemplateParameter[[T]], int> {};
+      class $Class[[BB]]<$TemplateParameter[[T]], $Primitive[[int]]> {};
       template<class $TemplateParameter[[T]]>
       class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
 
       template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
       $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
 
       template<typename>
       class $Class[[Foo]] {};
 
       template<typename $TemplateParameter[[T]]>
-      void $Function[[foo]]($TemplateParameter[[T]] ...);
+      $Primitive[[void]] $Function[[foo]]($TemplateParameter[[T]] ...);
     )cpp",
     R"cpp(
       template <class $TemplateParameter[[T]]>
       struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
-      extern template struct $Class[[Tmpl]]<float>;
-      template struct $Class[[Tmpl]]<double>;
+      extern template struct $Class[[Tmpl]]<$Primitive[[float]]>;
+      template struct $Class[[Tmpl]]<$Primitive[[double]]>;
     )cpp",
     // This test is to guard against highlightings disappearing when using
     // conversion operators as their behaviour in the clang AST differ from
     // other CXXMethodDecls.
     R"cpp(
       class $Class[[Foo]] {};
       struct $Class[[Bar]] {
         explicit operator $Class[[Foo]]*() const;
-        explicit operator int() const;
+        explicit operator $Primitive[[int]]() const;
         operator $Class[[Foo]]();
       };
-      void $Function[[f]]() {
+      $Primitive[[void]] $Function[[f]]() {
         $Class[[Bar]] $Variable[[B]];
         $Class[[Foo]] $Variable[[F]] = $Variable[[B]];
         $Class[[Foo]] *$Variable[[FP]] = ($Class[[Foo]]*)$Variable[[B]];
-        int $Variable[[I]] = (int)$Variable[[B]];
+        $Primitive[[int]] $Variable[[I]] = ($Primitive[[int]])$Variable[[B]];
       }
     )cpp"};
   for (const auto &TestCase : TestCases) {
     checkHighlightings(TestCase);
   }
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {

clang-tools-extra/trunk/clangd/unittests/TweakTests.cpp

TEST(TweaksTest, ExtractVariable) {
   }
 }
 
 TEST(TweaksTest, AnnotateHighlightings) {
   llvm::StringLiteral ID = "AnnotateHighlightings";
   checkAvailable(ID, "^vo^id^ ^f(^) {^}^"); // available everywhere.
   checkAvailable(ID, "[[int a; int b;]]");
   const char *Input = "void ^f() {}";
-  const char *Output = "void /* entity.name.function.cpp */f() {}";
+  const char *Output = "/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f() {}";
   checkTransform(ID, Input, Output);
 
   checkTransform(ID,
   R"cpp(
 [[void f1();
 void f2();]]
 )cpp",
   R"cpp(
-void /* entity.name.function.cpp */f1();
-void /* entity.name.function.cpp */f2();
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f1();
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f2();
 )cpp");
 
    checkTransform(ID,
   R"cpp(
 void f1();
 void f2() {^};
 )cpp",
 
   R"cpp(
 void f1();
-void /* entity.name.function.cpp */f2() {};
+/* storage.type.primitive.cpp */void /* entity.name.function.cpp */f2() {};
 )cpp");
 }
 
 TWEAK_TEST(ExpandMacro);
 TEST_F(ExpandMacroTest, Test) {
   Header = R"cpp(
     #define FOO 1 2 3
     #define FUNC(X) X+X+X