[clang][ExtractAPI] Add support for C++ member templates

[evelez7]

Summary

Visit and serialize C++ fields by checking if a var template's context is a CXXRecordDecl in VisitVarTemplateDecl.

Depends on D158027

[evelez7]

Update to use the new Template class constructor instead of old getFragments.

Also better functional style for appending.

[dang]

LGTM!

clang/include/clang/ExtractAPI/API.h

enum RecordKind {
     RK_GlobalVariableTemplatePartialSpecialization,
     RK_EnumConstant,
     RK_Enum,
     RK_StructField,
     RK_Struct,
     RK_Union,
     RK_StaticField,
     RK_CXXField,
+    RK_CXXFieldTemplate,
     RK_CXXClass,
     RK_ClassTemplate,
     RK_ClassTemplateSpecialization,
     RK_ClassTemplatePartialSpecialization,
     RK_Concept,
     RK_CXXStaticMethod,
     RK_CXXInstanceMethod,
     RK_CXXConstructorMethod,
struct CXXFieldRecord : APIRecord {
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_CXXField;
   }
 
 private:
   virtual void anchor();
 };
 
+struct CXXFieldTemplateRecord : CXXFieldRecord {
+  Template Templ;
+
+  CXXFieldTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
+                         AvailabilitySet Availabilities,
+                         const DocComment &Comment,
+                         DeclarationFragments Declaration,
+                         DeclarationFragments SubHeading, AccessControl Access,
+                         Template Template, bool IsFromSystemHeader)
+      : CXXFieldRecord(RK_CXXFieldTemplate, USR, Name, Loc,
+                       std::move(Availabilities), Comment, Declaration,
+                       SubHeading, Access, IsFromSystemHeader),
+        Templ(Template) {}
+
+  static bool classof(const APIRecord *Record) {
+    return Record->getKind() == RK_CXXFieldTemplate;
+  }
+};
+
 struct CXXMethodRecord : APIRecord {
   FunctionSignature Signature;
   AccessControl Access;
 
   CXXMethodRecord() = delete;
 
   CXXMethodRecord(RecordKind Kind, StringRef USR, StringRef Name,
                   PresumedLoc Loc, AvailabilitySet Availabilities,
struct has_function_signature<CXXMethodTemplateSpecRecord>
     : public std::true_type {};
 
 template <typename RecordTy> struct has_access : public std::false_type {};
 template <> struct has_access<CXXMethodRecord> : public std::true_type {};
 template <> struct has_access<CXXFieldRecord> : public std::true_type {};
 template <>
 struct has_access<CXXMethodTemplateRecord> : public std::true_type {};
 template <>
-struct has_access<CXXMethodTemplateSpecRecord>
-    : public std::true_type {};
+struct has_access<CXXMethodTemplateSpecRecord> : public std::true_type {};
+template <>
+struct has_access<CXXFieldTemplateRecord> : public std::true_type {};
 
 template <typename RecordTy> struct has_template : public std::false_type {};
 template <> struct has_template<ClassTemplateRecord> : public std::true_type {};
 template <>
 struct has_template<ClassTemplatePartialSpecRecord> : public std::true_type {};
 template <> struct has_template<ConceptRecord> : public std::true_type {};
 template <>
 struct has_template<GlobalVariableTemplateRecord> : public std::true_type {};
 template <>
 struct has_template<GlobalVariableTemplatePartialSpecRecord>
     : public std::true_type {};
 template <>
 struct has_template<GlobalFunctionTemplateRecord> : public std::true_type {};
 template <>
 struct has_template<CXXMethodTemplateRecord> : public std::true_type {};
+template <>
+struct has_template<CXXFieldTemplateRecord> : public std::true_type {};
 
 /// APISet holds the set of API records collected from given inputs.
 class APISet {
 public:
   /// Create and add a global variable record into the API set.
   ///
   /// Note: the caller is responsible for keeping the StringRef \p Name and
   /// \p USR alive. APISet::copyString provides a way to copy strings into
public:
   CXXFieldRecord *addCXXField(CXXClassRecord *CXXClass, StringRef Name,
                               StringRef USR, PresumedLoc Loc,
                               AvailabilitySet Availabilities,
                               const DocComment &Comment,
                               DeclarationFragments Declaration,
                               DeclarationFragments SubHeading,
                               AccessControl Access, bool IsFromSystemHeader);
 
+  CXXFieldTemplateRecord *addCXXFieldTemplate(
+      APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
+      AvailabilitySet Availability, const DocComment &Comment,
+      DeclarationFragments Declaration, DeclarationFragments SubHeading,
+      AccessControl Access, Template Template, bool IsFromSystemHeader);
+
   CXXClassRecord *
   addCXXClass(StringRef Name, StringRef USR, PresumedLoc Loc,
               AvailabilitySet Availability, const DocComment &Comment,
               DeclarationFragments Declaration, DeclarationFragments SubHeading,
               APIRecord::RecordKind Kind, bool IsFromSystemHeader);
 
   ClassTemplateRecord *
   addClassTemplate(StringRef Name, StringRef USR, PresumedLoc Loc,
public:
   const RecordMap<CXXClassRecord> &getCXXClasses() const { return CXXClasses; }
   const RecordMap<CXXMethodTemplateRecord> &getCXXMethodTemplates() const {
     return CXXMethodTemplates;
   }
   const RecordMap<CXXMethodTemplateSpecRecord> &
   getCXXMethodTemplateSpecializations() const {
     return CXXMethodTemplateSpecializations;
   }
+  const RecordMap<CXXFieldTemplateRecord> &getCXXFieldTemplates() const {
+    return CXXFieldTemplates;
+  }
   const RecordMap<ConceptRecord> &getConcepts() const { return Concepts; }
   const RecordMap<ClassTemplateRecord> &getClassTemplates() const {
     return ClassTemplates;
   }
   const RecordMap<ClassTemplateSpecRecord> &
   getClassTemplateSpecializations() const {
     return ClassTemplateSpecializations;
   }
RecordMap<GlobalVariableTemplatePartialSpecRecord>
       GlobalVariableTemplatePartialSpecializations;
   RecordMap<ConceptRecord> Concepts;
   RecordMap<StaticFieldRecord> StaticFields;
   RecordMap<EnumRecord> Enums;
   RecordMap<StructRecord> Structs;
   RecordMap<CXXClassRecord> CXXClasses;
   RecordMap<CXXMethodTemplateRecord> CXXMethodTemplates;
   RecordMap<CXXMethodTemplateSpecRecord> CXXMethodTemplateSpecializations;
+  RecordMap<CXXFieldTemplateRecord> CXXFieldTemplates;
   RecordMap<ClassTemplateRecord> ClassTemplates;
   RecordMap<ClassTemplateSpecRecord> ClassTemplateSpecializations;
   RecordMap<ClassTemplatePartialSpecRecord> ClassTemplatePartialSpecializations;
   RecordMap<ObjCCategoryRecord> ObjCCategories;
   RecordMap<ObjCInterfaceRecord> ObjCInterfaces;
   RecordMap<ObjCProtocolRecord> ObjCProtocols;
   RecordMap<MacroDefinitionRecord> Macros;
   RecordMap<TypedefRecord> Typedefs;

clang/include/clang/ExtractAPI/ExtractAPIVisitor.h

bool ExtractAPIVisitorBase<Derived>::VisitVarTemplateDecl(
   LinkageInfo Linkage = Decl->getLinkageAndVisibility();
   DocComment Comment;
   if (auto *RawComment =
           getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl))
     Comment = RawComment->getFormattedLines(Context.getSourceManager(),
                                             Context.getDiagnostics());
 
   // Build declaration fragments and sub-heading for the variable.
-  DeclarationFragments Declaration =
-      DeclarationFragmentsBuilder::getFragmentsForVarTemplate(
-          Decl->getTemplatedDecl());
+  DeclarationFragments Declaration;
+  Declaration.append(
+      DeclarationFragmentsBuilder::getFragmentsForRedeclarableTemplate(Decl));
+  Declaration.append(DeclarationFragmentsBuilder::getFragmentsForVarTemplate(
+      Decl->getTemplatedDecl()));
+  // Inject template fragments before var fragments.
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
 
-  // Inject template fragments before var fragments.
-  Declaration.insert(
-      Declaration.begin(),
-      DeclarationFragmentsBuilder::getFragmentsForRedeclarableTemplate(Decl));
-
-  API.addGlobalVariableTemplate(Name, USR, Loc, AvailabilitySet(Decl), Linkage,
-                                Comment, Declaration, SubHeading,
-                                DeclarationFragmentsBuilder::getTemplate(Decl),
-                                isInSystemHeader(Decl));
+  SmallString<128> ParentUSR;
+  index::generateUSRForDecl(dyn_cast<CXXRecordDecl>(Decl->getDeclContext()),
+                            ParentUSR);
+  if (Decl->getDeclContext()->getDeclKind() == Decl::CXXRecord)
+    API.addCXXFieldTemplate(
+        API.findRecordForUSR(ParentUSR), Name, USR, Loc, AvailabilitySet(Decl),
+        DeclarationFragmentsBuilder::getAccessControl(Decl),
+        DeclarationFragmentsBuilder::getTemplate(Decl), isInSystemHeader(Decl));
+  else
+    API.addGlobalVariableTemplate(
+        Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration,
+        SubHeading, DeclarationFragmentsBuilder::getTemplate(Decl),
   return true;
 }
 
 template <typename Derived>
 bool ExtractAPIVisitorBase<Derived>::VisitVarTemplateSpecializationDecl(
     const VarTemplateSpecializationDecl *Decl) {
   if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl))
     return true;

clang/include/clang/ExtractAPI/Serialization/SerializerBase.h

void traverseAPISet() {
     getDerived()->traverseClassTemplateSpecRecords();
 
     getDerived()->traverseClassTemplatePartialSpecRecords();
 
     getDerived()->traverseCXXMethodTemplates();
 
     getDerived()->traverseCXXMethodTemplateSpecializations();
 
+    getDerived()->traverseCXXFieldTemplates();
+
     getDerived()->traverseConcepts();
 
     getDerived()->traverseGlobalVariableTemplateRecords();
 
     getDerived()->traverseGlobalVariableTemplateSpecRecords();
 
     getDerived()->traverseGlobalVariableTemplatePartialSpecRecords();
 
public:
 
   void traverseClassTemplatePartialSpecRecords() {
     for (const auto &ClassTemplatePartialSpec :
          API.getClassTemplatePartialSpecializations())
       getDerived()->visitClassTemplatePartialSpecRecord(
           *ClassTemplatePartialSpec.second);
   }
 
+  void traverseCXXFieldTemplates() {
+    for (const auto &CXXFieldTemplate : API.getCXXFieldTemplates())
+      getDerived()->visitCXXFieldTemplateRecord(*CXXFieldTemplate.second);
+  }
+
   void traverseGlobalVariableTemplateRecords() {
     for (const auto &GlobalVariableTemplate : API.getGlobalVariableTemplates())
       getDerived()->visitGlobalVariableTemplateRecord(
           *GlobalVariableTemplate.second);
   }
 
   void traverseGlobalVariableTemplateSpecRecords() {
     for (const auto &GlobalVariableTemplateSpec :
public:
   void visitClassTemplatePartialSpecRecord(
       const ClassTemplatePartialSpecRecord &Record){};
 
   void visitMethodTemplateRecord(const CXXMethodTemplateRecord &Record){};
 
   void
   visitMethodTemplateSpecRecord(const CXXMethodTemplateSpecRecord &Record){};
 
+  void visitCXXFieldTemplateRecord(const CXXFieldTemplateRecord &Record){};
+
   void visitGlobalVariableTemplateRecord(
       const GlobalVariableTemplateRecord &Record) {}
 
   void visitGlobalVariableTemplateSpecRecord(
       const GlobalVariableTemplateSpecRecord &Record){};
 
   void visitGlobalVariableTemplatePartialSpecRecord(
       const GlobalVariableTemplatePartialSpecRecord &Record){};

clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h

public:
 
   void visitClassTemplatePartialSpecRecord(
       const ClassTemplatePartialSpecRecord &Record);
 
   void visitMethodTemplateRecord(const CXXMethodTemplateRecord &Record);
 
   void visitMethodTemplateSpecRecord(const CXXMethodTemplateSpecRecord &Record);
 
+  void visitCXXFieldTemplateRecord(const CXXFieldTemplateRecord &Record);
+
   void visitConceptRecord(const ConceptRecord &Record);
 
   void
   visitGlobalVariableTemplateRecord(const GlobalVariableTemplateRecord &Record);
 
   void visitGlobalVariableTemplateSpecRecord(
       const GlobalVariableTemplateSpecRecord &Record);
 

clang/lib/ExtractAPI/API.cpp

auto Record = std::make_unique<CXXFieldRecord>(
       USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
       SubHeading, Access, IsFromSystemHeader);
   Record->ParentInformation = APIRecord::HierarchyInformation(
       CXXClass->USR, CXXClass->Name, CXXClass->getKind(), CXXClass);
   USRBasedLookupTable.insert({USR, Record.get()});
   return CXXClass->Fields.emplace_back(std::move(Record)).get();
 }
 
+CXXFieldTemplateRecord *APISet::addCXXFieldTemplate(
+    APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
+    AvailabilitySet Availability, const DocComment &Comment,
+    DeclarationFragments Declaration, DeclarationFragments SubHeading,
+    AccessControl Access, Template Template, bool IsFromSystemHeader) {
+  auto *Record =
+      addTopLevelRecord(USRBasedLookupTable, CXXFieldTemplates, USR, Name, Loc,
+                        std::move(Availability), Comment, Declaration,
+                        SubHeading, Access, Template, IsFromSystemHeader);
+  Record->ParentInformation = APIRecord::HierarchyInformation(
+      Parent->USR, Parent->Name, Parent->getKind(), Parent);
+
+  return Record;
+}
+
 CXXClassRecord *
 APISet::addCXXClass(StringRef Name, StringRef USR, PresumedLoc Loc,
                     AvailabilitySet Availabilities, const DocComment &Comment,
                     DeclarationFragments Declaration,
                     DeclarationFragments SubHeading, APIRecord::RecordKind Kind,
                     bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, CXXClasses, USR, Name, Loc,
                            std::move(Availabilities), Comment, Declaration,

clang/lib/ExtractAPI/DeclarationFragments.cpp

DeclarationFragmentsBuilder::getFragmentsForVarTemplate(const VarDecl *Var) {
   if (Var->isConstexpr())
     Fragments.append("constexpr", DeclarationFragments::FragmentKind::Keyword)
         .appendSpace();
   QualType T =
       Var->getTypeSourceInfo()
           ? Var->getTypeSourceInfo()->getType()
           : Var->getASTContext().getUnqualifiedObjCPointerType(Var->getType());
 
+  // Might be a member, so might be static.
+  if (Var->isStaticDataMember())
+    Fragments.append("static", DeclarationFragments::FragmentKind::Keyword)
+        .appendSpace();
+
   DeclarationFragments After;
   DeclarationFragments ArgumentFragment =
       getFragmentsForType(T, Var->getASTContext(), After);
   if (ArgumentFragment.begin()->Spelling.substr(0, 14).compare(
           "type-parameter") == 0) {
     std::string ProperArgName = getNameForTemplateArgument(
         Var->getDescribedVarTemplate()->getTemplateParameters()->asArray(),
         ArgumentFragment.begin()->Spelling);

clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp

Object serializeSymbolKind(APIRecord::RecordKind RK, Language Lang) {
   case APIRecord::RK_CXXMethodTemplate:
     Kind["identifier"] = AddLangPrefix("method");
     Kind["displayName"] = "Method Template";
     break;
   case APIRecord::RK_CXXMethodTemplateSpecialization:
     Kind["identifier"] = AddLangPrefix("method");
     Kind["displayName"] = "Method Template Specialization";
     break;
+  case APIRecord::RK_CXXFieldTemplate:
+    Kind["identifier"] = AddLangPrefix("property");
+    Kind["displayName"] = "Template Property";
+    break;
   case APIRecord::RK_Concept:
     Kind["identifier"] = AddLangPrefix("concept");
     Kind["displayName"] = "Concept";
     break;
   case APIRecord::RK_CXXStaticMethod:
     Kind["identifier"] = AddLangPrefix("type.method");
     Kind["displayName"] = "Static Method";
     break;
void SymbolGraphSerializer::visitMethodTemplateSpecRecord(
   auto MethodTemplateSpec = serializeAPIRecord(Record);
   if (!MethodTemplateSpec)
     return;
   Symbols.emplace_back(std::move(*MethodTemplateSpec));
   serializeRelationship(RelationshipKind::MemberOf, Record,
                         Record.ParentInformation.ParentRecord);
 }
 
+void SymbolGraphSerializer::visitCXXFieldTemplateRecord(
+    const CXXFieldTemplateRecord &Record) {
+  if (!ShouldRecurse)
+    // Ignore child symbols
+    return;
+  auto CXXFieldTemplate = serializeAPIRecord(Record);
+  if (!CXXFieldTemplate)
+    return;
+  Symbols.emplace_back(std::move(*CXXFieldTemplate));
+  serializeRelationship(RelationshipKind::MemberOf, Record,
+                        Record.ParentInformation.ParentRecord);
+}
+
 void SymbolGraphSerializer::visitConceptRecord(const ConceptRecord &Record) {
   auto Concept = serializeAPIRecord(Record);
   if (!Concept)
     return;
 
   Symbols.emplace_back(std::move(*Concept));
 }
 

clang/test/ExtractAPI/field_template.cpp

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \
+// RUN: %t/reference.output.json.in >> %t/reference.output.json
+// RUN: %clang_cc1 -extract-api -triple arm64-apple-macosx \
+// RUN:   -x c++-header %t/input.h -o %t/output.json -verify
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+//--- input.h
+class Foo {
+  template<typename T> static T Bar;
+};
+
+/// expected-no-diagnostics
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationships": [
+    {
+      "kind": "memberOf",
+      "source": "c:@S@Foo@Bar",
+      "target": "c:@S@Foo",
+      "targetFallback": "Foo"
+    }
+  ],
+  "symbols": [
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "class"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "Foo"
+        },
+        {
+          "kind": "text",
+          "spelling": ";"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c++",
+        "precise": "c:@S@Foo"
+      },
+      "kind": {
+        "displayName": "Class",
+        "identifier": "c++.class"
+      },
+      "location": {
+        "position": {
+          "character": 7,
+          "line": 1
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "Foo"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "Foo"
+          }
+        ],
+        "title": "Foo"
+      },
+      "pathComponents": [
+        "Foo"
+      ]
+    },
+    {
+      "accessLevel": "private",
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "template"
+        },
+        {
+          "kind": "text",
+          "spelling": "<"
+        },
+        {
+          "kind": "keyword",
+          "spelling": "typename"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "genericParameter",
+          "spelling": "T"
+        },
+        {
+          "kind": "text",
+          "spelling": "> "
+        },
+        {
+          "kind": "keyword",
+          "spelling": "static"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:t0.0",
+          "spelling": "T"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "Bar"
+        },
+        {
+          "kind": "text",
+          "spelling": ";"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c++",
+        "precise": "c:@S@Foo@Bar"
+      },
+      "kind": {
+        "displayName": "Template Property",
+        "identifier": "c++.property"
+      },
+      "location": {
+        "position": {
+          "character": 33,
+          "line": 2
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "Bar"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "Bar"
+          }
+        ],
+        "title": "Bar"
+      },
+      "pathComponents": [
+        "Foo",
+        "Bar"
+      ],
+      "swiftGenerics": {
+        "parameters": [
+          {
+            "depth": 0,
+            "index": 0,
+            "name": "T"
+          }
+        ]
+      }
+    }
+  ]
+}