[clang][extractapi] Tie API and serialization to the FrontendAction

[dang]

Summary

Make the API records a property of the action instead of the ASTVisitor
so that it can be accessed outside the AST visitation and push back
serialization to the end of the frontend action.

This will allow accessing and modifying the API records outside of the
ASTVisitor, which is a prerequisite for supporting macros.

[dang]

Some errors slipped by when splitting this out of the macros patch.

[zixuw]

Mostly LGTM after addressing the inline comments.

clang/include/clang/ExtractAPI/FrontendActions.h

42–58 ↗

(On Diff #418907)

Should these methods be private?

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

590

Need rebase to pull in the Language fix D122495

[dang]

Rebase on top of latest changes

[dang]

Update using the correct patch this time around.

clang/include/clang/ExtractAPI/FrontendActions.h

42–58 ↗

(On Diff #418907)

Yes I checked and existing frontend actions mark these as private.

[zixuw]

LGTM!

[dang]

a9909d23e9bb8c4649cba1c14d479c28df4ca185

clang/include/clang/ExtractAPI/API.h

 #include "clang/ExtractAPI/AvailabilityInfo.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include <memory>
+#include <type_traits>
 
 namespace clang {
 namespace extractapi {
 
 /// DocComment is a vector of RawComment::CommentLine.
 ///
 /// Each line represents one line of striped documentation comment,
 /// with source range information. This simplifies calculating the source
 /// location of a character in the doc comment for pointing back to the source
 /// file.
 /// e.g.
 /// \code
 ///   /// This is a documentation comment
 ///       ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'  First line.
 ///   ///     with multiple lines.
 ///       ^~~~~~~~~~~~~~~~~~~~~~~'         Second line.
 /// \endcode
 using DocComment = std::vector<RawComment::CommentLine>;
 
+// Classes deriving from APIRecord need to have Name be the first constructor
+// argument. This is so that they are compatible with `addTopLevelRecord`
+// defined in API.cpp
 /// The base representation of an API record. Holds common symbol information.
 struct APIRecord {
   StringRef Name;
   StringRef USR;
   PresumedLoc Location;
   AvailabilityInfo Availability;
   LinkageInfo Linkage;
 
enum RecordKind {
     RK_Enum,
     RK_StructField,
     RK_Struct,
     RK_ObjCProperty,
     RK_ObjCIvar,
     RK_ObjCMethod,
     RK_ObjCInterface,
     RK_ObjCProtocol,
+    RK_MacroDefinition,
   };
 
 private:
   const RecordKind Kind;
 
 public:
   RecordKind getKind() const { return Kind; }
 

 
 /// This holds information associated with global variables or functions.
 struct GlobalRecord : APIRecord {
   GVKind GlobalKind;
 
   /// The function signature of the record if it is a function.
   FunctionSignature Signature;
 
-  GlobalRecord(GVKind Kind, StringRef Name, StringRef USR, PresumedLoc Loc,
+  GlobalRecord(StringRef Name, StringRef USR, PresumedLoc Loc,
                const AvailabilityInfo &Availability, LinkageInfo Linkage,
                const DocComment &Comment, DeclarationFragments Declaration,
-               DeclarationFragments SubHeading, FunctionSignature Signature)
+               DeclarationFragments SubHeading, GVKind Kind,
+               FunctionSignature Signature)
       : APIRecord(RK_Global, Name, USR, Loc, Availability, Linkage, Comment,
                   Declaration, SubHeading),
         GlobalKind(Kind), Signature(Signature) {}
 
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_Global;
   }
 
struct ObjCProtocolRecord : ObjCContainerRecord {
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_ObjCProtocol;
   }
 
 private:
   virtual void anchor();
 };
 
+struct MacroDefinitionRecord : APIRecord {
+  MacroDefinitionRecord(StringRef Name, StringRef USR, PresumedLoc Loc,
+                        DeclarationFragments Declaration,
+                        DeclarationFragments SubHeading)
+      : APIRecord(RK_MacroDefinition, Name, USR, Loc, AvailabilityInfo(),
+                  LinkageInfo(), {}, Declaration, SubHeading) {}
+
+  static bool classof(const APIRecord *Record) {
+    return Record->getKind() == RK_MacroDefinition;
+  }
+
+private:
+  virtual void anchor();
+};
+
 /// APISet holds the set of API records collected from given inputs.
 class APISet {
 public:
   /// Create and add a GlobalRecord of kind \p Kind into the API set.
   ///
   /// Note: the caller is responsible for keeping the StringRef \p Name and
   /// \p USR alive. APISet::copyString provides a way to copy strings into
   /// APISet itself, and APISet::recordUSR(const Decl *D) is a helper method
public:
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCProtocolRecord *addObjCProtocol(StringRef Name, StringRef USR,
                                       PresumedLoc Loc,
                                       const AvailabilityInfo &Availability,
                                       const DocComment &Comment,
                                       DeclarationFragments Declaration,
                                       DeclarationFragments SubHeading);
 
-  /// A map to store the set of GlobalRecord%s with the declaration name as the
-  /// key.
-  using GlobalRecordMap =
-      llvm::MapVector<StringRef, std::unique_ptr<GlobalRecord>>;
-
-  /// A map to store the set of EnumRecord%s with the declaration name as the
-  /// key.
-  using EnumRecordMap = llvm::MapVector<StringRef, std::unique_ptr<EnumRecord>>;
-
-  /// A map to store the set of StructRecord%s with the declaration name as the
-  /// key.
-  using StructRecordMap =
-      llvm::MapVector<StringRef, std::unique_ptr<StructRecord>>;
-
-  /// A map to store the set of ObjCInterfaceRecord%s with the declaration name
-  /// as the key.
-  using ObjCInterfaceRecordMap =
-      llvm::MapVector<StringRef, std::unique_ptr<ObjCInterfaceRecord>>;
-
-  /// A map to store the set of ObjCProtocolRecord%s with the declaration name
-  /// as the key.
-  using ObjCProtocolRecordMap =
-      llvm::MapVector<StringRef, std::unique_ptr<ObjCProtocolRecord>>;
+  /// Create a macro definition record into the API set.
+  ///
+  /// Note: the caller is responsible for keeping the StringRef \p Name and
+  /// \p USR alive. APISet::copyString provides a way to copy strings into
+  /// APISet itself, and APISet::recordUSRForMacro(StringRef Name,
+  /// SourceLocation SL, const SourceManager &SM) is a helper method to generate
+  /// the USR for the macro and keep it alive in APISet.
+  MacroDefinitionRecord *addMacroDefinition(StringRef Name, StringRef USR,
+                                            PresumedLoc Loc,
+                                            DeclarationFragments Declaration,
+                                            DeclarationFragments SubHeading);
+
+  /// A mapping type to store a set of APIRecord%s with the declaration name as
+  /// the key.
+  template <typename RecordTy,
+            typename =
+                std::enable_if_t<std::is_base_of<APIRecord, RecordTy>::value>>
+  using RecordMap = llvm::MapVector<StringRef, std::unique_ptr<RecordTy>>;
 
   /// Get the target triple for the ExtractAPI invocation.
   const llvm::Triple &getTarget() const { return Target; }
 
   /// Get the language used by the APIs.
   Language getLanguage() const { return Lang; }
 
-  const GlobalRecordMap &getGlobals() const { return Globals; }
-  const EnumRecordMap &getEnums() const { return Enums; }
-  const StructRecordMap &getStructs() const { return Structs; }
-  const ObjCInterfaceRecordMap &getObjCInterfaces() const {
+  const RecordMap<GlobalRecord> &getGlobals() const { return Globals; }
+  const RecordMap<EnumRecord> &getEnums() const { return Enums; }
+  const RecordMap<StructRecord> &getStructs() const { return Structs; }
+  const RecordMap<ObjCInterfaceRecord> &getObjCInterfaces() const {
     return ObjCInterfaces;
   }
-  const ObjCProtocolRecordMap &getObjCProtocols() const {
+  const RecordMap<ObjCProtocolRecord> &getObjCProtocols() const {
     return ObjCProtocols;
   }
+  const RecordMap<MacroDefinitionRecord> &getMacros() const { return Macros; }
 
   /// Generate and store the USR of declaration \p D.
   ///
   /// Note: The USR string is stored in and owned by Allocator.
   ///
   /// \returns a StringRef of the generated USR string.
   StringRef recordUSR(const Decl *D);
 
+  /// Generate and store the USR for a macro \p Name.
+  ///
+  /// Note: The USR string is stored in and owned by Allocator.
+  ///
+  /// \returns a StringRef to the generate USR string.
+  StringRef recordUSRForMacro(StringRef Name, SourceLocation SL,
+                              const SourceManager &SM);
+
   /// Copy \p String into the Allocator in this APISet.
   ///
   /// \returns a StringRef of the copied string in APISet::Allocator.
   StringRef copyString(StringRef String);
 
   APISet(const llvm::Triple &Target, Language Lang)
       : Target(Target), Lang(Lang) {}
 
 private:
   /// BumpPtrAllocator to store generated/copied strings.
   ///
   /// Note: The main use for this is being able to deduplicate strings.
   llvm::BumpPtrAllocator StringAllocator;
 
   const llvm::Triple Target;
   const Language Lang;
 
-  GlobalRecordMap Globals;
-  EnumRecordMap Enums;
-  StructRecordMap Structs;
-  ObjCInterfaceRecordMap ObjCInterfaces;
-  ObjCProtocolRecordMap ObjCProtocols;
+  RecordMap<GlobalRecord> Globals;
+  RecordMap<EnumRecord> Enums;
+  RecordMap<StructRecord> Structs;
+  RecordMap<ObjCInterfaceRecord> ObjCInterfaces;
+  RecordMap<ObjCProtocolRecord> ObjCProtocols;
+  RecordMap<MacroDefinitionRecord> Macros;
 };
 
 } // namespace extractapi
 } // namespace clang
 
 #endif // LLVM_CLANG_EXTRACTAPI_API_H

clang/include/clang/ExtractAPI/DeclarationFragments.h

 
 #ifndef LLVM_CLANG_EXTRACTAPI_DECLARATION_FRAGMENTS_H
 #define LLVM_CLANG_EXTRACTAPI_DECLARATION_FRAGMENTS_H
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include <vector>
 
 namespace clang {
 namespace extractapi {
 
 /// DeclarationFragments is a vector of tagged important parts of a symbol's
 /// declaration.
public:
   static DeclarationFragments
   getFragmentsForObjCProperty(const ObjCPropertyDecl *);
 
   /// Build DeclarationFragments for an Objective-C protocol declaration
   /// ObjCProtocolDecl.
   static DeclarationFragments
   getFragmentsForObjCProtocol(const ObjCProtocolDecl *);
 
+  /// Build DeclarationFragments for a macro.
+  ///
+  /// \param Name name of the macro.
+  /// \param MD the associated MacroDirective.
+  static DeclarationFragments getFragmentsForMacro(StringRef Name,
+                                                   const MacroDirective *MD);
+
   /// Build sub-heading fragments for a NamedDecl.
   static DeclarationFragments getSubHeading(const NamedDecl *);
 
   /// Build sub-heading fragments for an Objective-C method.
   static DeclarationFragments getSubHeading(const ObjCMethodDecl *);
 
+  /// Build a sub-heading for macro \p Name.
+  static DeclarationFragments getSubHeadingForMacro(StringRef Name);
+
   /// Build FunctionSignature for a function-like declaration \c FunctionT like
   /// FunctionDecl or ObjCMethodDecl.
   ///
   /// The logic and implementation of building a signature for a FunctionDecl
   /// and an ObjCMethodDecl are exactly the same, but they do not share a common
   /// base. This template helps reuse the code.
   template <typename FunctionT>
   static FunctionSignature getFunctionSignature(const FunctionT *);

clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h

private:
   void serializeEnumRecord(const EnumRecord &Record);
 
   /// Serialize a struct record.
   void serializeStructRecord(const StructRecord &Record);
 
   /// Serialize an Objective-C container record.
   void serializeObjCContainerRecord(const ObjCContainerRecord &Record);
 
+  /// Serialize a macro defintion record.
+  void serializeMacroDefinitionRecord(const MacroDefinitionRecord &Record);
+
 public:
   SymbolGraphSerializer(const APISet &API, StringRef ProductName,
                         APISerializerOption Options = {})
       : APISerializer(API, ProductName, Options) {}
 };
 
 } // namespace extractapi
 } // namespace clang
 
 #endif // LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SYMBOLGRAPHSERIALIZER_H

clang/lib/ExtractAPI/API.cpp

 ///
 //===----------------------------------------------------------------------===//
 
 #include "clang/ExtractAPI/API.h"
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/AST/CommentLexer.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Index/USRGeneration.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include <memory>
 
 using namespace clang::extractapi;
 using namespace llvm;
 
+namespace {
+
+template <typename RecordTy, typename... CtorArgsTy>
+RecordTy *addTopLevelRecord(APISet::RecordMap<RecordTy> &RecordMap,
+                            StringRef Name, CtorArgsTy &&...CtorArgs) {
+  auto Result = RecordMap.insert({Name, nullptr});
+
+  // Create the record if it does not already exist
+  if (Result.second)
+    Result.first->second =
+        std::make_unique<RecordTy>(Name, std::forward<CtorArgsTy>(CtorArgs)...);
+
+  return Result.first->second.get();
+}
+
+} // namespace
+
 GlobalRecord *APISet::addGlobal(GVKind Kind, StringRef Name, StringRef USR,
                                 PresumedLoc Loc,
                                 const AvailabilityInfo &Availability,
                                 LinkageInfo Linkage, const DocComment &Comment,
                                 DeclarationFragments Fragments,
                                 DeclarationFragments SubHeading,
                                 FunctionSignature Signature) {
-  auto Result = Globals.insert({Name, nullptr});
-  if (Result.second) {
-    // Create the record if it does not already exist.
-    auto Record = std::make_unique<GlobalRecord>(
-        Kind, Name, USR, Loc, Availability, Linkage, Comment, Fragments,
-        SubHeading, Signature);
-    Result.first->second = std::move(Record);
-  }
-  return Result.first->second.get();
+  return addTopLevelRecord(Globals, Name, USR, Loc, Availability, Linkage,
+                           Comment, Fragments, SubHeading, Kind, Signature);
 }
 
 GlobalRecord *
 APISet::addGlobalVar(StringRef Name, StringRef USR, PresumedLoc Loc,
                      const AvailabilityInfo &Availability, LinkageInfo Linkage,
                      const DocComment &Comment, DeclarationFragments Fragments,
                      DeclarationFragments SubHeading) {
   return addGlobal(GVKind::Variable, Name, USR, Loc, Availability, Linkage,
EnumConstantRecord *APISet::addEnumConstant(
   return Enum->Constants.emplace_back(std::move(Record)).get();
 }
 
 EnumRecord *APISet::addEnum(StringRef Name, StringRef USR, PresumedLoc Loc,
                             const AvailabilityInfo &Availability,
                             const DocComment &Comment,
                             DeclarationFragments Declaration,
                             DeclarationFragments SubHeading) {
-  auto Result = Enums.insert({Name, nullptr});
-  if (Result.second) {
-    // Create the record if it does not already exist.
-    auto Record = std::make_unique<EnumRecord>(
-        Name, USR, Loc, Availability, Comment, Declaration, SubHeading);
-    Result.first->second = std::move(Record);
-  }
-  return Result.first->second.get();
+  return addTopLevelRecord(Enums, Name, USR, Loc, Availability, Comment,
+                           Declaration, SubHeading);
 }
 
 StructFieldRecord *APISet::addStructField(StructRecord *Struct, StringRef Name,
                                           StringRef USR, PresumedLoc Loc,
                                           const AvailabilityInfo &Availability,
                                           const DocComment &Comment,
                                           DeclarationFragments Declaration,
                                           DeclarationFragments SubHeading) {
   auto Record = std::make_unique<StructFieldRecord>(
       Name, USR, Loc, Availability, Comment, Declaration, SubHeading);
   return Struct->Fields.emplace_back(std::move(Record)).get();
 }
 
 StructRecord *APISet::addStruct(StringRef Name, StringRef USR, PresumedLoc Loc,
                                 const AvailabilityInfo &Availability,
                                 const DocComment &Comment,
                                 DeclarationFragments Declaration,
                                 DeclarationFragments SubHeading) {
-  auto Result = Structs.insert({Name, nullptr});
-  if (Result.second) {
-    // Create the record if it does not already exist.
-    auto Record = std::make_unique<StructRecord>(
-        Name, USR, Loc, Availability, Comment, Declaration, SubHeading);
-    Result.first->second = std::move(Record);
-  }
-  return Result.first->second.get();
+  return addTopLevelRecord(Structs, Name, USR, Loc, Availability, Comment,
+                           Declaration, SubHeading);
 }
 
 ObjCInterfaceRecord *APISet::addObjCInterface(
     StringRef Name, StringRef USR, PresumedLoc Loc,
     const AvailabilityInfo &Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, SymbolReference SuperClass) {
-  auto Result = ObjCInterfaces.insert({Name, nullptr});
-  if (Result.second) {
-    // Create the record if it does not already exist.
-    auto Record = std::make_unique<ObjCInterfaceRecord>(
-        Name, USR, Loc, Availability, Linkage, Comment, Declaration, SubHeading,
+  return addTopLevelRecord(ObjCInterfaces, Name, USR, Loc, Availability,
+                           Linkage, Comment, Declaration, SubHeading,
                            SuperClass);
-    Result.first->second = std::move(Record);
-  }
-  return Result.first->second.get();
 }
 
 ObjCMethodRecord *APISet::addObjCMethod(
     ObjCContainerRecord *Container, StringRef Name, StringRef USR,
     PresumedLoc Loc, const AvailabilityInfo &Availability,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, FunctionSignature Signature,
     bool IsInstanceMethod) {
auto Record = std::make_unique<ObjCInstanceVariableRecord>(
       Name, USR, Loc, Availability, Comment, Declaration, SubHeading, Access);
   return Container->Ivars.emplace_back(std::move(Record)).get();
 }
 
 ObjCProtocolRecord *APISet::addObjCProtocol(
     StringRef Name, StringRef USR, PresumedLoc Loc,
     const AvailabilityInfo &Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading) {
-  auto Result = ObjCProtocols.insert({Name, nullptr});
-  if (Result.second) {
-    // Create the record if it does not already exist.
-    auto Record = std::make_unique<ObjCProtocolRecord>(
-        Name, USR, Loc, Availability, Comment, Declaration, SubHeading);
-    Result.first->second = std::move(Record);
-  }
-  return Result.first->second.get();
+  return addTopLevelRecord(ObjCProtocols, Name, USR, Loc, Availability, Comment,
+                           Declaration, SubHeading);
+
+MacroDefinitionRecord *
+APISet::addMacroDefinition(StringRef Name, StringRef USR, PresumedLoc Loc,
+                           DeclarationFragments Declaration,
+                           DeclarationFragments SubHeading) {
+  return addTopLevelRecord(Macros, Name, USR, Loc, Declaration, SubHeading);
 }
 
 StringRef APISet::recordUSR(const Decl *D) {
   SmallString<128> USR;
   index::generateUSRForDecl(D, USR);
   return copyString(USR);
 }
 
+StringRef APISet::recordUSRForMacro(StringRef Name, SourceLocation SL,
+                                    const SourceManager &SM) {
+  SmallString<128> USR;
+  index::generateUSRForMacro(Name, SL, SM, USR);
+  return copyString(USR);
+}
+
 StringRef APISet::copyString(StringRef String) {
   if (String.empty())
     return {};
 
   // No need to allocate memory and copy if the string has already been stored.
   if (StringAllocator.identifyObject(String.data()))
     return String;
 

 void EnumRecord::anchor() {}
 void StructFieldRecord::anchor() {}
 void StructRecord::anchor() {}
 void ObjCPropertyRecord::anchor() {}
 void ObjCInstanceVariableRecord::anchor() {}
 void ObjCMethodRecord::anchor() {}
 void ObjCInterfaceRecord::anchor() {}
 void ObjCProtocolRecord::anchor() {}
+void MacroDefinitionRecord::anchor() {}

clang/lib/ExtractAPI/DeclarationFragments.cpp

DeclarationFragmentsBuilder::getFragmentsForStruct(const RecordDecl *Record) {
   // struct just use the declaration fragments of the typedef decl.
 
   DeclarationFragments Fragments;
   Fragments.append("struct", DeclarationFragments::FragmentKind::Keyword);
 
   if (!Record->getName().empty())
     Fragments.appendSpace().append(
         Record->getName(), DeclarationFragments::FragmentKind::Identifier);
+  return Fragments;
+}
+
+DeclarationFragments
+DeclarationFragmentsBuilder::getFragmentsForMacro(StringRef Name,
+                                                  const MacroDirective *MD) {
+  DeclarationFragments Fragments;
+  Fragments.append("#define", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace();
+  Fragments.append(Name, DeclarationFragments::FragmentKind::Identifier);
 
+  auto *MI = MD->getMacroInfo();
+
+  if (MI->isFunctionLike()) {
+    Fragments.append("(", DeclarationFragments::FragmentKind::Text);
+    unsigned numParameters = MI->getNumParams();
+    if (MI->isC99Varargs())
+      --numParameters;
+    for (unsigned i = 0; i < numParameters; ++i) {
+      if (i)
+        Fragments.append(", ", DeclarationFragments::FragmentKind::Text);
+      Fragments.append(MI->params()[i]->getName(),
+                       DeclarationFragments::FragmentKind::InternalParam);
+    }
+    if (MI->isVariadic()) {
+      if (numParameters && MI->isC99Varargs())
+        Fragments.append(", ", DeclarationFragments::FragmentKind::Text);
+      Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+    }
+    Fragments.append(")", DeclarationFragments::FragmentKind::Text);
+  }
   return Fragments;
 }
 
 DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForObjCInterface(
     const ObjCInterfaceDecl *Interface) {
   DeclarationFragments Fragments;
   // Build the base of the Objective-C interface declaration.
   Fragments.append("@interface", DeclarationFragments::FragmentKind::Keyword)
DeclarationFragmentsBuilder::getSubHeading(const ObjCMethodDecl *Method) {
   if (Method->isClassMethod())
     Fragments.append("+ ", DeclarationFragments::FragmentKind::Text);
   else if (Method->isInstanceMethod())
     Fragments.append("- ", DeclarationFragments::FragmentKind::Text);
 
   return Fragments.append(Method->getNameAsString(),
                           DeclarationFragments::FragmentKind::Identifier);
 }
+
+// Subheading of a symbol defaults to its name.
+DeclarationFragments
+DeclarationFragmentsBuilder::getSubHeadingForMacro(StringRef Name) {
+  DeclarationFragments Fragments;
+  Fragments.append(Name, DeclarationFragments::FragmentKind::Identifier);
+  return Fragments;
+}

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

 #include "clang/ExtractAPI/API.h"
 #include "clang/ExtractAPI/AvailabilityInfo.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "clang/ExtractAPI/FrontendActions.h"
 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendOptions.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
 using namespace extractapi;
 
 namespace {
void HandleTranslationUnit(ASTContext &Context) override {
     // Use ExtractAPIVisitor to traverse symbol declarations in the context.
     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
   }
 
 private:
   ExtractAPIVisitor Visitor;
 };
 
+class MacroCallback : public PPCallbacks {
+public:
+  MacroCallback(const SourceManager &SM, APISet &API) : SM(SM), API(API) {}
+
+  void MacroDefined(const Token &MacroNameToken,
+                    const MacroDirective *MD) override {
+    auto *MacroInfo = MD->getMacroInfo();
+
+    if (MacroInfo->isBuiltinMacro())
+      return;
+
+    auto SourceLoc = MacroNameToken.getLocation();
+    if (SM.isWrittenInBuiltinFile(SourceLoc) ||
+        SM.isWrittenInCommandLineFile(SourceLoc))
+      return;
+
+    PendingMacros.emplace_back(MacroNameToken, MD);
+  }
+
+  // If a macro gets undefined at some point during preprocessing of the inputs
+  // it means that it isn't an exposed API and we should therefore not add a
+  // macro definition for it.
+  void MacroUndefined(const Token &MacroNameToken, const MacroDefinition &MD,
+                      const MacroDirective *Undef) override {
+    llvm::erase_if(PendingMacros, [&MD](const PendingMacro &PM) {
+      return MD.getMacroInfo()->getDefinitionLoc() ==
+             PM.MD->getMacroInfo()->getDefinitionLoc();
+    });
+  }
+
+  void EndOfMainFile() override {
+    for (auto &PM : PendingMacros) {
+      // `isUsedForHeaderGuard` is only set when the preprocessor leaves the
+      // file so check for it here.
+      if (PM.MD->getMacroInfo()->isUsedForHeaderGuard())
+        continue;
+
+      StringRef Name = PM.MacroNameToken.getIdentifierInfo()->getName();
+      PresumedLoc Loc = SM.getPresumedLoc(PM.MacroNameToken.getLocation());
+      StringRef USR =
+          API.recordUSRForMacro(Name, PM.MacroNameToken.getLocation(), SM);
+
+      API.addMacroDefinition(
+          Name, USR, Loc,
+          DeclarationFragmentsBuilder::getFragmentsForMacro(Name, PM.MD),
+          DeclarationFragmentsBuilder::getSubHeadingForMacro(Name));
+    }
+
+    PendingMacros.clear();
+  }
+
+private:
+  struct PendingMacro {
+    Token MacroNameToken;
+    const MacroDirective *MD;
+
+    PendingMacro(const Token &MacroNameToken, const MacroDirective *MD)
+        : MacroNameToken(MacroNameToken), MD(MD) {}
+  };
+
+  const SourceManager &SM;
+  APISet &API;
+  llvm::SmallVector<PendingMacro> PendingMacros;
+};
+
 } // namespace
 
 std::unique_ptr<ASTConsumer>
 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   OS = CreateOutputFile(CI, InFile);
   if (!OS)
     return nullptr;
 
   ProductName = CI.getFrontendOpts().ProductName;
 
   // Now that we have enough information about the language options and the
   // target triple, let's create the APISet before anyone uses it.
   API = std::make_unique<APISet>(
       CI.getTarget().getTriple(),
       CI.getFrontendOpts().Inputs.back().getKind().getLanguage());
 
+  // Register preprocessor callbacks that will add macro definitions to API.
+  CI.getPreprocessor().addPPCallbacks(
+      std::make_unique<MacroCallback>(CI.getSourceManager(), *API));
+
   return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(), *API);
 }
 
 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
   auto &Inputs = CI.getFrontendOpts().Inputs;
   if (Inputs.empty())
     return true;
 
   auto Kind = Inputs[0].getKind();
 
   // Convert the header file inputs into a single input buffer.
   SmallString<256> HeaderContents;
   for (const FrontendInputFile &FIF : Inputs) {
     if (Kind.isObjectiveC())
       HeaderContents += "#import";
     else
       HeaderContents += "#include";
     HeaderContents += " \"";
     HeaderContents += FIF.getFile();
     HeaderContents += "\"\n";
   }
 
-  Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
+  auto Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
                                                      getInputBufferName());
 
   // Set that buffer up as our "real" input in the CompilerInstance.
   Inputs.clear();
   Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);
 
+  // Tell the processor about the input file.
+  CI.getPreprocessorOpts().addRemappedFile(Buffer->getBufferIdentifier(),
+                                           Buffer.release());
+
   return true;
 }
 
 void ExtractAPIAction::EndSourceFileAction() {
   if (!OS)
     return;
 
   // Setup a SymbolGraphSerializer to write out collected API information in

clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp

Object serializeSymbolKind(const APIRecord &Record, Language Lang) {
   case APIRecord::RK_ObjCInterface:
     Kind["identifier"] = AddLangPrefix("class");
     Kind["displayName"] = "Class";
     break;
   case APIRecord::RK_ObjCProtocol:
     Kind["identifier"] = AddLangPrefix("protocol");
     Kind["displayName"] = "Protocol";
     break;
+  case APIRecord::RK_MacroDefinition:
+    Kind["identifier"] = AddLangPrefix("macro");
+    Kind["displayName"] = "Macro";
   }
 
   return Kind;
 }
 
 } // namespace
 
 void SymbolGraphSerializer::anchor() {}
void SymbolGraphSerializer::serializeObjCContainerRecord(
   if (auto *ObjCInterface = dyn_cast<ObjCInterfaceRecord>(&Record))
     if (!ObjCInterface->SuperClass.empty())
       // If Record is an Objective-C interface record and it has a super class,
       // record that Record is inherited from SuperClass.
       serializeRelationship(RelationshipKind::InheritsFrom, Record,
                             ObjCInterface->SuperClass);
 }
 
+void SymbolGraphSerializer::serializeMacroDefinitionRecord(
+    const MacroDefinitionRecord &Record) {
+  auto Macro = serializeAPIRecord(Record);
+  if (!Macro)
+    return;
+
+  Symbols.emplace_back(std::move(*Macro));
+}
+
 Object SymbolGraphSerializer::serialize() {
   Object Root;
   serializeObject(Root, "metadata", serializeMetadata());
   serializeObject(Root, "module", serializeModule());
 
   // Serialize global records in the API set.
   for (const auto &Global : API.getGlobals())
     serializeGlobalRecord(*Global.second);
Object SymbolGraphSerializer::serialize() {
   // Serialize Objective-C interface records in the API set.
   for (const auto &ObjCInterface : API.getObjCInterfaces())
     serializeObjCContainerRecord(*ObjCInterface.second);
 
   // Serialize Objective-C protocol records in the API set.
   for (const auto &ObjCProtocol : API.getObjCProtocols())
     serializeObjCContainerRecord(*ObjCProtocol.second);
 
+  for (const auto &Macro : API.getMacros())
+    serializeMacroDefinitionRecord(*Macro.second);
+
   Root["symbols"] = std::move(Symbols);
   Root["relationhips"] = std::move(Relationships);
 
   return Root;
 }
 
 void SymbolGraphSerializer::serialize(raw_ostream &os) {
   Object root = serialize();
   if (Options.Compact)
     os << formatv("{0}", Value(std::move(root))) << "\n";
   else
     os << formatv("{0:2}", Value(std::move(root))) << "\n";
 }

clang/test/ExtractAPI/macro_undefined.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/reference.output.json.in >> \
+// RUN: %t/reference.output.json
+// RUN: %clang -extract-api --product-name=Macros -target arm64-apple-macosx \
+// RUN: -x objective-c-header %t/input.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input.h
+#define HELLO 1
+#define FUNC_GEN(NAME, ...) void NAME(__VA_ARGS__);
+FUNC_GEN(foo)
+FUNC_GEN(bar, const int *, unsigned);
+#undef FUNC_GEN
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "Macros",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationhips": [],
+  "symbols": [
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "foo"
+        },
+        {
+          "kind": "text",
+          "spelling": "()"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:@F@foo"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "objective-c.func"
+      },
+      "location": {
+        "character": 1,
+        "line": 3,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "foo"
+          }
+        ],
+        "title": "foo"
+      },
+      "parameters": {
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "bar"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "keyword",
+          "spelling": "const"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": ""
+        },
+        {
+          "kind": "text",
+          "spelling": ", "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:i",
+          "spelling": "unsigned int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "internalParam",
+          "spelling": ""
+        },
+        {
+          "kind": "text",
+          "spelling": ")"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:@F@bar"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "objective-c.func"
+      },
+      "location": {
+        "character": 1,
+        "line": 4,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "bar"
+          }
+        ],
+        "title": "bar"
+      },
+      "parameters": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "keyword",
+                "spelling": "const"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " *"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": ""
+              }
+            ],
+            "name": ""
+          },
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:i",
+                "spelling": "unsigned int"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "internalParam",
+                "spelling": ""
+              }
+            ],
+            "name": ""
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "HELLO"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@8@macro@HELLO"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 1,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "HELLO"
+          }
+        ],
+        "title": "HELLO"
+      }
+    }
+  ]
+}

clang/test/ExtractAPI/macros.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/reference.output.json.in >> \
+// RUN: %t/reference.output.json
+// RUN: %clang -extract-api --product-name=Macros -target arm64-apple-macosx \
+// RUN: -x objective-c-header %t/input.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input.h
+#define HELLO 1
+#define WORLD 2
+#define MACRO_FUN(x) x x
+#define FUN(x, y, z) x + y + z
+#define FUNC99(x, ...)
+#define FUNGNU(x...)
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "Macros",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationhips": [],
+  "symbols": [
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "HELLO"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@8@macro@HELLO"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 1,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "HELLO"
+          }
+        ],
+        "title": "HELLO"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "WORLD"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@24@macro@WORLD"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 2,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "WORLD"
+          }
+        ],
+        "title": "WORLD"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "MACRO_FUN"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "x"
+        },
+        {
+          "kind": "text",
+          "spelling": ")"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@40@macro@MACRO_FUN"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 3,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "MACRO_FUN"
+          }
+        ],
+        "title": "MACRO_FUN"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "FUN"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "x"
+        },
+        {
+          "kind": "text",
+          "spelling": ", "
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "y"
+        },
+        {
+          "kind": "text",
+          "spelling": ", "
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "z"
+        },
+        {
+          "kind": "text",
+          "spelling": ")"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@65@macro@FUN"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 4,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "FUN"
+          }
+        ],
+        "title": "FUN"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "FUNC99"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "x"
+        },
+        {
+          "kind": "text",
+          "spelling": ", ...)"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@96@macro@FUNC99"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 5,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "FUNC99"
+          }
+        ],
+        "title": "FUNC99"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "keyword",
+          "spelling": "#define"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "FUNGNU"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "x"
+        },
+        {
+          "kind": "text",
+          "spelling": "...)"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "objective-c",
+        "precise": "c:input.h@119@macro@FUNGNU"
+      },
+      "kind": {
+        "displayName": "Macro",
+        "identifier": "objective-c.macro"
+      },
+      "location": {
+        "character": 9,
+        "line": 6,
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "FUNGNU"
+          }
+        ],
+        "title": "FUNGNU"
+      }
+    }
+  ]
+}