[clang-tools-extra][ExtractAPI] create clang-symbolgraph-merger

[Arsenic]

Summary

Create a clang tool to merge all the JSON symbolgraph emited by --emit-symbol-graph or -extract-api options into one unified JSON symbolgraph file.

[dang]

As per https://discourse.llvm.org/t/pull-request-migration-schedule/71595 we should move this review to GitHub to make sure we don't lose track of it.

[Arsenic]

In D158646#4639599, @dang wrote:

As per https://discourse.llvm.org/t/pull-request-migration-schedule/71595 we should move this review to GitHub to make sure we don't lose track of it.

The patch has been moved to GitHub ( https://github.com/llvm/llvm-project/pull/65894 )

clang-tools-extra/CMakeLists.txt

   endif()
 endif()
 
+add_subdirectory(clang-symbolgraph-merger)
 add_subdirectory(clang-apply-replacements)
 add_subdirectory(clang-reorder-fields)
 add_subdirectory(modularize)

clang-tools-extra/clang-symbolgraph-merger/CMakeLists.txt

+include_directories(include)
+add_subdirectory(lib)
+add_subdirectory(tool)

clang-tools-extra/clang-symbolgraph-merger/include/clang-symbolgraph-merger/SymbolGraph.h

+#ifndef SYMBOLGRAPH_H
+#define SYMBOLGRAPH_H
+
+#include "clang/Basic/LangStandard.h"
+#include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/AvailabilityInfo.h"
+#include "clang/ExtractAPI/DeclarationFragments.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
+#include <memory>
+#include <vector>
+
+namespace sgmerger {
+
+// see https://github.com/apple/swift-docc-symbolkit/bdob/main/openapi.yaml
+struct SymbolGraph {
+
+  struct Symbol {
+    Symbol(const llvm::json::Object &SymbolObj);
+
+    llvm::json::Object SymbolObj;
+    std::string AccessLevel;
+    clang::extractapi::APIRecord::RecordKind Kind;
+    clang::extractapi::DeclarationFragments DeclFragments;
+    clang::extractapi::FunctionSignature FunctionSign;
+    std::string Name;
+    std::string USR;
+    clang::extractapi::AvailabilitySet Availabilities;
+    clang::extractapi::DocComment Comments;
+    clang::extractapi::RecordLocation Location;
+    clang::extractapi::DeclarationFragments SubHeadings;
+
+    // underlying type in case of Typedef
+    clang::extractapi::SymbolReference UnderLyingType;
+  };
+
+  SymbolGraph(const llvm::StringRef JSON);
+  llvm::json::Object SymbolGraphObject;
+  llvm::json::Object Metadata;
+  llvm::json::Object Module;
+  std::vector<Symbol> Symbols;
+  llvm::json::Array Relationships;
+};
+
+} // namespace sgmerger
+
+#endif /* SYMBOLGRAPH_H */

clang-tools-extra/clang-symbolgraph-merger/include/clang-symbolgraph-merger/SymbolGraphMerger.h

+#ifndef SYMBOLGRAPHMERGER_H
+#define SYMBOLGRAPHMERGER_H
+
+#include "clang-symbolgraph-merger/SymbolGraph.h"
+#include "clang-symbolgraph-merger/SymbolGraphVisitor.h"
+#include "clang/Basic/LangStandard.h"
+#include "clang/ExtractAPI/API.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/TargetParser/Triple.h"
+#include <memory>
+
+namespace sgmerger {
+
+using SymbolMap = llvm::DenseMap<llvm::StringRef, const SymbolGraph::Symbol *>;
+
+class SymbolGraphMerger : public SymbolGraphVisitor<SymbolGraphMerger> {
+public:
+  SymbolGraphMerger(const clang::SmallVector<SymbolGraph> &SymbolGraphs,
+                    const std::string &ProductName = "")
+      : ProductName(ProductName), Lang(clang::Language::Unknown),
+        SymbolGraphs(SymbolGraphs) {}
+  bool merge();
+  bool visitMetadata(const llvm::json::Object &Metadata);
+  bool visitModule(const llvm::json::Object &Module);
+  bool visitSymbol(const SymbolGraph::Symbol &Symbol);
+  bool visitRelationship(const llvm::json::Object &Relationship);
+
+private:
+  std::string Generator;
+
+  // stuff required to construct the APISet
+  std::string ProductName;
+  llvm::Triple Target;
+  clang::Language Lang;
+
+  SymbolMap PendingSymbols;
+  SymbolMap VisitedSymbols;
+
+  const clang::SmallVector<SymbolGraph> &SymbolGraphs;
+};
+
+} // namespace sgmerger
+
+#endif /* SYMBOLGRAPHMERGER_H */

clang-tools-extra/clang-symbolgraph-merger/include/clang-symbolgraph-merger/SymbolGraphVisitor.h

+#ifndef SYMBOLGRAPHVISITOR_H
+#define SYMBOLGRAPHVISITOR_H
+
+#include "clang-symbolgraph-merger/SymbolGraph.h"
+#include "clang/ExtractAPI/API.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
+#include <memory>
+
+namespace sgmerger {
+
+// Visits a symbol graph obbect and record the extracted info to API
+template <typename Derived> class SymbolGraphVisitor {
+public:
+  bool traverseSymbolGraph(const SymbolGraph &SG) {
+    bool Success = true;
+    Success = (getDerived()->visitMetadata(SG.Metadata) &&
+               getDerived()->visitModule(SG.Module) &&
+               getDerived()->traverseSymbols(SG.Symbols) &&
+               getDerived()->traverseRelationships(SG.Relationships));
+
+    return Success;
+  }
+
+  bool traverseSymbols(const std::vector<SymbolGraph::Symbol> &Symbols) {
+    bool Success = true;
+    for (const auto &Symbol : Symbols)
+      Success = getDerived()->visitSymbol(Symbol);
+    return Success;
+  }
+
+  bool traverseRelationships(const llvm::json::Array &Relationships) {
+    bool Success = true;
+    for (const auto &RelValue : Relationships) {
+      if (const auto *RelObj = RelValue.getAsObject())
+        Success = getDerived()->visitRelationship(*RelObj);
+    }
+    return Success;
+  }
+
+  bool visitMetadata(const llvm::json::Object &Metadata);
+  bool visitModule(const llvm::json::Object &Module);
+  bool visitSymbol(const SymbolGraph::Symbol &Symbol);
+  bool visitRelationship(const llvm::json::Object &Relationship);
+
+  std::unique_ptr<clang::extractapi::APISet> getAPISet() {
+    return std::move(API);
+  }
+
+protected:
+  std::unique_ptr<clang::extractapi::APISet> API;
+
+public:
+  SymbolGraphVisitor(const SymbolGraphVisitor &) = delete;
+  SymbolGraphVisitor(SymbolGraphVisitor &&) = delete;
+  SymbolGraphVisitor &operator=(const SymbolGraphVisitor &) = delete;
+  SymbolGraphVisitor &operator=(SymbolGraphVisitor &&) = delete;
+
+protected:
+  SymbolGraphVisitor() : API(nullptr) {}
+  ~SymbolGraphVisitor() = default;
+
+  Derived *getDerived() { return static_cast<Derived *>(this); };
+};
+
+} // namespace sgmerger
+
+#endif /* SYMBOLGRAPHVISITOR_H */

clang-tools-extra/clang-symbolgraph-merger/lib/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS Support)
+
+add_clang_library(clangSymbolGraphMerger
+  SymbolGraphMerger.cpp
+  SymbolGraph.cpp
+  )
+
+clang_target_link_libraries(clangSymbolGraphMerger
+  PRIVATE
+  clangBasic
+  clangToolingCore
+  clangToolingInclusions
+  clangExtractAPI
+)

clang-tools-extra/clang-symbolgraph-merger/lib/SymbolGraph.cpp

+#include "clang-symbolgraph-merger/SymbolGraph.h"
+#include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/AvailabilityInfo.h"
+#include "clang/ExtractAPI/DeclarationFragments.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/VersionTuple.h"
+#include <cassert>
+#include <cstring>
+#include <memory>
+#include <vector>
+
+using namespace sgmerger;
+using namespace llvm;
+using namespace llvm::json;
+using namespace clang::extractapi;
+
+namespace {
+
+APIRecord::RecordKind getSymbolKind(const Object &Kind) {
+
+  if (auto Identifier = Kind.getString("identifier")) {
+    // Remove danguage prefix
+    auto Id = Identifier->split('.').second;
+    if (Id.equals("func"))
+      return APIRecord::RK_GlobalFunction;
+    if (Id.equals("var"))
+      return APIRecord::RK_GlobalVariable;
+    if (Id.equals("enum.case"))
+      return APIRecord::RK_EnumConstant;
+    if (Id.equals("enum"))
+      return APIRecord::RK_Enum;
+    if (Id.equals("property"))
+      return APIRecord::RK_StructField;
+    if (Id.equals("struct"))
+      return APIRecord::RK_Struct;
+    if (Id.equals("ivar"))
+      return APIRecord::RK_ObjCIvar;
+    if (Id.equals("method"))
+      return APIRecord::RK_ObjCInstanceMethod;
+    if (Id.equals("type.method"))
+      return APIRecord::RK_ObjCClassMethod;
+    if (Id.equals("property"))
+      return APIRecord::RK_ObjCInstanceProperty;
+    if (Id.equals("type.property"))
+      return APIRecord::RK_ObjCClassProperty;
+    if (Id.equals("class"))
+      return APIRecord::RK_ObjCInterface;
+    if (Id.equals("protocod"))
+      return APIRecord::RK_ObjCProtocol;
+    if (Id.equals("macro"))
+      return APIRecord::RK_MacroDefinition;
+    if (Id.equals("typealias"))
+      return APIRecord::RK_Typedef;
+  }
+  return APIRecord::RK_Unknown;
+}
+
+VersionTuple parseVersionTupleFromJSON(const Object *VTObj) {
+  auto Major = VTObj->getInteger("major").value_or(0);
+  auto Minor = VTObj->getInteger("minor").value_or(0);
+  auto Patch = VTObj->getInteger("patch").value_or(0);
+  return VersionTuple(Major, Minor, Patch);
+}
+
+RecordLocation parseSourcePositionFromJSON(const Object *PosObj,
+                                           std::string Filename = "") {
+  assert(PosObj);
+  unsigned Line = PosObj->getInteger("line").value_or(0);
+  unsigned Col = PosObj->getInteger("character").value_or(0);
+  return RecordLocation(Line, Col, Filename);
+}
+
+RecordLocation parseRecordLocationFromJSON(const Object *LocObj) {
+  assert(LocObj);
+
+  std::string Filename(LocObj->getString("uri").value_or(""));
+  // extract file name from URI
+  std::string URIScheme = "file://";
+  if (Filename.find(URIScheme) == 0)
+    Filename.erase(0, URIScheme.length());
+
+  const auto *PosObj = LocObj->getObject("position");
+
+  return parseSourcePositionFromJSON(PosObj, Filename);
+}
+
+DocComment parseCommentsFromJSON(const Object *CommentsObj) {
+  assert(CommentsObj);
+  const auto *LinesArray = CommentsObj->getArray("lines");
+  DocComment Comments;
+  if (LinesArray) {
+    for (auto &LineValue : *LinesArray) {
+      const auto *LineObj = LineValue.getAsObject();
+      auto Text = LineObj->getString("text").value_or("");
+
+      // parse range
+      const auto *BeginLocObj = LineObj->getObject("start");
+      RecordLocation BeginLoc = parseSourcePositionFromJSON(BeginLocObj);
+      const auto *EndLocObj = LineObj->getObject("end");
+      RecordLocation EndLoc = parseSourcePositionFromJSON(EndLocObj);
+      Comments.push_back(CommentLine(Text, BeginLoc, EndLoc));
+    }
+  }
+  return Comments;
+}
+
+AvailabilitySet parseAvailabilitiesFromJSON(const Array *AvailablityArray) {
+  if (AvailablityArray) {
+    SmallVector<AvailabilityInfo, 4> AList;
+    for (auto &AvailablityValue : *AvailablityArray) {
+      const auto *AvailablityObj = AvailablityValue.getAsObject();
+      auto Domain = AvailablityObj->getString("domain").value_or("");
+      auto IntroducedVersion = parseVersionTupleFromJSON(
+          AvailablityObj->getObject("introducedVersion"));
+      auto ObsoletedVersion = parseVersionTupleFromJSON(
+          AvailablityObj->getObject("obsoletedVersion"));
+      auto DeprecatedVersion = parseVersionTupleFromJSON(
+          AvailablityObj->getObject("deprecatedVersion"));
+      AList.emplace_back(AvailabilityInfo(Domain, IntroducedVersion,
+                                          DeprecatedVersion, ObsoletedVersion,
+                                          false));
+    }
+    return AvailabilitySet(AList);
+  }
+  return nullptr;
+}
+
+DeclarationFragments parseDeclFragmentsFromJSON(const Array *FragmentsArray) {
+  DeclarationFragments Fragments;
+  if (FragmentsArray) {
+    for (auto &FragmentValue : *FragmentsArray) {
+      Object FragmentObj = *(FragmentValue.getAsObject());
+      auto Spelling = FragmentObj.getString("spelling").value_or("");
+      auto FragmentKind = DeclarationFragments::parseFragmentKindFromString(
+          FragmentObj.getString("kind").value_or(""));
+      StringRef PreciseIdentifier =
+          FragmentObj.getString("preciseIdentifier").value_or("");
+      Fragments.append(Spelling, FragmentKind, PreciseIdentifier);
+    }
+  }
+  return Fragments;
+}
+
+FunctionSignature parseFunctionSignaturesFromJSON(const Object *SignaturesObj) {
+  FunctionSignature ParsedSignatures;
+  if (SignaturesObj) {
+    // parse return type
+    const auto *RT = SignaturesObj->getArray("returns");
+    ParsedSignatures.setReturnType(parseDeclFragmentsFromJSON(RT));
+
+    // parse function parameters
+    if (const auto *ParamArray = SignaturesObj->getArray("parameters")) {
+      for (auto &Param : *ParamArray) {
+        auto ParamObj = *(Param.getAsObject());
+        auto Name = ParamObj.getString("name").value_or("");
+        auto Fragments = parseDeclFragmentsFromJSON(
+            ParamObj.getArray("declarationFragments"));
+        ParsedSignatures.addParameter(Name, Fragments);
+      }
+    }
+  }
+  return ParsedSignatures;
+}
+
+std::vector<SymbolGraph::Symbol>
+parseSymbolsFromJSON(const Array *SymbolsArray) {
+  std::vector<SymbolGraph::Symbol> SymbolsVector;
+  if (SymbolsArray) {
+    for (const auto &S : *SymbolsArray)
+      if (const auto *Symbol = S.getAsObject())
+        SymbolsVector.push_back(SymbolGraph::Symbol(*Symbol));
+  }
+  return SymbolsVector;
+}
+
+} // namespace
+
+SymbolGraph::Symbol::Symbol(const Object &SymbolObject)
+    : SymbolObj(SymbolObject) {
+
+  AccessLevel = SymbolObj.getString("accessLevel").value_or("unknown");
+  Kind = getSymbolKind(*(SymbolObject.getObject("kind")));
+
+  // parse Doc comments
+  if (const auto *CommentsArray = SymbolObject.getObject("docComment"))
+    Comments = parseCommentsFromJSON(CommentsArray);
+
+  // parse Availabilityinfo
+  if (const auto *AvailabilityArray = SymbolObj.getArray("availability"))
+    Availabilities = parseAvailabilitiesFromJSON(AvailabilityArray);
+
+  // parse declaration fragments
+  if (const auto *FragmentsArray = SymbolObj.getArray("declarationFragments"))
+    DeclFragments = parseDeclFragmentsFromJSON(FragmentsArray);
+
+  // parse function signatures if any
+  if (const auto *FunctionSignObj = SymbolObj.getObject("functionSignature"))
+    FunctionSign = parseFunctionSignaturesFromJSON(FunctionSignObj);
+
+  // parse identifier
+  if (const auto *IDObj = SymbolObj.getObject("identifier"))
+    USR = IDObj->getString("precise").value_or("");
+
+  // parse Location
+  if (const auto *LocObj = SymbolObject.getObject("location"))
+    Location = parseRecordLocationFromJSON(LocObj);
+
+  // parse name and subheadings.
+  if (const auto *NamesObj = SymbolObj.getObject("names")) {
+    Name = NamesObj->getString("title").value_or("");
+    if (const auto *SubHObj = NamesObj->getArray("subHeading"))
+      SubHeadings = parseDeclFragmentsFromJSON(SubHObj);
+  }
+
+  // parse underlying type in case of Typedef
+  auto UType = SymbolObject.getString("type");
+  if (UType.has_value()) {
+    auto UTypeUSR = UType.value();
+    // FIXME: this is a hacky way for Underlying type to be
+    // serialized into the final graph. Get someway to extract the
+    // actual name of the underlying type from USR
+    UnderLyingType = SymbolReference(" ", UTypeUSR);
+  }
+}
+
+SymbolGraph::SymbolGraph(const llvm::StringRef JSON) {
+  Expected<llvm::json::Value> SGValue = llvm::json::parse(JSON);
+  if (SGValue) {
+    assert(SGValue && SGValue->kind() == llvm::json::Value::Object);
+    if (const auto *SGObject = SGValue->getAsObject()) {
+      SymbolGraphObject = *SGObject;
+      if (const auto *MetadataObj = SGObject->getObject("metadata"))
+        Metadata = *MetadataObj;
+      if (const auto *ModuleObj = SGObject->getObject("module"))
+        Module = *ModuleObj;
+      if (const auto *RelArray = SGObject->getArray("relationships"))
+        Relationships = *RelArray;
+
+      Symbols = parseSymbolsFromJSON(SGObject->getArray("symbols"));
+    }
+  }
+}

clang-tools-extra/clang-symbolgraph-merger/lib/SymbolGraphMerger.cpp

+#include "clang-symbolgraph-merger/SymbolGraphMerger.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/AvailabilityInfo.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include <memory>
+
+using namespace llvm;
+using namespace llvm::json;
+using namespace clang;
+using namespace clang::extractapi;
+using namespace sgmerger;
+
+namespace {
+ObjCInstanceVariableRecord::AccessControl
+getAccessFromString(const StringRef AccessLevel) {
+  if (AccessLevel.equals("Private"))
+    return ObjCInstanceVariableRecord::AccessControl::Private;
+  if (AccessLevel.equals("Protected"))
+    return ObjCInstanceVariableRecord::AccessControl::Protected;
+  if (AccessLevel.equals("Public"))
+    return ObjCInstanceVariableRecord::AccessControl::Public;
+  if (AccessLevel.equals("Package"))
+    return ObjCInstanceVariableRecord::AccessControl::Package;
+  return ObjCInstanceVariableRecord::AccessControl::None;
+}
+
+Language getLanguageFromString(const StringRef LangName) {
+  if (LangName.equals("c"))
+    return Language::C;
+  if (LangName.equals("objective-c"))
+    return Language::ObjC;
+  if (LangName.equals("C++"))
+    return Language::CXX;
+
+  return Language::Unknown;
+}
+
+template <typename Lambda>
+bool addWithContainerRecord(APIRecord::RecordKind Kind, APIRecord *TargetRecord,
+                            Lambda Inserter) {
+  switch (Kind) {
+  case APIRecord::RK_ObjCInterface: {
+    if (ObjCInterfaceRecord *Container =
+            dyn_cast_or_null<ObjCInterfaceRecord>(TargetRecord))
+      Inserter(Container);
+  } break;
+  case APIRecord::RK_ObjCProtocol: {
+    if (ObjCProtocolRecord *Container =
+            dyn_cast_or_null<ObjCProtocolRecord>(TargetRecord))
+      Inserter(Container);
+  } break;
+  case APIRecord::RK_ObjCCategory: {
+    if (ObjCCategoryRecord *Container =
+            dyn_cast_or_null<ObjCCategoryRecord>(TargetRecord))
+      Inserter(Container);
+  } break;
+  default:
+    return false;
+  }
+  return true;
+}
+} // namespace
+
+bool SymbolGraphMerger::merge() {
+  for (const auto &SG : SymbolGraphs)
+    traverseSymbolGraph(SG);
+  return true;
+}
+
+bool SymbolGraphMerger::visitMetadata(const Object &Metadata) {
+  // TODO: check if all the symbol graphs are generated form same
+  // generator or not. Info from metadata is currently not needed to
+  // construct the APISet,
+  return true;
+}
+
+bool SymbolGraphMerger::visitModule(const Object &Module) {
+  if (!API) {
+    // If the product name is not provided via command line then extract product
+    // name from SymbolGraph
+    if (ProductName.empty())
+      if (auto NameStr = Module.getString("name"))
+        ProductName = NameStr->str();
+
+    // extract target triple info
+    if (const auto *Platform = Module.getObject("platform")) {
+      auto Arch = Platform->getString("architecture");
+      const auto *OSObj = Platform->getObject("operatingSystem");
+      auto Vendor = Platform->getString("vendor");
+      if (!(Arch && Vendor && OSObj))
+        return false;
+      if (auto OSStr = OSObj->getString("name")) {
+        Target = llvm::Triple(*Arch, *Vendor, *OSStr);
+        return true;
+      }
+    }
+    return false;
+  }
+  return true;
+}
+
+bool SymbolGraphMerger::visitRelationship(const Object &Relationship) {
+  std::string SourceUSR(Relationship.getString("source").value_or(""));
+  std::string TargetUSR(Relationship.getString("target").value_or(""));
+
+  auto *SourceSymbol = PendingSymbols.lookup(SourceUSR);
+  auto *TargetSymbol = VisitedSymbols.lookup(TargetUSR);
+  auto *TargetRecord = API->findRecordForUSR(TargetUSR);
+
+  switch (SourceSymbol->Kind) {
+  case APIRecord::RK_StructField: {
+    if (StructRecord *ParentStruct =
+            dyn_cast_or_null<StructRecord>(TargetRecord))
+      API->addStructField(ParentStruct, SourceSymbol->Name, SourceSymbol->USR,
+                          SourceSymbol->Location, SourceSymbol->Availabilities,
+                          SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                          SourceSymbol->SubHeadings,
+                          false /*IsFromSystemHeader*/);
+  } break;
+  case APIRecord::RK_ObjCIvar: {
+    auto AddRecord = [&](ObjCContainerRecord *Container) {
+      ObjCInstanceVariableRecord::AccessControl Access =
+          getAccessFromString(SourceSymbol->AccessLevel);
+
+      API->addObjCInstanceVariable(
+          Container, SourceSymbol->Name, SourceSymbol->USR,
+          SourceSymbol->Location, SourceSymbol->Availabilities,
+          SourceSymbol->Comments, SourceSymbol->DeclFragments,
+          SourceSymbol->SubHeadings, Access, false /*IsFromSystemHeader*/);
+    };
+    if (!addWithContainerRecord(TargetSymbol->Kind, TargetRecord, AddRecord))
+      return false;
+  } break;
+  case APIRecord::RK_ObjCInstanceMethod: {
+    auto AddRecord = [&](ObjCContainerRecord *Container) {
+      API->addObjCMethod(Container, SourceSymbol->Name, SourceSymbol->USR,
+                         SourceSymbol->Location, SourceSymbol->Availabilities,
+                         SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                         SourceSymbol->SubHeadings, SourceSymbol->FunctionSign,
+                         true
+                         /*IsInstanceMethod*/,
+                         false /*IsFromSystemHeader*/);
+    };
+    if (!addWithContainerRecord(TargetSymbol->Kind, TargetRecord, AddRecord))
+      return false;
+  } break;
+  case APIRecord::RK_EnumConstant: {
+    if (EnumRecord *Enum = dyn_cast_or_null<EnumRecord>(TargetRecord))
+      API->addEnumConstant(Enum, SourceSymbol->Name, SourceSymbol->USR,
+                           SourceSymbol->Location, SourceSymbol->Availabilities,
+                           SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                           SourceSymbol->SubHeadings,
+                           false /*IsFromSystemHeader*/);
+  } break;
+  case APIRecord::RK_ObjCClassMethod: {
+    auto AddRecord = [&](ObjCContainerRecord *Container) {
+      API->addObjCMethod(Container, SourceSymbol->Name, SourceSymbol->USR,
+                         SourceSymbol->Location, SourceSymbol->Availabilities,
+                         SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                         SourceSymbol->SubHeadings, SourceSymbol->FunctionSign,
+                         false
+                         /*IsInstanceMethod*/,
+                         false /*IsFromSystemHeader*/);
+    };
+    if (!addWithContainerRecord(TargetSymbol->Kind, TargetRecord, AddRecord))
+      return false;
+  } break;
+  case APIRecord::RK_ObjCInstanceProperty: {
+    auto AddRecord = [&](ObjCContainerRecord *Container) {
+      API->addObjCProperty(Container, SourceSymbol->Name, SourceSymbol->USR,
+                           SourceSymbol->Location, SourceSymbol->Availabilities,
+                           SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                           SourceSymbol->SubHeadings,
+                           ObjCPropertyRecord::AttributeKind::NoAttr, "", "",
+                           false /*IsOptional*/, true /*IsInstanceProperty*/,
+                           false /*IsFromSystemHeader*/);
+    };
+    if (!addWithContainerRecord(TargetSymbol->Kind, TargetRecord, AddRecord))
+      return false;
+  } break;
+  case APIRecord::RK_ObjCClassProperty: {
+    auto AddRecord = [&](ObjCContainerRecord *Container) {
+      API->addObjCProperty(Container, SourceSymbol->Name, SourceSymbol->USR,
+                           SourceSymbol->Location, SourceSymbol->Availabilities,
+                           SourceSymbol->Comments, SourceSymbol->DeclFragments,
+                           SourceSymbol->SubHeadings,
+                           ObjCPropertyRecord::AttributeKind::NoAttr, "", "",
+                           false /*IsOptional*/, false /*IsInstanceProperty*/,
+                           false /*IsFromSystemHeader*/);
+    };
+    if (!addWithContainerRecord(TargetSymbol->Kind, TargetRecord, AddRecord))
+      return false;
+  } break;
+  case APIRecord::RK_ObjCInterface: {
+    if (TargetRecord) {
+      SymbolReference SuperClass(TargetRecord);
+
+      API->addObjCInterface(
+          SourceSymbol->Name, SourceSymbol->USR, SourceSymbol->Location,
+          SourceSymbol->Availabilities, LinkageInfo(), SourceSymbol->Comments,
+          SourceSymbol->DeclFragments, SourceSymbol->SubHeadings, SuperClass,
+          false /*IsFromSystemHeader*/);
+    }
+  } break;
+  case APIRecord::RK_ObjCCategory: {
+    if (TargetRecord) {
+      SymbolReference Interface(TargetRecord);
+
+      API->addObjCCategory(
+          SourceSymbol->Name, SourceSymbol->USR, SourceSymbol->Location,
+          SourceSymbol->Availabilities, SourceSymbol->Comments,
+          SourceSymbol->DeclFragments, SourceSymbol->SubHeadings, Interface,
+          false /*IsFromSystemHeader*/, true /*IsFromExternalModule*/);
+    }
+  } break;
+  default:
+    return false;
+  }
+  return true;
+}
+
+bool SymbolGraphMerger::visitSymbol(const SymbolGraph::Symbol &Symbol) {
+  // If the APISet is not yet created, then create it ( it's generally
+  // the Language information that is pending uptill this point )
+  if (!API) {
+    if (const auto *Id = Symbol.SymbolObj.getObject("identifier"))
+      if (const auto LangName = Id->getString("interfaceLanguage"))
+        Lang = getLanguageFromString(LangName.value_or(""));
+    API = std::make_unique<extractapi::APISet>(Target, Lang, ProductName);
+  }
+
+  switch (Symbol.Kind) {
+  // TODO: Handle unknown symbols properly
+  case APIRecord::RK_Unknown:
+    break;
+  case APIRecord::RK_GlobalVariable: {
+    API->addGlobalVar(Symbol.Name, Symbol.USR, Symbol.Location,
+                      Symbol.Availabilities, LinkageInfo(), Symbol.Comments,
+                      Symbol.DeclFragments, Symbol.SubHeadings,
+                      false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  case APIRecord::RK_GlobalFunction: {
+    API->addGlobalFunction(
+        Symbol.Name, Symbol.USR, Symbol.Location, Symbol.Availabilities,
+        LinkageInfo(), Symbol.Comments, Symbol.DeclFragments,
+        Symbol.SubHeadings, Symbol.FunctionSign, false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+
+  case APIRecord::RK_Enum: {
+    API->addEnum(Symbol.Name, Symbol.USR, Symbol.Location,
+                 Symbol.Availabilities, Symbol.Comments, Symbol.DeclFragments,
+                 Symbol.SubHeadings, false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  case APIRecord::RK_Struct: {
+    API->addStruct(Symbol.Name, Symbol.USR, Symbol.Location,
+                   Symbol.Availabilities, Symbol.Comments, Symbol.DeclFragments,
+                   Symbol.SubHeadings, false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  case APIRecord::RK_ObjCProtocol: {
+    API->addObjCProtocol(Symbol.Name, Symbol.USR, Symbol.Location,
+                         Symbol.Availabilities, Symbol.Comments,
+                         Symbol.DeclFragments, Symbol.SubHeadings,
+                         false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  case APIRecord::RK_MacroDefinition: {
+    API->addMacroDefinition(Symbol.Name, Symbol.USR, Symbol.Location,
+                            Symbol.DeclFragments, Symbol.SubHeadings,
+                            false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  case APIRecord::RK_Typedef: {
+    API->addTypedef(Symbol.Name, Symbol.USR, Symbol.Location,
+                    Symbol.Availabilities, Symbol.Comments,
+                    Symbol.DeclFragments, Symbol.SubHeadings,
+                    Symbol.UnderLyingType, false /*IsFromSystemHeader*/);
+    VisitedSymbols[Symbol.USR] = &Symbol;
+  } break;
+  default:
+    // Try again when visiting Relationships
+    PendingSymbols[Symbol.USR] = &Symbol;
+  }
+  return true;
+}

clang-tools-extra/clang-symbolgraph-merger/tool/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS Support)
+
+add_clang_tool(clang-symbolgraph-merger SymbolGraphMergerMain.cpp)
+
+target_link_libraries(clang-symbolgraph-merger PRIVATE
+  clangBasic
+  clangFrontend
+  clangSerialization
+  clangTooling
+)
+
+target_link_libraries(clang-symbolgraph-merger PRIVATE
+  clangSymbolGraphMerger)

clang-tools-extra/clang-symbolgraph-merger/tool/SymbolGraphMergerMain.cpp

+#include "clang-symbolgraph-merger/SymbolGraphMerger.h"
+#include "clang-symbolgraph-merger/SymbolGraphVisitor.h"
+#include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Execution.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+#include <system_error>
+
+using namespace clang::tooling;
+using namespace llvm;
+using namespace sgmerger;
+
+namespace {
+
+bool collectSymbolGraphs(StringRef SGDirectory,
+                         SmallVector<SymbolGraph> &SymbolGraphs) {
+  std::error_code Error;
+  for (sys::fs::directory_iterator I(SGDirectory, Error), End; I != End;
+       I.increment(Error)) {
+    if (Error)
+      return false;
+    std::string File(I->path());
+    llvm::ErrorOr<sys::fs::basic_file_status> Status = I->status();
+    if (!Status)
+      return false;
+    sys::fs::file_type Type = Status->type();
+    // If the file is a directory, ignore the name and recurse.
+    if (Type == sys::fs::file_type::directory_file) {
+      if (!collectSymbolGraphs(File, SymbolGraphs))
+        return false;
+      continue;
+    }
+
+    // Ignore all the non json files
+    if (!sys::path::extension(File).equals(".json"))
+      continue;
+
+    // Read the Symbol Graph from the file
+    int FileFD;
+    if (auto OpenError = sys::fs::openFileForRead(File, FileFD))
+      return false;
+
+    llvm::SmallString<256> Payload;
+    if (auto ReadError = sys::fs::readNativeFileToEOF(FileFD, Payload))
+      return false;
+
+    SymbolGraphs.emplace_back(SymbolGraph(Payload));
+    llvm::sys::fs::closeFile(FileFD);
+  }
+  return true;
+}
+
+} // namespace
+
+static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
+// TODO: add more help text
+// static cl::extrahelp MoreHelp("\nMore help text...\n");
+
+static cl::OptionCategory
+    SymbolGraphMergerCategory("clang-symbolgraph-merger options");
+
+static cl::opt<std::string> ProjectName("project-name",
+                                        cl::desc("Name of project."),
+                                        cl::cat(SymbolGraphMergerCategory),
+                                        cl::init(""));
+
+static cl::opt<std::string>
+    OutFile("o", cl::desc("File for outputing generated Symbol Graph."),
+            cl::cat(SymbolGraphMergerCategory), cl::value_desc("filepath"),
+            cl::init("output"));
+
+static cl::opt<std::string>
+    InputDir(cl::Positional, cl::Required, cl::cat(SymbolGraphMergerCategory),
+             cl::value_desc("filepath"),
+             cl::desc("Input directory containing all the SymbolGraphs"));
+
+int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  cl::ParseCommandLineOptions(argc, argv);
+
+  // collect symbol graphs from input dir
+  SmallVector<SymbolGraph> SymbolGraphs;
+  if (collectSymbolGraphs(InputDir, SymbolGraphs)) {
+    llvm::outs() << "found " << SymbolGraphs.size() << " Symbol-graphs in "
+                 << InputDir << "\n";
+
+    // merge them together to form unified APIset
+    llvm::outs() << "merging ...\n";
+    SymbolGraphMerger Merger(SymbolGraphs);
+    if (Merger.merge()) {
+      // serialize the unified symbol graph
+      std::error_code Error;
+      if (!sys::path::extension(OutFile).equals(".json"))
+        OutFile.append(".json");
+
+      llvm::outs() << "serializing...\n";
+      auto OS = std::make_unique<raw_fd_ostream>(OutFile, Error);
+      const auto APISet = Merger.getAPISet();
+      if (APISet) {
+        clang::extractapi::SymbolGraphSerializer SGSerializer(
+            *APISet, clang::extractapi::APIIgnoresList());
+        SGSerializer.serialize(*OS);
+        OS.reset();
+        llvm::outs() << "successfully serialized resultant SymbolGraph to "
+                     << OutFile << "\n";
+      }
+    } else {
+      llvm::errs() << "merge faliure\n";
+      return 1;
+    }
+  } else {
+    llvm::errs() << "some error occured while accessing " << InputDir << "\n";
+    return 1;
+  }
+  return 0;
+}

clang/include/clang/ExtractAPI/AvailabilityInfo.h

 
 public:
   AvailabilitySet(const Decl *Decl);
+  AvailabilitySet(AvailabilityList &List,
+                  bool UnconditionallyDeprecated = false,
+                  bool UnconditionallyUnavailable = false)
+      : Availabilities(List),
+        UnconditionallyDeprecated(UnconditionallyDeprecated),
+        UnconditionallyUnavailable(UnconditionallyUnavailable) {}
   AvailabilitySet() = default;
 
   AvailabilityList::const_iterator begin() const {