[libc] Expose APIGenerator.

[PaulkaToast]

Summary

This is split off from D79192 and exposes APIGenerator (renames to APIIndexer) for use in generating the integrations tests.

[sivachandra]

Mostly LGTM with few minor comments.

libc/utils/HdrGen/PublicAPICommand.cpp

185

After renaming the class to APIIndexer, the write method should be decoupled from the class and given a name say writeAPIFromIndex or something. AFAICT, it only needs access to the members you have made public in this patch.

libc/utils/HdrGen/PublicAPICommand.h

28

These names are generic enough that they should be nested in the class. You can make them public if you want.

42

At this point, it probably makes sense to call this class APIIndexer?

68

Can you add comments explaining the difference between the two constructors?

70

Empty line after this?

[PaulkaToast]

libc/utils/HdrGen/PublicAPICommand.h

42

Yeah, the primary function of this is to read the TableGen files so this name makes more sense. (:
This was the same reasoning behind naming the exposed class APIReader from my previous patch.

libc/utils/HdrGen/PublicAPICommand.h

 namespace llvm {
 
 class raw_ostream;
 class Record;
 class RecordKeeper;
 
 } // namespace llvm
 
+using NameToRecordMapping = std::unordered_map<std::string, llvm::Record *>;
+using NameSet = std::unordered_set<std::string>;
+
 namespace llvm_libc {
 
 class PublicAPICommand : public Command {
 public:
   static const char Name[];
 
   void run(llvm::raw_ostream &OS, const ArgVector &Args,
            llvm::StringRef StdHeader, llvm::RecordKeeper &Records,
            const Command::ErrorReporter &Reporter) const override;
 };
 
+class APIGenerator {
+private:
+  llvm::Optional<llvm::StringRef> StdHeader;
+
+  // TableGen classes in spec.td.
+  llvm::Record *NamedTypeClass;
+  llvm::Record *PtrTypeClass;
+  llvm::Record *RestrictedPtrTypeClass;
+  llvm::Record *ConstTypeClass;
+  llvm::Record *StructClass;
+  llvm::Record *StandardSpecClass;
+  llvm::Record *PublicAPIClass;
+
+  bool isaNamedType(llvm::Record *Def);
+  bool isaStructType(llvm::Record *Def);
+  bool isaPtrType(llvm::Record *Def);
+  bool isaConstType(llvm::Record *Def);
+  bool isaRestrictedPtrType(llvm::Record *Def);
+  bool isaStandardSpec(llvm::Record *Def);
+  bool isaPublicAPI(llvm::Record *Def);
+
+  void indexStandardSpecDef(llvm::Record *StandardSpec);
+  void indexPublicAPIDef(llvm::Record *PublicAPI);
+  void index(llvm::RecordKeeper &Records);
+
+public:
+  explicit APIGenerator(llvm::RecordKeeper &Records) : StdHeader(llvm::None) {
+    index(Records);
+  }
+  APIGenerator(llvm::StringRef Header, llvm::RecordKeeper &Records)
+      : StdHeader(Header) {
+    index(Records);
+  }
+
+  // Mapping from names to records defining them.
+  NameToRecordMapping MacroSpecMap;
+  NameToRecordMapping TypeSpecMap;
+  NameToRecordMapping EnumerationSpecMap;
+  NameToRecordMapping FunctionSpecMap;
+  NameToRecordMapping MacroDefsMap;
+  NameToRecordMapping TypeDeclsMap;
+
+  NameSet Structs;
+  NameSet Enumerations;
+  NameSet Functions;
+  NameSet PublicHeaders;
+
+  std::string getTypeAsString(llvm::Record *TypeRecord);
+
+  void write(llvm::raw_ostream &OS);
+};
+
 } // namespace llvm_libc
 
 #endif // LLVM_LIBC_UTILS_HDRGEN_PUBLICAPICOMMAND_H

libc/utils/HdrGen/PublicAPICommand.cpp

if (IndentSize < shortest_indent)
       shortest_indent = IndentSize;
   }
   for (llvm::StringRef L : Lines) {
     if (L.size() >= shortest_indent)
       OS << L.drop_front(shortest_indent) << '\n';
   }
 }
 
-class APIGenerator {
-  llvm::StringRef StdHeader;
+namespace llvm_libc {
 
-  // TableGen classes in spec.td.
-  llvm::Record *NamedTypeClass;
-  llvm::Record *PtrTypeClass;
-  llvm::Record *RestrictedPtrTypeClass;
-  llvm::Record *ConstTypeClass;
-  llvm::Record *StructClass;
-  llvm::Record *StandardSpecClass;
-  llvm::Record *PublicAPIClass;
-
-  using NameToRecordMapping = std::unordered_map<std::string, llvm::Record *>;
-  using NameSet = std::unordered_set<std::string>;
-
-  // Mapping from names to records defining them.
-  NameToRecordMapping MacroSpecMap;
-  NameToRecordMapping TypeSpecMap;
-  NameToRecordMapping EnumerationSpecMap;
-  NameToRecordMapping FunctionSpecMap;
-  NameToRecordMapping MacroDefsMap;
-  NameToRecordMapping TypeDeclsMap;
-
-  NameSet Structs;
-  NameSet Enumerations;
-  NameSet Functions;
-
-  bool isaNamedType(llvm::Record *Def) { return isa(Def, NamedTypeClass); }
+bool APIGenerator::isaNamedType(llvm::Record *Def) {
+  return isa(Def, NamedTypeClass);
+}
 
-  bool isaStructType(llvm::Record *Def) { return isa(Def, StructClass); }
+bool APIGenerator::isaStructType(llvm::Record *Def) {
+  return isa(Def, StructClass);
+}
 
-  bool isaPtrType(llvm::Record *Def) { return isa(Def, PtrTypeClass); }
+bool APIGenerator::isaPtrType(llvm::Record *Def) {
+  return isa(Def, PtrTypeClass);
+}
 
-  bool isaConstType(llvm::Record *Def) { return isa(Def, ConstTypeClass); }
+bool APIGenerator::isaConstType(llvm::Record *Def) {
+  return isa(Def, ConstTypeClass);
+}
 
-  bool isaRestrictedPtrType(llvm::Record *Def) {
-    return isa(Def, RestrictedPtrTypeClass);
-  }
+bool APIGenerator::isaRestrictedPtrType(llvm::Record *Def) {
 
-  bool isaStandardSpec(llvm::Record *Def) {
-    return isa(Def, StandardSpecClass);
-  }
+bool APIGenerator::isaStandardSpec(llvm::Record *Def) {
 
-  bool isaPublicAPI(llvm::Record *Def) { return isa(Def, PublicAPIClass); }
+bool APIGenerator::isaPublicAPI(llvm::Record *Def) {
+  return isa(Def, PublicAPIClass);
+}
 
-  std::string getTypeAsString(llvm::Record *TypeRecord) {
-    if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) {
-      return std::string(TypeRecord->getValueAsString("Name"));
-    } else if (isaPtrType(TypeRecord)) {
-      return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *";
-    } else if (isaConstType(TypeRecord)) {
-      return std::string("const ") +
-             getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType"));
-    } else if (isaRestrictedPtrType(TypeRecord)) {
-      return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) +
-             " *__restrict";
-    } else {
-      llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n");
-    }
-  }
+std::string APIGenerator::getTypeAsString(llvm::Record *TypeRecord) {
 
-  void indexStandardSpecDef(llvm::Record *StandardSpec) {
-    auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
-    for (llvm::Record *HeaderSpec : HeaderSpecList) {
-      if (HeaderSpec->getValueAsString("Name") == StdHeader) {
-        auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
-        // TODO: Trigger a fatal error on duplicate specs.
-        for (llvm::Record *MacroSpec : MacroSpecList)
-          MacroSpecMap[std::string(MacroSpec->getValueAsString("Name"))] =
-              MacroSpec;
+void APIGenerator::indexStandardSpecDef(llvm::Record *StandardSpec) {
+    llvm::StringRef Header = HeaderSpec->getValueAsString("Name");
+    if (!StdHeader.hasValue() || Header == StdHeader) {
+      PublicHeaders.emplace(Header);
 
-        auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
-        for (llvm::Record *TypeSpec : TypeSpecList)
-          TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] =
-              TypeSpec;
+        TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] = TypeSpec;
 
-        auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
-        for (llvm::Record *FunctionSpec : FunctionSpecList) {
-          FunctionSpecMap[std::string(FunctionSpec->getValueAsString("Name"))] =
-              FunctionSpec;
-        }
 
-        auto EnumerationSpecList =
-            HeaderSpec->getValueAsListOfDefs("Enumerations");
-        for (llvm::Record *EnumerationSpec : EnumerationSpecList) {
-          EnumerationSpecMap[std::string(
-              EnumerationSpec->getValueAsString("Name"))] = EnumerationSpec;
-        }
-      }
-    }
-  }
 
-  void indexPublicAPIDef(llvm::Record *PublicAPI) {
-    // While indexing the public API, we do not check if any of the entities
-    // requested is from an included standard. Such a check is done while
-    // generating the API.
-    auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
-    for (llvm::Record *MacroDef : MacroDefList)
-      MacroDefsMap[std::string(MacroDef->getValueAsString("Name"))] = MacroDef;
+void APIGenerator::indexPublicAPIDef(llvm::Record *PublicAPI) {
 
-    auto TypeDeclList = PublicAPI->getValueAsListOfDefs("TypeDeclarations");
-    for (llvm::Record *TypeDecl : TypeDeclList)
-      TypeDeclsMap[std::string(TypeDecl->getValueAsString("Name"))] = TypeDecl;
 
-    auto StructList = PublicAPI->getValueAsListOfStrings("Structs");
-    for (llvm::StringRef StructName : StructList)
-      Structs.insert(std::string(StructName));
 
-    auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions");
-    for (llvm::StringRef FunctionName : FunctionList)
-      Functions.insert(std::string(FunctionName));
 
-    auto EnumerationList = PublicAPI->getValueAsListOfStrings("Enumerations");
-    for (llvm::StringRef EnumerationName : EnumerationList)
-      Enumerations.insert(std::string(EnumerationName));
-  }
 
-  void index(llvm::RecordKeeper &Records) {
-    NamedTypeClass = Records.getClass(NamedTypeClassName);
-    PtrTypeClass = Records.getClass(PtrTypeClassName);
-    RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName);
-    StructClass = Records.getClass(StructTypeClassName);
-    ConstTypeClass = Records.getClass(ConstTypeClassName);
-    StandardSpecClass = Records.getClass(StandardSpecClassName);
-    PublicAPIClass = Records.getClass(PublicAPIClassName);
+void APIGenerator::index(llvm::RecordKeeper &Records) {
 
-    const auto &DefsMap = Records.getDefs();
-    for (auto &Pair : DefsMap) {
-      llvm::Record *Def = Pair.second.get();
-      if (isaStandardSpec(Def))
-        indexStandardSpecDef(Def);
-      if (isaPublicAPI(Def)) {
-        if (Def->getValueAsString("HeaderName") == StdHeader)
-          indexPublicAPIDef(Def);
-      }
-    }
-  }
+      if (!StdHeader.hasValue() ||
+          Def->getValueAsString("HeaderName") == StdHeader)
 
-public:
-  APIGenerator(llvm::StringRef Header, llvm::RecordKeeper &Records)
-      : StdHeader(Header) {
-    index(Records);
-  }
-
-  void write(llvm::raw_ostream &OS) {
-    for (auto &Pair : MacroDefsMap) {
-      const std::string &Name = Pair.first;
-      if (MacroSpecMap.find(Name) == MacroSpecMap.end())
-        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
+void APIGenerator::write(llvm::raw_ostream &OS) {
 
-      llvm::Record *MacroDef = Pair.second;
-      dedentAndWrite(MacroDef->getValueAsString("Defn"), OS);
 
-      OS << '\n';
-    }
 
-    for (auto &Pair : TypeDeclsMap) {
-      const std::string &Name = Pair.first;
-      if (TypeSpecMap.find(Name) == TypeSpecMap.end())
-        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
 
-      llvm::Record *TypeDecl = Pair.second;
-      dedentAndWrite(TypeDecl->getValueAsString("Decl"), OS);
 
-      OS << '\n';
-    }
 
-    if (Enumerations.size() != 0)
-      OS << "enum {" << '\n';
-    for (const auto &Name : Enumerations) {
-      if (EnumerationSpecMap.find(Name) == EnumerationSpecMap.end())
-        llvm::PrintFatalError(
-            Name + " is not listed as an enumeration in any standard spec.\n");
 
-      llvm::Record *EnumerationSpec = EnumerationSpecMap[Name];
-      OS << "  " << EnumerationSpec->getValueAsString("Name");
-      auto Value = EnumerationSpec->getValueAsString("Value");
-      if (Value == "__default__") {
-        OS << ",\n";
-      } else {
-        OS << " = " << Value << ",\n";
-      }
-    }
-    if (Enumerations.size() != 0)
-      OS << "};\n\n";
 
-    OS << "__BEGIN_C_DECLS\n\n";
-    for (auto &Name : Functions) {
-      if (FunctionSpecMap.find(Name) == FunctionSpecMap.end())
-        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
 
-      llvm::Record *FunctionSpec = FunctionSpecMap[Name];
-      llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return");
-      llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType");
 
-      OS << getTypeAsString(ReturnType) << " " << Name << "(";
 
-      auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args");
-      for (size_t i = 0; i < ArgsList.size(); ++i) {
-        llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType");
-        OS << getTypeAsString(ArgType);
-        if (i < ArgsList.size() - 1)
-          OS << ", ";
-      }
 
-      OS << ");\n\n";
-    }
-    OS << "__END_C_DECLS\n";
-  }
-};
-
-namespace llvm_libc {
 
 void writePublicAPI(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {}
 
 const char PublicAPICommand::Name[] = "public_api";
 
 void PublicAPICommand::run(llvm::raw_ostream &OS, const ArgVector &Args,
                            llvm::StringRef StdHeader,
                            llvm::RecordKeeper &Records,