[clang-doc] Implement a (simple) Markdown generator

[juliehockett]

Summary

Implementing a simple Markdown generator from the emitted bitcode summary of declarations. Very primitive at this point, but will be expanded. Currently emits an .md file for each class and namespace, listing its contents.

For a more detailed overview of the tool, see the design document on the mailing list: http://lists.llvm.org/pipermail/cfe-dev/2017-December/056203.html

[lebedev.ri]

It will be good to have the tests for generators.

clang-doc/generators/Generators.h

28

Is this code (and the code in two parent Differentials) formatted with clang-format?
This line is certainly longer than 80 columns.

[juliehockett]

Updating based on parent revision changes -- still rough, will continue to improve.

[Eugene.Zelenko]

Please run Clang-format and Clang-tidy modernize.

clang-doc/generators/Generators.h

46

Please remove semicolon after constructor body. Please enable CLang's -Wextra-semi.

47

Please use = default;

[ioeric]

clang-doc/generators/Generators.h

44

Could you add high-level comment on what this does? This seems to build up some directory structure and write different infos into different sub-directories. Could you elaborate a bit on this?

[rja]

clang-doc/generators/MDGenerator.cpp

58

remove commented code?

[juliehockett]

Updating to reflect changes to the framework (and make it work).

[Eugene.Zelenko]

clang-tools-extra/clang-doc/MDGenerator.cpp

33 ↗

(On Diff #152135)

return {};

[juliehockett]

Updating based on recent landed patches & updated tests

[leonardchan]

clang-tools-extra/clang-doc/MDGenerator.cpp

79 ↗

(On Diff #159091)

nit: make Num unsigned since the std::string fill ctor takes an unsigned value

[juliehockett]

Reopening because it was reverted and I haven't had time to look into it yet

clang-doc/CMakeLists.txt

add_clang_library(clangDoc
   clangBasic
   clangFrontend
   clangLex
   clangTooling
   clangToolingCore
   )
 
 add_subdirectory(tool)
+add_subdirectory(generators)

clang-doc/generators/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS support)
+
+add_clang_library(clangDocGenerators
+  GeneratorBase.cpp
+  MDGenerator.cpp
+
+  LINK_LIBS
+  clangDoc
+  )

clang-doc/generators/GeneratorBase.cpp

+//===-- GeneratorBase.cpp - ClangDoc Generator ------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../ClangDocRepresentation.h"
+#include "../ClangDocBinary.h"
+#include "Generators.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace clang {
+namespace doc {
+
+std::unique_ptr<InfoSet> Generator::readIRFile(StringRef FileName) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Out = MemoryBuffer::getFile(FileName);
+  if (std::error_code Err = Out.getError()) {
+    errs() << "Error reading " << FileName << ": " << Err.message()
+           << "\n";
+    return nullptr;
+  }
+  ClangDocBinaryReader Reader;
+  auto IS = Reader.readBitstream(Out.get()->getBuffer());
+  if (IS) return std::move(IS);
+  return nullptr;
+}
+
+bool Generator::buildDirectory(StringRef Dir) {
+  std::error_code DirectoryStatus = sys::fs::create_directories(Dir);
+  if (DirectoryStatus != OK) {
+    errs() << "Unable to create documentation directories.\n";
+    return false;
+  }
+  return true;
+}
+
+bool Generator::removeExistingDirectory(StringRef Dir) {
+  std::error_code DirectoryStatus = sys::fs::remove_directories(Dir);
+  if (DirectoryStatus != OK) {
+    errs() << "Unable to remove existing documentation directories.\n";
+    return false;
+  }
+  return true;
+}
+
+} // doc
+} // clang

clang-doc/generators/Generators.h

+//===-- Generators.h - ClangDoc Generator ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H  
+
+#include "../ClangDocRepresentation.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+
+using namespace llvm;
+
+namespace clang {
+namespace doc {
+
+class Generator {
+public:
+  Generator(std::unique_ptr<InfoSet> &IS, StringRef Root, StringRef Format) : IS(IS), Root(Root), Format(Format) {};
+  virtual ~Generator() {};
+  
+  virtual int generate() = 0;
+  
+protected:
+  virtual std::unique_ptr<InfoSet> readIRFile(StringRef Filename);
+  virtual bool buildDirectory(StringRef Dir);
+  virtual bool removeExistingDirectory(StringRef Dir);
+
+  std::unique_ptr<InfoSet> &IS;
+  std::string Root;
+  std::string Format;
+  std::error_code OK;
+};
+
+class MDGenerator : public Generator {
+public:
+  MDGenerator(std::unique_ptr<InfoSet> &IS, StringRef Root, StringRef Format) : Generator(IS, Root, Format) {};
+  virtual ~MDGenerator() {};
+  
+  virtual int generate();
+  
+private:
+  int buildDirTree();
+  
+  bool writeNamespaces();
+  bool writeClasses();
+  bool writeNamespacePage(const NamespaceInfo &I);
+  bool writeClassPage(const RecordInfo &I);
+  
+  void writeRecordSummary(const RecordInfo &I, raw_ostream &OS);
+  void writeMember(const NamedType &N, raw_ostream &OS);
+  void writeEnum(const EnumInfo &I, raw_ostream &OS);
+  void writeFunction(const FunctionInfo &I, raw_ostream &OS);
+  void writeDescription(const CommentInfo &I, raw_ostream &OS);
+  void writeFileDefinition(int LineNumber, StringRef DefFile, raw_ostream &OS);
+
+  void sortRecordInfos(llvm::SmallVector<const RecordInfo *, 8> Records);
+
+  void writeLine(StringRef Text, raw_ostream &OS);
+  void writeBlankLine(raw_ostream &OS);
+  std::string genItalic(const Twine &Text);
+  std::string genEmphasis(const Twine &Text);
+  std::string genStrikethrough(const Twine &Text);
+  std::string genH1(const Twine &Text);
+  std::string genH2(const Twine &Text);
+  std::string genH3(const Twine &Text);
+  std::string genH4(const Twine &Text);
+  std::string genH5(const Twine &Text);
+  std::string genH6(const Twine &Text);
+  std::string genTableCell(const Twine &Text);
+  std::string genUnorderedItem(const Twine &Text, int Indent=0);
+  std::string genOrderedItem(const Twine &Text, int Pos, int Indent=0);
+  std::string genLink(const Twine &Text, const Twine &Link);
+  std::string genLinkWithTooltip(const Twine &Text, const Twine &Link, const Twine &Tooltip);
+  std::string genInlineCode(const Twine &Text);
+
+  SmallString<128> NamespacesPath;
+  SmallString<128> ClassPath;
+};
+
+class GeneratorFactory {
+public: 
+  static std::unique_ptr<Generator> create(std::unique_ptr<InfoSet> &IS, StringRef Root, StringRef Format) {
+    if (Format == "md") return llvm::make_unique<MDGenerator>(IS, Root, Format);
+    
+    errs() << "Unsupported documentation format.\n";
+    return nullptr;
+  }
+};
+
+}
+}
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H

clang-doc/generators/MDGenerator.cpp

+//===-- MDGenerator.cpp - Markdown Generator --------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../ClangDocRepresentation.h"
+#include "Generators.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace clang {
+namespace doc {
+
+int MDGenerator::generate() {
+  if (!buildDirTree() || !writeNamespaces() || !writeClasses()) return 1;
+  return 0;
+}
+
+// File creation and I/O
+
+int MDGenerator::buildDirTree() {
+  removeExistingDirectory(Root);
+  sys::path::native(Root, NamespacesPath);
+  sys::path::append(NamespacesPath, "namespaces");
+  sys::path::native(Root, ClassPath);
+  sys::path::append(ClassPath, "classes");
+  return buildDirectory(NamespacesPath) && buildDirectory(ClassPath);
+}
+
+// Documentation generation
+
+bool MDGenerator::writeNamespaces() {
+  // TODO: Generate summary page
+  bool Success = true;
+  for (const auto &I : IS->getNamespaceInfos()) Success = writeNamespacePage(I);
+
+  return Success;
+}
+
+bool MDGenerator::writeClasses() {
+  bool Success = true;
+  for (const auto &I : IS->getRecordInfos()) {
+    if (I.TagType == TagTypeKind::TTK_Class) Success = writeClassPage(I);
+  }
+  return Success;
+}
+
+bool MDGenerator::writeNamespacePage(const NamespaceInfo &I) {
+  SmallString<128> Path;
+  sys::path::native(NamespacesPath, Path);
+  sys::path::append(Path, I.FullyQualifiedName + ".md");
+  std::error_code OutErrorInfo;
+  raw_fd_ostream OS(Path, OutErrorInfo, sys::fs::F_None);
+  if (OutErrorInfo != OK) {
+    errs() << "Error creating class file.\n";
+    return false;
+  }
+
+  writeLine(genH1("namespace " + I.SimpleName), OS);
+  writeBlankLine(OS);
+
+  // TODO: Write comment description
+
+  // TODO: Write subnamespaces
+
+  // Write functions.
+  bool wroteFunctionHeader = false;
+  for (const auto &F : IS->getFunctionInfos()) {
+    if (F.Namespace == I.SimpleName) {
+      if (!wroteFunctionHeader) {
+        wroteFunctionHeader = true;
+        writeLine(genH2("Functions"), OS);
+      }
+      writeFunction(F, OS);
+      writeBlankLine(OS);
+    }
+  }
+
+  // Fetch and sort records.
+  llvm::SmallVector<const RecordInfo *, 8> Structs;
+  llvm::SmallVector<const RecordInfo *, 8> Classes;
+  llvm::SmallVector<const RecordInfo *, 8> Unions;
+  for (const auto &R : IS->getRecordInfos()) {
+    if (R.Namespace == I.SimpleName) {
+      switch (R.TagType) {
+        case TagTypeKind::TTK_Class:
+          Classes.push_back(&R);
+          break;
+        case TagTypeKind::TTK_Struct:
+          Structs.push_back(&R);
+          break;
+        case TagTypeKind::TTK_Union:
+          Unions.push_back(&R);
+          break;
+        default:
+          continue;
+      }
+    }
+  }
+
+  // Write structs.
+  bool wroteHeader = false;
+  sortRecordInfos(Structs);
+  for (const auto &S : Structs) {
+    if (!wroteHeader) {
+      wroteHeader = true;
+      writeLine(genH2("Structs"), OS);
+    }
+    writeRecordSummary(*S, OS);
+    writeBlankLine(OS);
+  }
+
+  // Write classes.
+  wroteHeader = false;
+  sortRecordInfos(Classes);
+  for (const auto &C : Classes) {
+    if (!wroteHeader) {
+      wroteHeader = true;
+      writeLine(genH2("Classes"), OS);
+    }
+    writeRecordSummary(*C, OS);
+    writeBlankLine(OS);
+  }
+
+  // Write unions.
+  wroteHeader = false;
+  sortRecordInfos(Unions);
+  for (const auto &U : Unions) {
+    if (!wroteHeader) {
+      wroteHeader = true;
+      writeLine(genH2("Unions"), OS);
+    }
+    writeRecordSummary(*U, OS);
+    writeBlankLine(OS);
+  }
+
+  // Write enums.
+  wroteHeader = false;
+  for (const auto &E : IS->getEnumInfos()) {
+    if (E.Namespace == I.FullyQualifiedName) {
+      if (!wroteHeader) {
+        wroteHeader = true;
+        writeLine(genH2("Enums"), OS);
+      }
+      writeEnum(E, OS);
+      writeBlankLine(OS);
+    }
+  }
+  return true;
+}
+
+bool MDGenerator::writeClassPage(const RecordInfo &I) {
+  SmallString<128> Path;
+  sys::path::native(ClassPath, Path);
+  sys::path::append(Path, I.FullyQualifiedName + ".md");
+  std::error_code OutErrorInfo;
+  raw_fd_ostream OS(Path, OutErrorInfo, sys::fs::F_None);
+  if (OutErrorInfo != OK) {
+    errs() << "Error creating class file.\n";
+    return false;
+  }
+
+  writeLine(genH1("class " + I.SimpleName), OS);
+  writeFileDefinition(I.DefLoc.LineNumber, I.DefLoc.Filename, OS);
+  writeBlankLine(OS);
+
+  // TODO: Write comment description
+
+  bool wroteMethodHeader = false;
+  for (const auto &F : IS->getFunctionInfos()) {
+    if (F.Parent == I.SimpleName) {
+      if (!wroteMethodHeader) {
+        wroteMethodHeader = true;
+        writeLine(genH2("Methods"), OS);
+      }
+      writeFunction(F, OS);
+      writeBlankLine(OS);
+    }
+  }
+
+  if (!I.Members.empty()) writeLine(genH2("Members"), OS);
+  for (const auto &M : I.Members) {
+    writeMember(M, OS);
+    writeBlankLine(OS);
+  }
+
+  bool wroteEnumHeader = false;
+  writeLine(genH2("Enums"), OS);
+  for (const auto &E : IS->getEnumInfos()) {
+    if (E.Namespace == I.FullyQualifiedName) {
+      if (!wroteEnumHeader) {
+        wroteEnumHeader = true;
+        writeLine(genH2("Methods"), OS);
+      }
+      writeEnum(E, OS);
+      writeBlankLine(OS);
+    }
+  }
+
+  OS.close();
+  return true;
+}
+
+void MDGenerator::writeRecordSummary(const RecordInfo &I, raw_ostream &OS) {
+  // TODO: Add brief comment.
+  switch (I.TagType) {
+    case TagTypeKind::TTK_Class:
+      writeLine("class " + I.SimpleName, OS);
+      break;
+    case TagTypeKind::TTK_Struct:
+      writeLine("struct " + I.SimpleName, OS);
+      break;
+    case TagTypeKind::TTK_Union:
+      writeLine("union " + I.SimpleName, OS);
+      break;
+    default:
+      return;
+  }
+}
+
+void MDGenerator::writeFunction(const FunctionInfo &I, raw_ostream &OS) {
+  std::string Params;
+  llvm::raw_string_ostream Stream(Params);
+  for (const auto &N : I.Params) Stream << N.Type + " " + N.Name << ", ";
+  writeLine(I.ReturnType.Type + " " + I.SimpleName + "(" + Params + ")", OS);
+  // TODO: Write comment description
+  writeFileDefinition(I.DefLoc.LineNumber, I.DefLoc.Filename, OS);
+}
+
+void MDGenerator::writeEnum(const EnumInfo &I, raw_ostream &OS) {
+  writeLine(genTableCell(I.SimpleName), OS);
+  writeLine(genTableCell("--"), OS);
+  for (const auto &M : I.Members) writeLine(genTableCell(M.Type), OS);
+  // TODO: Write comment description
+  writeFileDefinition(I.DefLoc.LineNumber, I.DefLoc.Filename, OS);
+}
+
+void MDGenerator::writeMember(const NamedType &N, raw_ostream &OS) {
+  writeLine(N.Type + " " + N.Name, OS);
+  // TODO: Write comment description
+  // TODO: Add location to member definition
+}
+
+void MDGenerator::writeDescription(const CommentInfo &I, raw_ostream &OS) {}
+
+void MDGenerator::writeFileDefinition(int LineNumber, StringRef DefFile,
+                                      raw_ostream &OS) {
+  writeLine(genItalic("Defined at line " + std::to_string(LineNumber) + " of " +
+                      DefFile + "."),
+            OS);
+}
+
+// Helper functions
+
+void MDGenerator::sortRecordInfos(
+    llvm::SmallVector<const RecordInfo *, 8> Records) {
+  std::sort(Records.begin(), Records.end(),
+            [](const RecordInfo *A, const RecordInfo *B) {
+              return A->FullyQualifiedName.compare(B->FullyQualifiedName) > 0
+                         ? false
+                         : true;
+            });
+}
+
+// Markdown generation
+
+void MDGenerator::writeLine(StringRef Text, raw_ostream &OS) {
+  OS << Text << "\n";
+}
+
+void MDGenerator::writeBlankLine(raw_ostream &OS) { OS << "\n"; }
+
+std::string MDGenerator::genItalic(const Twine &Text) {
+  return "*" + Text.str() + "*";
+}
+
+std::string MDGenerator::genEmphasis(const Twine &Text) {
+  return "**" + Text.str() + "**";
+}
+
+std::string MDGenerator::genStrikethrough(const Twine &Text) {
+  return "~~" + Text.str() + "~~";
+}
+
+std::string MDGenerator::genH1(const Twine &Text) { return "# " + Text.str(); }
+
+std::string MDGenerator::genH2(const Twine &Text) { return "## " + Text.str(); }
+
+std::string MDGenerator::genH3(const Twine &Text) {
+  return "### " + Text.str();
+}
+
+std::string MDGenerator::genH4(const Twine &Text) {
+  return "#### " + Text.str();
+}
+
+std::string MDGenerator::genH5(const Twine &Text) {
+  return "##### " + Text.str();
+}
+
+std::string MDGenerator::genH6(const Twine &Text) {
+  return "###### " + Text.str();
+}
+
+std::string MDGenerator::genTableCell(const Twine &Text) {
+  return "| " + Text.str() + " |";
+}
+
+std::string MDGenerator::genUnorderedItem(const Twine &Text, int Indent) {
+  return ' ' * Indent + "- " + Text.str();
+}
+
+std::string MDGenerator::genOrderedItem(const Twine &Text, int Pos,
+                                        int Indent) {
+  return ' ' * Indent + Pos + ". " + Text.str();
+}
+
+std::string MDGenerator::genLink(const Twine &Text, const Twine &Link) {
+  return "[" + Text.str() + "](" + Link.str() + ")";
+}
+
+std::string MDGenerator::genLinkWithTooltip(const Twine &Text,
+                                            const Twine &Link,
+                                            const Twine &Tooltip) {
+  return "[" + Text.str() + "](" + Link.str() + " \"" + Tooltip.str() + "\")";
+}
+
+std::string MDGenerator::genInlineCode(const Twine &Text) {
+  return "`" + Text.str() + "`";
+}
+
+}  // namespace doc
+}  // namespace clang

clang-doc/tool/CMakeLists.txt

 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..)
 
 add_clang_executable(clang-doc
   ClangDocMain.cpp
   )
 
 target_link_libraries(clang-doc
   PRIVATE
   clangAST
   clangASTMatchers
   clangBasic
   clangFrontend
   clangDoc
+  clangDocGenerators
   clangTooling
   clangToolingCore
   )

clang-doc/tool/ClangDocMain.cpp

 //===-- ClangDocMain.cpp - Clangdoc -----------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include <string>
 #include "ClangDoc.h"
 #include "ClangDocBinary.h"
 #include "ClangDocReducer.h"
+#include "generators/Generators.h"
 #include "clang/AST/AST.h"
 #include "clang/AST/Decl.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchersInternal.h"
 #include "clang/Driver/Options.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Execution.h"
if (DumpResult) {
     if (OutErrorInfo != OK) {
       errs() << "Error opening documentation file.\n";
       return 1;
     }
     OS << Buffer;
     OS.close();
   }
 
-  return 0;
+  errs() << "Generating docs...\n";
+  SmallString<128> DocsRootPath;
+  sys::path::native(OutDirectory, DocsRootPath);
+  sys::path::append(DocsRootPath, Format);
+  std::error_code DirectoryStatus = sys::fs::create_directories(DocsRootPath);
+  if (DirectoryStatus != OK) {
+    errs() << "Unable to create documentation directories.\n";
+    return 1;
+  }
+  std::unique_ptr<doc::Generator> G =
+      doc::GeneratorFactory::create(Infos, DocsRootPath, Format);
+  if (!G) return 1;
+  return G->generate();
 }