[mlir] Generate C++ doc comments for interfaces

[ftynse]

Summary

When emitting the declarations for interface methods defined in ODS,
also emit their descriptions as C++ comments. This makes the
documentation accessible to C++ tooling such as IDEs that offers better
usability than reading it form the .td or the website.

[jpienaar]

mlir/tools/mlir-tblgen/DocGenUtilities.h

17

Is this required?

mlir/tools/mlir-tblgen/OpDocGen.cpp

66

Why not just use raw_indented_ostream and then set prefix before? (this function could also effectively be printReindented if that allowed for specifying a prefix of "/// ")

mlir/tools/mlir-tblgen/OpInterfacesGen.cpp

412

If you switched, then you could also get rid of keeping manual track of lists of spaces everywhere.

[ftynse]

Address review.

[ftynse]

mlir/tools/mlir-tblgen/DocGenUtilities.h

17

Yes. Default-valued argument below needs full type definition.

[ftynse]

Ping.

[jpienaar]

Thanks!

[rriddle]

Nice!

mlir/include/mlir/Support/IndentedOstream.h

public:
   /// Returns DelimitedScope.
   DelimitedScope scope(StringRef open = "", StringRef close = "",
                        bool indent = true) {
     return DelimitedScope(*this, open, close, indent);
   }
 
   /// Prints a string re-indented to the current indent. Re-indents by removing
   /// the leading whitespace from the first non-empty line from every line of
-  /// the string, skipping over empty lines at the start.
-  raw_indented_ostream &printReindented(StringRef str);
+  /// the string, skipping over empty lines at the start. Prefixes each line
+  /// with extraPrefix after the indentation.
+  raw_indented_ostream &printReindented(StringRef str,
+                                        StringRef extraPrefix = "");
 
   /// Increases the indent and returning this raw_indented_ostream.
   raw_indented_ostream &indent() {
     currentIndent += indentSize;
     return *this;
   }
 
   /// Decreases the indent and returning this raw_indented_ostream.
private:
 
   /// Return the current position within the stream, not counting the bytes
   /// currently in the buffer.
   uint64_t current_pos() const override { return os.tell(); }
 
   /// Constant indent added/removed.
   static constexpr int indentSize = 2;
 
-  // Tracker for current indentation.
+  /// Tracker for current indentation.
   int currentIndent = 0;
 
-  // The leading whitespace of the string being printed, if reindent is used.
+  /// The leading whitespace of the string being printed, if reindent is used.
   int leadingWs = 0;
 
-  // Tracks whether at start of line and so indent is required or not.
+  /// The extra prefix to be printed, if reindent is used.
+  StringRef currentExtraPrefix;
+
+  /// Tracks whether at start of line and so indent is required or not.
   bool atStartOfLine = true;
 
-  // The underlying raw_ostream.
+  /// The underlying raw_ostream.
   raw_ostream &os;
 };
 
 } // namespace mlir
 #endif // MLIR_SUPPORT_INDENTEDOSTREAM_H_

mlir/lib/Support/IndentedOstream.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Support/IndentedOstream.h"
 
 using namespace mlir;
 
 raw_indented_ostream &
-mlir::raw_indented_ostream::printReindented(StringRef str) {
+mlir::raw_indented_ostream::printReindented(StringRef str,
+                                            StringRef extraPrefix) {
   StringRef output = str;
   // Skip empty lines.
   while (!output.empty()) {
     auto split = output.split('\n');
     size_t indent = split.first.find_first_not_of(" \t");
     if (indent != StringRef::npos) {
       // Set an initial value.
       leadingWs = indent;
       break;
     }
     output = split.second;
   }
   // Determine the maximum indent.
   StringRef remaining = output;
   while (!remaining.empty()) {
     auto split = remaining.split('\n');
     size_t indent = split.first.find_first_not_of(" \t");
     if (indent != StringRef::npos)
       leadingWs = std::min(leadingWs, static_cast<int>(indent));
     remaining = split.second;
   }
   // Print, skipping the empty lines.
+  std::swap(currentExtraPrefix, extraPrefix);
   *this << output;
+  std::swap(currentExtraPrefix, extraPrefix);
   leadingWs = 0;
   return *this;
 }
 
 void mlir::raw_indented_ostream::write_impl(const char *ptr, size_t size) {
   StringRef str(ptr, size);
   // Print out indented.
   auto print = [this](StringRef str) {
     if (atStartOfLine)
-      os.indent(currentIndent) << str.substr(leadingWs);
+      os.indent(currentIndent) << currentExtraPrefix << str.substr(leadingWs);
     else
       os << str.substr(leadingWs);
   };
 
   while (!str.empty()) {
     size_t idx = str.find('\n');
     if (idx == StringRef::npos) {
       if (!str.substr(leadingWs).empty()) {
         print(str);
         atStartOfLine = false;
       }
       break;
     }
 
     auto split =
         std::make_pair(str.slice(0, idx), str.slice(idx + 1, StringRef::npos));
-    // Print empty new line without spaces if line only has spaces.
-    if (!split.first.ltrim().empty())
+    // Print empty new line without spaces if line only has spaces and no extra
+    // prefix is requested.
+    if (!split.first.ltrim().empty() || !currentExtraPrefix.empty())
       print(split.first);
     os << '\n';
     atStartOfLine = true;
     str = split.second;
   }
 }

mlir/test/mlir-tblgen/op-interface.td

def DeclareMethodsOp : Op<TestDialect, "declare_methods_op",
                           [DeclareOpInterfaceMethods<TestOpInterface>]>;
 
 def DeclareMethodsWithDefaultOp : Op<TestDialect, "declare_methods_op",
       [DeclareOpInterfaceMethods<TestOpInterface, ["default_foo"]>]>;
 
 // DECL-LABEL: TestOpInterfaceInterfaceTraits
 // DECL: class TestOpInterface : public ::mlir::OpInterface<TestOpInterface, detail::TestOpInterfaceInterfaceTraits>
 
+// DECL: /// some function comment
 // DECL: int foo(int input);
 
 // DECL: template<typename ConcreteOp>
 // DECL: int detail::TestOpInterfaceInterfaceTraits::Model<ConcreteOp>::foo
 
 // DECL-LABEL: struct TestOpInterfaceVerifyTrait
 // DECL: verifyTrait
 

mlir/tools/mlir-tblgen/DocGenUtilities.h

 //===- DocGenUtilities.h - MLIR doc gen utilities ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines common utilities for generating documents from tablegen
 // structures.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef MLIR_TOOLS_MLIRTBLGEN_DOCGENUTILITIES_H_
 #define MLIR_TOOLS_MLIRTBLGEN_DOCGENUTILITIES_H_
 
+#include "llvm/ADT/StringRef.h"
+
 namespace llvm {
 class raw_ostream;
-class StringRef;
 } // namespace llvm
 
 namespace mlir {
 namespace tblgen {
 
 // Emit the description by aligning the text to the left per line (e.g.
 // removing the minimum indentation across the block).
 //
 // This expects that the description in the tablegen file is already formatted
 // in a way the user wanted but has some additional indenting due to being
 // nested.
 void emitDescription(llvm::StringRef description, llvm::raw_ostream &os);
 
+// Emit the description as a C++ comment while realigning it.
+void emitDescriptionComment(llvm::StringRef description, llvm::raw_ostream &os,
+                            llvm::StringRef prefix = "");
+
 } // namespace tblgen
 } // namespace mlir
 
 #endif // MLIR_TOOLS_MLIRTBLGEN_DOCGENUTILITIES_H_

mlir/tools/mlir-tblgen/OpDocGen.cpp

 // This expects that the description in the tablegen file is already formatted
 // in a way the user wanted but has some additional indenting due to being
 // nested in the op definition.
 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
   raw_indented_ostream ros(os);
   ros.printReindented(description.rtrim(" \t"));
 }
 
+void mlir::tblgen::emitDescriptionComment(StringRef description,
+                                          raw_ostream &os, StringRef prefix) {
+  if (description.empty())
+    return;
+  raw_indented_ostream ros(os);
+  StringRef trimmed = description.rtrim(" \t");
+  ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
+  if (!trimmed.endswith("\n"))
+    ros << "\n";
+}
+
 // Emits `str` with trailing newline if not empty.
 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
   if (!str.empty()) {
     emitDescription(str, os);
     os << "\n";
   }
 }
 
 /// Emit the given named constraint.
 template <typename T>
 static void emitNamedConstraint(const T &it, raw_ostream &os) {
   if (!it.name.empty())
     os << "| `" << it.name << "`";

mlir/tools/mlir-tblgen/OpInterfacesGen.cpp

struct TypeInterfaceGenerator : public InterfaceGenerator {
   }
 };
 } // namespace
 
 //===----------------------------------------------------------------------===//
 // GEN: Interface definitions
 //===----------------------------------------------------------------------===//
 
+static void emitInterfaceMethodDoc(const InterfaceMethod &method,
+                                   raw_ostream &os, StringRef prefix = "") {
+  if (Optional<StringRef> description = method.getDescription())
+    tblgen::emitDescriptionComment(*description, os, prefix);
+}
+
 static void emitInterfaceDef(const Interface &interface, StringRef valueType,
                              raw_ostream &os) {
   StringRef interfaceName = interface.getName();
   StringRef cppNamespace = interface.getCppNamespace();
   cppNamespace.consume_front("::");
 
   // Insert the method definitions.
   bool isOpInterface = isa<OpInterface>(interface);
   for (auto &method : interface.getMethods()) {
+    emitInterfaceMethodDoc(method, os);
     emitCPPType(method.getReturnType(), os);
     if (!cppNamespace.empty())
       os << cppNamespace << "::";
     os << interfaceName << "::";
     emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
                           /*addConst=*/!isOpInterface);
 
     // Forward to the method on the concrete operation type.
if (method.getName() == "verifyTrait")
       PrintFatalError(
           formatv("'verifyTrait' method cannot be specified as interface "
                   "method for '{0}'; use the 'verify' field instead",
                   interfaceName));
     auto defaultImpl = method.getDefaultImplementation();
     if (!defaultImpl)
       continue;
 
+    emitInterfaceMethodDoc(method, os, "    ");
     os << "    " << (method.isStatic() ? "static " : "");
     emitCPPType(method.getReturnType(), os);
     emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
                           /*addConst=*/!isOpInterface && !method.isStatic());
     os << " {\n      " << tblgen::tgfmt(defaultImpl->trim(), &traitMethodFmt)
        << "\n    }\n";
   }
 
   if (auto verify = interface.getVerify()) {
void InterfaceGenerator::emitInterfaceDecl(const Interface &interface) {
   // Emit a utility wrapper trait class.
   os << llvm::formatv("  template <typename {1}>\n"
                       "  struct Trait : public detail::{0}Trait<{1}> {{};\n",
                       interfaceName, valueTemplate);
 
   // Insert the method declarations.
   bool isOpInterface = isa<OpInterface>(interface);
   for (auto &method : interface.getMethods()) {
+    emitInterfaceMethodDoc(method, os, "  ");
     emitCPPType(method.getReturnType(), os << "  ");
     emitMethodNameAndArgs(method, os, valueType, /*addThisArg=*/false,
                           /*addConst=*/!isOpInterface);
     os << ";\n";
   }
 
   // Emit any extra declarations.
   if (Optional<StringRef> extraDecls = interface.getExtraClassDeclaration())